tools/perf/util/cs-etm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015-2018 Linaro Limited.
   4  *
   5  * Author: Tor Jeremiassen <tor@ti.com>
   6  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/err.h>
  11 #include <linux/kernel.h>
  12 #include <linux/log2.h>
  13 #include <linux/types.h>
  14 #include <linux/zalloc.h>
  15
  16 #include <opencsd/ocsd_if_types.h>
  17 #include <stdlib.h>
  18
  19 #include "auxtrace.h"
  20 #include "color.h"
  21 #include "cs-etm.h"
  22 #include "cs-etm-decoder/cs-etm-decoder.h"
  23 #include "debug.h"
  24 #include "dso.h"
  25 #include "evlist.h"
  26 #include "intlist.h"
  27 #include "machine.h"
  28 #include "map.h"
  29 #include "perf.h"
  30 #include "symbol.h"
  31 #include "tool.h"
  32 #include "thread.h"
  33 #include "thread-stack.h"
  34 #include <tools/libc_compat.h>
  35 #include "util.h"
  36
  37 #define MAX_TIMESTAMP (~0ULL)
  38
  39 struct cs_etm_auxtrace {
  40         struct auxtrace auxtrace;
  41         struct auxtrace_queues queues;
  42         struct auxtrace_heap heap;
  43         struct itrace_synth_opts synth_opts;
  44         struct perf_session *session;
  45         struct machine *machine;
  46         struct thread *unknown_thread;
  47
  48         u8 timeless_decoding;
  49         u8 snapshot_mode;
  50         u8 data_queued;
  51         u8 sample_branches;
  52         u8 sample_instructions;
  53
  54         int num_cpu;
  55         u32 auxtrace_type;
  56         u64 branches_sample_type;
  57         u64 branches_id;
  58         u64 instructions_sample_type;
  59         u64 instructions_sample_period;
  60         u64 instructions_id;
  61         u64 **metadata;
  62         u64 kernel_start;
  63         unsigned int pmu_type;
  64 };
  65
  66 struct cs_etm_traceid_queue {
  67         u8 trace_chan_id;
  68         pid_t pid, tid;
  69         u64 period_instructions;
  70         size_t last_branch_pos;
  71         union perf_event *event_buf;
  72         struct thread *thread;
  73         struct branch_stack *last_branch;
  74         struct branch_stack *last_branch_rb;
  75         struct cs_etm_packet *prev_packet;
  76         struct cs_etm_packet *packet;
  77         struct cs_etm_packet_queue packet_queue;
  78 };
  79
  80 struct cs_etm_queue {
  81         struct cs_etm_auxtrace *etm;
  82         struct cs_etm_decoder *decoder;
  83         struct auxtrace_buffer *buffer;
  84         unsigned int queue_nr;
  85         u8 pending_timestamp;
  86         u64 offset;
  87         const unsigned char *buf;
  88         size_t buf_len, buf_used;
  89         /* Conversion between traceID and index in traceid_queues array */
  90         struct intlist *traceid_queues_list;
  91         struct cs_etm_traceid_queue **traceid_queues;
  92 };
  93
  94 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
  95 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm);
  96 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
  97                                            pid_t tid);
  98 static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
  99 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
 100
 101 /* PTMs ETMIDR [11:8] set to b0011 */
 102 #define ETMIDR_PTM_VERSION 0x00000300
 103
 104 /*
 105  * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
 106  * work with.  One option is to modify to auxtrace_heap_XYZ() API or simply
 107  * encode the etm queue number as the upper 16 bit and the channel as
 108  * the lower 16 bit.
 109  */
 110 #define TO_CS_QUEUE_NR(queue_nr, trace_id_chan) \
 111                       (queue_nr << 16 | trace_chan_id)
 112 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
 113 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
 114
 115 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
 116 {
 117         etmidr &= ETMIDR_PTM_VERSION;
 118
 119         if (etmidr == ETMIDR_PTM_VERSION)
 120                 return CS_ETM_PROTO_PTM;
 121
 122         return CS_ETM_PROTO_ETMV3;
 123 }
 124
 125 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
 126 {
 127         struct int_node *inode;
 128         u64 *metadata;
 129
 130         inode = intlist__find(traceid_list, trace_chan_id);
 131         if (!inode)
 132                 return -EINVAL;
 133
 134         metadata = inode->priv;
 135         *magic = metadata[CS_ETM_MAGIC];
 136         return 0;
 137 }
 138
 139 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
 140 {
 141         struct int_node *inode;
 142         u64 *metadata;
 143
 144         inode = intlist__find(traceid_list, trace_chan_id);
 145         if (!inode)
 146                 return -EINVAL;
 147
 148         metadata = inode->priv;
 149         *cpu = (int)metadata[CS_ETM_CPU];
 150         return 0;
 151 }
 152
 153 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
 154                                               u8 trace_chan_id)
 155 {
 156         /*
 157          * Wnen a timestamp packet is encountered the backend code
 158          * is stopped so that the front end has time to process packets
 159          * that were accumulated in the traceID queue.  Since there can
 160          * be more than one channel per cs_etm_queue, we need to specify
 161          * what traceID queue needs servicing.
 162          */
 163         etmq->pending_timestamp = trace_chan_id;
 164 }
 165
 166 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
 167                                       u8 *trace_chan_id)
 168 {
 169         struct cs_etm_packet_queue *packet_queue;
 170
 171         if (!etmq->pending_timestamp)
 172                 return 0;
 173
 174         if (trace_chan_id)
 175                 *trace_chan_id = etmq->pending_timestamp;
 176
 177         packet_queue = cs_etm__etmq_get_packet_queue(etmq,
 178                                                      etmq->pending_timestamp);
 179         if (!packet_queue)
 180                 return 0;
 181
 182         /* Acknowledge pending status */
 183         etmq->pending_timestamp = 0;
 184
 185         /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
 186         return packet_queue->timestamp;
 187 }
 188
 189 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
 190 {
 191         int i;
 192
 193         queue->head = 0;
 194         queue->tail = 0;
 195         queue->packet_count = 0;
 196         for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
 197                 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
 198                 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
 199                 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
 200                 queue->packet_buffer[i].instr_count = 0;
 201                 queue->packet_buffer[i].last_instr_taken_branch = false;
 202                 queue->packet_buffer[i].last_instr_size = 0;
 203                 queue->packet_buffer[i].last_instr_type = 0;
 204                 queue->packet_buffer[i].last_instr_subtype = 0;
 205                 queue->packet_buffer[i].last_instr_cond = 0;
 206                 queue->packet_buffer[i].flags = 0;
 207                 queue->packet_buffer[i].exception_number = UINT32_MAX;
 208                 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
 209                 queue->packet_buffer[i].cpu = INT_MIN;
 210         }
 211 }
 212
 213 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
 214 {
 215         int idx;
 216         struct int_node *inode;
 217         struct cs_etm_traceid_queue *tidq;
 218         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 219
 220         intlist__for_each_entry(inode, traceid_queues_list) {
 221                 idx = (int)(intptr_t)inode->priv;
 222                 tidq = etmq->traceid_queues[idx];
 223                 cs_etm__clear_packet_queue(&tidq->packet_queue);
 224         }
 225 }
 226
 227 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 228                                       struct cs_etm_traceid_queue *tidq,
 229                                       u8 trace_chan_id)
 230 {
 231         int rc = -ENOMEM;
 232         struct auxtrace_queue *queue;
 233         struct cs_etm_auxtrace *etm = etmq->etm;
 234
 235         cs_etm__clear_packet_queue(&tidq->packet_queue);
 236
 237         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
 238         tidq->tid = queue->tid;
 239         tidq->pid = -1;
 240         tidq->trace_chan_id = trace_chan_id;
 241
 242         tidq->packet = zalloc(sizeof(struct cs_etm_packet));
 243         if (!tidq->packet)
 244                 goto out;
 245
 246         tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
 247         if (!tidq->prev_packet)
 248                 goto out_free;
 249
 250         if (etm->synth_opts.last_branch) {
 251                 size_t sz = sizeof(struct branch_stack);
 252
 253                 sz += etm->synth_opts.last_branch_sz *
 254                       sizeof(struct branch_entry);
 255                 tidq->last_branch = zalloc(sz);
 256                 if (!tidq->last_branch)
 257                         goto out_free;
 258                 tidq->last_branch_rb = zalloc(sz);
 259                 if (!tidq->last_branch_rb)
 260                         goto out_free;
 261         }
 262
 263         tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 264         if (!tidq->event_buf)
 265                 goto out_free;
 266
 267         return 0;
 268
 269 out_free:
 270         zfree(&tidq->last_branch_rb);
 271         zfree(&tidq->last_branch);
 272         zfree(&tidq->prev_packet);
 273         zfree(&tidq->packet);
 274 out:
 275         return rc;
 276 }
 277
 278 static struct cs_etm_traceid_queue
 279 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 280 {
 281         int idx;
 282         struct int_node *inode;
 283         struct intlist *traceid_queues_list;
 284         struct cs_etm_traceid_queue *tidq, **traceid_queues;
 285         struct cs_etm_auxtrace *etm = etmq->etm;
 286
 287         if (etm->timeless_decoding)
 288                 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
 289
 290         traceid_queues_list = etmq->traceid_queues_list;
 291
 292         /*
 293          * Check if the traceid_queue exist for this traceID by looking
 294          * in the queue list.
 295          */
 296         inode = intlist__find(traceid_queues_list, trace_chan_id);
 297         if (inode) {
 298                 idx = (int)(intptr_t)inode->priv;
 299                 return etmq->traceid_queues[idx];
 300         }
 301
 302         /* We couldn't find a traceid_queue for this traceID, allocate one */
 303         tidq = malloc(sizeof(*tidq));
 304         if (!tidq)
 305                 return NULL;
 306
 307         memset(tidq, 0, sizeof(*tidq));
 308
 309         /* Get a valid index for the new traceid_queue */
 310         idx = intlist__nr_entries(traceid_queues_list);
 311         /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
 312         inode = intlist__findnew(traceid_queues_list, trace_chan_id);
 313         if (!inode)
 314                 goto out_free;
 315
 316         /* Associate this traceID with this index */
 317         inode->priv = (void *)(intptr_t)idx;
 318
 319         if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
 320                 goto out_free;
 321
 322         /* Grow the traceid_queues array by one unit */
 323         traceid_queues = etmq->traceid_queues;
 324         traceid_queues = reallocarray(traceid_queues,
 325                                       idx + 1,
 326                                       sizeof(*traceid_queues));
 327
 328         /*
 329          * On failure reallocarray() returns NULL and the original block of
 330          * memory is left untouched.
 331          */
 332         if (!traceid_queues)
 333                 goto out_free;
 334
 335         traceid_queues[idx] = tidq;
 336         etmq->traceid_queues = traceid_queues;
 337
 338         return etmq->traceid_queues[idx];
 339
 340 out_free:
 341         /*
 342          * Function intlist__remove() removes the inode from the list
 343          * and delete the memory associated to it.
 344          */
 345         intlist__remove(traceid_queues_list, inode);
 346         free(tidq);
 347
 348         return NULL;
 349 }
 350
 351 struct cs_etm_packet_queue
 352 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 353 {
 354         struct cs_etm_traceid_queue *tidq;
 355
 356         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 357         if (tidq)
 358                 return &tidq->packet_queue;
 359
 360         return NULL;
 361 }
 362
 363 static void cs_etm__packet_dump(const char *pkt_string)
 364 {
 365         const char *color = PERF_COLOR_BLUE;
 366         int len = strlen(pkt_string);
 367
 368         if (len && (pkt_string[len-1] == '\n'))
 369                 color_fprintf(stdout, color, "  %s", pkt_string);
 370         else
 371                 color_fprintf(stdout, color, "  %s\n", pkt_string);
 372
 373         fflush(stdout);
 374 }
 375
 376 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
 377                                           struct cs_etm_auxtrace *etm, int idx,
 378                                           u32 etmidr)
 379 {
 380         u64 **metadata = etm->metadata;
 381
 382         t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
 383         t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
 384         t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
 385 }
 386
 387 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
 388                                           struct cs_etm_auxtrace *etm, int idx)
 389 {
 390         u64 **metadata = etm->metadata;
 391
 392         t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
 393         t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
 394         t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
 395         t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
 396         t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
 397         t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
 398         t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
 399 }
 400
 401 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
 402                                      struct cs_etm_auxtrace *etm)
 403 {
 404         int i;
 405         u32 etmidr;
 406         u64 architecture;
 407
 408         for (i = 0; i < etm->num_cpu; i++) {
 409                 architecture = etm->metadata[i][CS_ETM_MAGIC];
 410
 411                 switch (architecture) {
 412                 case __perf_cs_etmv3_magic:
 413                         etmidr = etm->metadata[i][CS_ETM_ETMIDR];
 414                         cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
 415                         break;
 416                 case __perf_cs_etmv4_magic:
 417                         cs_etm__set_trace_param_etmv4(t_params, etm, i);
 418                         break;
 419                 default:
 420                         return -EINVAL;
 421                 }
 422         }
 423
 424         return 0;
 425 }
 426
 427 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
 428                                        struct cs_etm_queue *etmq,
 429                                        enum cs_etm_decoder_operation mode)
 430 {
 431         int ret = -EINVAL;
 432
 433         if (!(mode < CS_ETM_OPERATION_MAX))
 434                 goto out;
 435
 436         d_params->packet_printer = cs_etm__packet_dump;
 437         d_params->operation = mode;
 438         d_params->data = etmq;
 439         d_params->formatted = true;
 440         d_params->fsyncs = false;
 441         d_params->hsyncs = false;
 442         d_params->frame_aligned = true;
 443
 444         ret = 0;
 445 out:
 446         return ret;
 447 }
 448
 449 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
 450                                struct auxtrace_buffer *buffer)
 451 {
 452         int ret;
 453         const char *color = PERF_COLOR_BLUE;
 454         struct cs_etm_decoder_params d_params;
 455         struct cs_etm_trace_params *t_params;
 456         struct cs_etm_decoder *decoder;
 457         size_t buffer_used = 0;
 458
 459         fprintf(stdout, "\n");
 460         color_fprintf(stdout, color,
 461                      ". ... CoreSight ETM Trace data: size %zu bytes\n",
 462                      buffer->size);
 463
 464         /* Use metadata to fill in trace parameters for trace decoder */
 465         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 466
 467         if (!t_params)
 468                 return;
 469
 470         if (cs_etm__init_trace_params(t_params, etm))
 471                 goto out_free;
 472
 473         /* Set decoder parameters to simply print the trace packets */
 474         if (cs_etm__init_decoder_params(&d_params, NULL,
 475                                         CS_ETM_OPERATION_PRINT))
 476                 goto out_free;
 477
 478         decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 479
 480         if (!decoder)
 481                 goto out_free;
 482         do {
 483                 size_t consumed;
 484
 485                 ret = cs_etm_decoder__process_data_block(
 486                                 decoder, buffer->offset,
 487                                 &((u8 *)buffer->data)[buffer_used],
 488                                 buffer->size - buffer_used, &consumed);
 489                 if (ret)
 490                         break;
 491
 492                 buffer_used += consumed;
 493         } while (buffer_used < buffer->size);
 494
 495         cs_etm_decoder__free(decoder);
 496
 497 out_free:
 498         zfree(&t_params);
 499 }
 500
 501 static int cs_etm__flush_events(struct perf_session *session,
 502                                 struct perf_tool *tool)
 503 {
 504         int ret;
 505         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
 506                                                    struct cs_etm_auxtrace,
 507                                                    auxtrace);
 508         if (dump_trace)
 509                 return 0;
 510
 511         if (!tool->ordered_events)
 512                 return -EINVAL;
 513
 514         ret = cs_etm__update_queues(etm);
 515
 516         if (ret < 0)
 517                 return ret;
 518
 519         if (etm->timeless_decoding)
 520                 return cs_etm__process_timeless_queues(etm, -1);
 521
 522         return cs_etm__process_queues(etm);
 523 }
 524
 525 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
 526 {
 527         int idx;
 528         uintptr_t priv;
 529         struct int_node *inode, *tmp;
 530         struct cs_etm_traceid_queue *tidq;
 531         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 532
 533         intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
 534                 priv = (uintptr_t)inode->priv;
 535                 idx = priv;
 536
 537                 /* Free this traceid_queue from the array */
 538                 tidq = etmq->traceid_queues[idx];
 539                 thread__zput(tidq->thread);
 540                 zfree(&tidq->event_buf);
 541                 zfree(&tidq->last_branch);
 542                 zfree(&tidq->last_branch_rb);
 543                 zfree(&tidq->prev_packet);
 544                 zfree(&tidq->packet);
 545                 zfree(&tidq);
 546
 547                 /*
 548                  * Function intlist__remove() removes the inode from the list
 549                  * and delete the memory associated to it.
 550                  */
 551                 intlist__remove(traceid_queues_list, inode);
 552         }
 553
 554         /* Then the RB tree itself */
 555         intlist__delete(traceid_queues_list);
 556         etmq->traceid_queues_list = NULL;
 557
 558         /* finally free the traceid_queues array */
 559         zfree(&etmq->traceid_queues);
 560 }
 561
 562 static void cs_etm__free_queue(void *priv)
 563 {
 564         struct cs_etm_queue *etmq = priv;
 565
 566         if (!etmq)
 567                 return;
 568
 569         cs_etm_decoder__free(etmq->decoder);
 570         cs_etm__free_traceid_queues(etmq);
 571         free(etmq);
 572 }
 573
 574 static void cs_etm__free_events(struct perf_session *session)
 575 {
 576         unsigned int i;
 577         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 578                                                    struct cs_etm_auxtrace,
 579                                                    auxtrace);
 580         struct auxtrace_queues *queues = &aux->queues;
 581
 582         for (i = 0; i < queues->nr_queues; i++) {
 583                 cs_etm__free_queue(queues->queue_array[i].priv);
 584                 queues->queue_array[i].priv = NULL;
 585         }
 586
 587         auxtrace_queues__free(queues);
 588 }
 589
 590 static void cs_etm__free(struct perf_session *session)
 591 {
 592         int i;
 593         struct int_node *inode, *tmp;
 594         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 595                                                    struct cs_etm_auxtrace,
 596                                                    auxtrace);
 597         cs_etm__free_events(session);
 598         session->auxtrace = NULL;
 599
 600         /* First remove all traceID/metadata nodes for the RB tree */
 601         intlist__for_each_entry_safe(inode, tmp, traceid_list)
 602                 intlist__remove(traceid_list, inode);
 603         /* Then the RB tree itself */
 604         intlist__delete(traceid_list);
 605
 606         for (i = 0; i < aux->num_cpu; i++)
 607                 zfree(&aux->metadata[i]);
 608
 609         thread__zput(aux->unknown_thread);
 610         zfree(&aux->metadata);
 611         zfree(&aux);
 612 }
 613
 614 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
 615 {
 616         struct machine *machine;
 617
 618         machine = etmq->etm->machine;
 619
 620         if (address >= etmq->etm->kernel_start) {
 621                 if (machine__is_host(machine))
 622                         return PERF_RECORD_MISC_KERNEL;
 623                 else
 624                         return PERF_RECORD_MISC_GUEST_KERNEL;
 625         } else {
 626                 if (machine__is_host(machine))
 627                         return PERF_RECORD_MISC_USER;
 628                 else if (perf_guest)
 629                         return PERF_RECORD_MISC_GUEST_USER;
 630                 else
 631                         return PERF_RECORD_MISC_HYPERVISOR;
 632         }
 633 }
 634
 635 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
 636                               u64 address, size_t size, u8 *buffer)
 637 {
 638         u8  cpumode;
 639         u64 offset;
 640         int len;
 641         struct thread *thread;
 642         struct machine *machine;
 643         struct addr_location al;
 644         struct cs_etm_traceid_queue *tidq;
 645
 646         if (!etmq)
 647                 return 0;
 648
 649         machine = etmq->etm->machine;
 650         cpumode = cs_etm__cpu_mode(etmq, address);
 651         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 652         if (!tidq)
 653                 return 0;
 654
 655         thread = tidq->thread;
 656         if (!thread) {
 657                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 658                         return 0;
 659                 thread = etmq->etm->unknown_thread;
 660         }
 661
 662         if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso)
 663                 return 0;
 664
 665         if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 666             dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE))
 667                 return 0;
 668
 669         offset = al.map->map_ip(al.map, address);
 670
 671         map__load(al.map);
 672
 673         len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size);
 674
 675         if (len <= 0)
 676                 return 0;
 677
 678         return len;
 679 }
 680
 681 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm)
 682 {
 683         struct cs_etm_decoder_params d_params;
 684         struct cs_etm_trace_params  *t_params = NULL;
 685         struct cs_etm_queue *etmq;
 686
 687         etmq = zalloc(sizeof(*etmq));
 688         if (!etmq)
 689                 return NULL;
 690
 691         etmq->traceid_queues_list = intlist__new(NULL);
 692         if (!etmq->traceid_queues_list)
 693                 goto out_free;
 694
 695         /* Use metadata to fill in trace parameters for trace decoder */
 696         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 697
 698         if (!t_params)
 699                 goto out_free;
 700
 701         if (cs_etm__init_trace_params(t_params, etm))
 702                 goto out_free;
 703
 704         /* Set decoder parameters to decode trace packets */
 705         if (cs_etm__init_decoder_params(&d_params, etmq,
 706                                         CS_ETM_OPERATION_DECODE))
 707                 goto out_free;
 708
 709         etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 710
 711         if (!etmq->decoder)
 712                 goto out_free;
 713
 714         /*
 715          * Register a function to handle all memory accesses required by
 716          * the trace decoder library.
 717          */
 718         if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
 719                                               0x0L, ((u64) -1L),
 720                                               cs_etm__mem_access))
 721                 goto out_free_decoder;
 722
 723         zfree(&t_params);
 724         return etmq;
 725
 726 out_free_decoder:
 727         cs_etm_decoder__free(etmq->decoder);
 728 out_free:
 729         intlist__delete(etmq->traceid_queues_list);
 730         free(etmq);
 731
 732         return NULL;
 733 }
 734
 735 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
 736                                struct auxtrace_queue *queue,
 737                                unsigned int queue_nr)
 738 {
 739         int ret = 0;
 740         unsigned int cs_queue_nr;
 741         u8 trace_chan_id;
 742         u64 timestamp;
 743         struct cs_etm_queue *etmq = queue->priv;
 744
 745         if (list_empty(&queue->head) || etmq)
 746                 goto out;
 747
 748         etmq = cs_etm__alloc_queue(etm);
 749
 750         if (!etmq) {
 751                 ret = -ENOMEM;
 752                 goto out;
 753         }
 754
 755         queue->priv = etmq;
 756         etmq->etm = etm;
 757         etmq->queue_nr = queue_nr;
 758         etmq->offset = 0;
 759
 760         if (etm->timeless_decoding)
 761                 goto out;
 762
 763         /*
 764          * We are under a CPU-wide trace scenario.  As such we need to know
 765          * when the code that generated the traces started to execute so that
 766          * it can be correlated with execution on other CPUs.  So we get a
 767          * handle on the beginning of traces and decode until we find a
 768          * timestamp.  The timestamp is then added to the auxtrace min heap
 769          * in order to know what nibble (of all the etmqs) to decode first.
 770          */
 771         while (1) {
 772                 /*
 773                  * Fetch an aux_buffer from this etmq.  Bail if no more
 774                  * blocks or an error has been encountered.
 775                  */
 776                 ret = cs_etm__get_data_block(etmq);
 777                 if (ret <= 0)
 778                         goto out;
 779
 780                 /*
 781                  * Run decoder on the trace block.  The decoder will stop when
 782                  * encountering a timestamp, a full packet queue or the end of
 783                  * trace for that block.
 784                  */
 785                 ret = cs_etm__decode_data_block(etmq);
 786                 if (ret)
 787                         goto out;
 788
 789                 /*
 790                  * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
 791                  * the timestamp calculation for us.
 792                  */
 793                 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
 794
 795                 /* We found a timestamp, no need to continue. */
 796                 if (timestamp)
 797                         break;
 798
 799                 /*
 800                  * We didn't find a timestamp so empty all the traceid packet
 801                  * queues before looking for another timestamp packet, either
 802                  * in the current data block or a new one.  Packets that were
 803                  * just decoded are useless since no timestamp has been
 804                  * associated with them.  As such simply discard them.
 805                  */
 806                 cs_etm__clear_all_packet_queues(etmq);
 807         }
 808
 809         /*
 810          * We have a timestamp.  Add it to the min heap to reflect when
 811          * instructions conveyed by the range packets of this traceID queue
 812          * started to execute.  Once the same has been done for all the traceID
 813          * queues of each etmq, redenring and decoding can start in
 814          * chronological order.
 815          *
 816          * Note that packets decoded above are still in the traceID's packet
 817          * queue and will be processed in cs_etm__process_queues().
 818          */
 819         cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_id_chan);
 820         ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
 821 out:
 822         return ret;
 823 }
 824
 825 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
 826 {
 827         unsigned int i;
 828         int ret;
 829
 830         if (!etm->kernel_start)
 831                 etm->kernel_start = machine__kernel_start(etm->machine);
 832
 833         for (i = 0; i < etm->queues.nr_queues; i++) {
 834                 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);
 835                 if (ret)
 836                         return ret;
 837         }
 838
 839         return 0;
 840 }
 841
 842 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)
 843 {
 844         if (etm->queues.new_data) {
 845                 etm->queues.new_data = false;
 846                 return cs_etm__setup_queues(etm);
 847         }
 848
 849         return 0;
 850 }
 851
 852 static inline
 853 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
 854                                  struct cs_etm_traceid_queue *tidq)
 855 {
 856         struct branch_stack *bs_src = tidq->last_branch_rb;
 857         struct branch_stack *bs_dst = tidq->last_branch;
 858         size_t nr = 0;
 859
 860         /*
 861          * Set the number of records before early exit: ->nr is used to
 862          * determine how many branches to copy from ->entries.
 863          */
 864         bs_dst->nr = bs_src->nr;
 865
 866         /*
 867          * Early exit when there is nothing to copy.
 868          */
 869         if (!bs_src->nr)
 870                 return;
 871
 872         /*
 873          * As bs_src->entries is a circular buffer, we need to copy from it in
 874          * two steps.  First, copy the branches from the most recently inserted
 875          * branch ->last_branch_pos until the end of bs_src->entries buffer.
 876          */
 877         nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
 878         memcpy(&bs_dst->entries[0],
 879                &bs_src->entries[tidq->last_branch_pos],
 880                sizeof(struct branch_entry) * nr);
 881
 882         /*
 883          * If we wrapped around at least once, the branches from the beginning
 884          * of the bs_src->entries buffer and until the ->last_branch_pos element
 885          * are older valid branches: copy them over.  The total number of
 886          * branches copied over will be equal to the number of branches asked by
 887          * the user in last_branch_sz.
 888          */
 889         if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
 890                 memcpy(&bs_dst->entries[nr],
 891                        &bs_src->entries[0],
 892                        sizeof(struct branch_entry) * tidq->last_branch_pos);
 893         }
 894 }
 895
 896 static inline
 897 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
 898 {
 899         tidq->last_branch_pos = 0;
 900         tidq->last_branch_rb->nr = 0;
 901 }
 902
 903 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
 904                                          u8 trace_chan_id, u64 addr)
 905 {
 906         u8 instrBytes[2];
 907
 908         cs_etm__mem_access(etmq, trace_chan_id, addr,
 909                            ARRAY_SIZE(instrBytes), instrBytes);
 910         /*
 911          * T32 instruction size is indicated by bits[15:11] of the first
 912          * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
 913          * denote a 32-bit instruction.
 914          */
 915         return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
 916 }
 917
 918 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
 919 {
 920         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 921         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 922                 return 0;
 923
 924         return packet->start_addr;
 925 }
 926
 927 static inline
 928 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
 929 {
 930         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 931         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 932                 return 0;
 933
 934         return packet->end_addr - packet->last_instr_size;
 935 }
 936
 937 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
 938                                      u64 trace_chan_id,
 939                                      const struct cs_etm_packet *packet,
 940                                      u64 offset)
 941 {
 942         if (packet->isa == CS_ETM_ISA_T32) {
 943                 u64 addr = packet->start_addr;
 944
 945                 while (offset > 0) {
 946                         addr += cs_etm__t32_instr_size(etmq,
 947                                                        trace_chan_id, addr);
 948                         offset--;
 949                 }
 950                 return addr;
 951         }
 952
 953         /* Assume a 4 byte instruction size (A32/A64) */
 954         return packet->start_addr + offset * 4;
 955 }
 956
 957 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
 958                                           struct cs_etm_traceid_queue *tidq)
 959 {
 960         struct branch_stack *bs = tidq->last_branch_rb;
 961         struct branch_entry *be;
 962
 963         /*
 964          * The branches are recorded in a circular buffer in reverse
 965          * chronological order: we start recording from the last element of the
 966          * buffer down.  After writing the first element of the stack, move the
 967          * insert position back to the end of the buffer.
 968          */
 969         if (!tidq->last_branch_pos)
 970                 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
 971
 972         tidq->last_branch_pos -= 1;
 973
 974         be       = &bs->entries[tidq->last_branch_pos];
 975         be->from = cs_etm__last_executed_instr(tidq->prev_packet);
 976         be->to   = cs_etm__first_executed_instr(tidq->packet);
 977         /* No support for mispredict */
 978         be->flags.mispred = 0;
 979         be->flags.predicted = 1;
 980
 981         /*
 982          * Increment bs->nr until reaching the number of last branches asked by
 983          * the user on the command line.
 984          */
 985         if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
 986                 bs->nr += 1;
 987 }
 988
 989 static int cs_etm__inject_event(union perf_event *event,
 990                                struct perf_sample *sample, u64 type)
 991 {
 992         event->header.size = perf_event__sample_event_size(sample, type, 0);
 993         return perf_event__synthesize_sample(event, type, 0, sample);
 994 }
 995
 996
 997 static int
 998 cs_etm__get_trace(struct cs_etm_queue *etmq)
 999 {
1000         struct auxtrace_buffer *aux_buffer = etmq->buffer;
1001         struct auxtrace_buffer *old_buffer = aux_buffer;
1002         struct auxtrace_queue *queue;
1003
1004         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1005
1006         aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1007
1008         /* If no more data, drop the previous auxtrace_buffer and return */
1009         if (!aux_buffer) {
1010                 if (old_buffer)
1011                         auxtrace_buffer__drop_data(old_buffer);
1012                 etmq->buf_len = 0;
1013                 return 0;
1014         }
1015
1016         etmq->buffer = aux_buffer;
1017
1018         /* If the aux_buffer doesn't have data associated, try to load it */
1019         if (!aux_buffer->data) {
1020                 /* get the file desc associated with the perf data file */
1021                 int fd = perf_data__fd(etmq->etm->session->data);
1022
1023                 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1024                 if (!aux_buffer->data)
1025                         return -ENOMEM;
1026         }
1027
1028         /* If valid, drop the previous buffer */
1029         if (old_buffer)
1030                 auxtrace_buffer__drop_data(old_buffer);
1031
1032         etmq->buf_used = 0;
1033         etmq->buf_len = aux_buffer->size;
1034         etmq->buf = aux_buffer->data;
1035
1036         return etmq->buf_len;
1037 }
1038
1039 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
1040                                     struct cs_etm_traceid_queue *tidq)
1041 {
1042         if ((!tidq->thread) && (tidq->tid != -1))
1043                 tidq->thread = machine__find_thread(etm->machine, -1,
1044                                                     tidq->tid);
1045
1046         if (tidq->thread)
1047                 tidq->pid = tidq->thread->pid_;
1048 }
1049
1050 int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq,
1051                          pid_t tid, u8 trace_chan_id)
1052 {
1053         int cpu, err = -EINVAL;
1054         struct cs_etm_auxtrace *etm = etmq->etm;
1055         struct cs_etm_traceid_queue *tidq;
1056
1057         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1058         if (!tidq)
1059                 return err;
1060
1061         if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0)
1062                 return err;
1063
1064         err = machine__set_current_tid(etm->machine, cpu, tid, tid);
1065         if (err)
1066                 return err;
1067
1068         tidq->tid = tid;
1069         thread__zput(tidq->thread);
1070
1071         cs_etm__set_pid_tid_cpu(etm, tidq);
1072         return 0;
1073 }
1074
1075 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1076 {
1077         return !!etmq->etm->timeless_decoding;
1078 }
1079
1080 static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1081                               u64 trace_chan_id,
1082                               const struct cs_etm_packet *packet,
1083                               struct perf_sample *sample)
1084 {
1085         /*
1086          * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1087          * packet, so directly bail out with 'insn_len' = 0.
1088          */
1089         if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1090                 sample->insn_len = 0;
1091                 return;
1092         }
1093
1094         /*
1095          * T32 instruction size might be 32-bit or 16-bit, decide by calling
1096          * cs_etm__t32_instr_size().
1097          */
1098         if (packet->isa == CS_ETM_ISA_T32)
1099                 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1100                                                           sample->ip);
1101         /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1102         else
1103                 sample->insn_len = 4;
1104
1105         cs_etm__mem_access(etmq, trace_chan_id, sample->ip,
1106                            sample->insn_len, (void *)sample->insn);
1107 }
1108
1109 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1110                                             struct cs_etm_traceid_queue *tidq,
1111                                             u64 addr, u64 period)
1112 {
1113         int ret = 0;
1114         struct cs_etm_auxtrace *etm = etmq->etm;
1115         union perf_event *event = tidq->event_buf;
1116         struct perf_sample sample = {.ip = 0,};
1117
1118         event->sample.header.type = PERF_RECORD_SAMPLE;
1119         event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
1120         event->sample.header.size = sizeof(struct perf_event_header);
1121
1122         sample.ip = addr;
1123         sample.pid = tidq->pid;
1124         sample.tid = tidq->tid;
1125         sample.id = etmq->etm->instructions_id;
1126         sample.stream_id = etmq->etm->instructions_id;
1127         sample.period = period;
1128         sample.cpu = tidq->packet->cpu;
1129         sample.flags = tidq->prev_packet->flags;
1130         sample.cpumode = event->sample.header.misc;
1131
1132         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1133
1134         if (etm->synth_opts.last_branch) {
1135                 cs_etm__copy_last_branch_rb(etmq, tidq);
1136                 sample.branch_stack = tidq->last_branch;
1137         }
1138
1139         if (etm->synth_opts.inject) {
1140                 ret = cs_etm__inject_event(event, &sample,
1141                                            etm->instructions_sample_type);
1142                 if (ret)
1143                         return ret;
1144         }
1145
1146         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1147
1148         if (ret)
1149                 pr_err(
1150                         "CS ETM Trace: failed to deliver instruction event, error %d\n",
1151                         ret);
1152
1153         if (etm->synth_opts.last_branch)
1154                 cs_etm__reset_last_branch_rb(tidq);
1155
1156         return ret;
1157 }
1158
1159 /*
1160  * The cs etm packet encodes an instruction range between a branch target
1161  * and the next taken branch. Generate sample accordingly.
1162  */
1163 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1164                                        struct cs_etm_traceid_queue *tidq)
1165 {
1166         int ret = 0;
1167         struct cs_etm_auxtrace *etm = etmq->etm;
1168         struct perf_sample sample = {.ip = 0,};
1169         union perf_event *event = tidq->event_buf;
1170         struct dummy_branch_stack {
1171                 u64                     nr;
1172                 struct branch_entry     entries;
1173         } dummy_bs;
1174         u64 ip;
1175
1176         ip = cs_etm__last_executed_instr(tidq->prev_packet);
1177
1178         event->sample.header.type = PERF_RECORD_SAMPLE;
1179         event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
1180         event->sample.header.size = sizeof(struct perf_event_header);
1181
1182         sample.ip = ip;
1183         sample.pid = tidq->pid;
1184         sample.tid = tidq->tid;
1185         sample.addr = cs_etm__first_executed_instr(tidq->packet);
1186         sample.id = etmq->etm->branches_id;
1187         sample.stream_id = etmq->etm->branches_id;
1188         sample.period = 1;
1189         sample.cpu = tidq->packet->cpu;
1190         sample.flags = tidq->prev_packet->flags;
1191         sample.cpumode = event->sample.header.misc;
1192
1193         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1194                           &sample);
1195
1196         /*
1197          * perf report cannot handle events without a branch stack
1198          */
1199         if (etm->synth_opts.last_branch) {
1200                 dummy_bs = (struct dummy_branch_stack){
1201                         .nr = 1,
1202                         .entries = {
1203                                 .from = sample.ip,
1204                                 .to = sample.addr,
1205                         },
1206                 };
1207                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1208         }
1209
1210         if (etm->synth_opts.inject) {
1211                 ret = cs_etm__inject_event(event, &sample,
1212                                            etm->branches_sample_type);
1213                 if (ret)
1214                         return ret;
1215         }
1216
1217         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1218
1219         if (ret)
1220                 pr_err(
1221                 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1222                 ret);
1223
1224         return ret;
1225 }
1226
1227 struct cs_etm_synth {
1228         struct perf_tool dummy_tool;
1229         struct perf_session *session;
1230 };
1231
1232 static int cs_etm__event_synth(struct perf_tool *tool,
1233                                union perf_event *event,
1234                                struct perf_sample *sample __maybe_unused,
1235                                struct machine *machine __maybe_unused)
1236 {
1237         struct cs_etm_synth *cs_etm_synth =
1238                       container_of(tool, struct cs_etm_synth, dummy_tool);
1239
1240         return perf_session__deliver_synth_event(cs_etm_synth->session,
1241                                                  event, NULL);
1242 }
1243
1244 static int cs_etm__synth_event(struct perf_session *session,
1245                                struct perf_event_attr *attr, u64 id)
1246 {
1247         struct cs_etm_synth cs_etm_synth;
1248
1249         memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1250         cs_etm_synth.session = session;
1251
1252         return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1253                                            &id, cs_etm__event_synth);
1254 }
1255
1256 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1257                                 struct perf_session *session)
1258 {
1259         struct evlist *evlist = session->evlist;
1260         struct evsel *evsel;
1261         struct perf_event_attr attr;
1262         bool found = false;
1263         u64 id;
1264         int err;
1265
1266         evlist__for_each_entry(evlist, evsel) {
1267                 if (evsel->core.attr.type == etm->pmu_type) {
1268                         found = true;
1269                         break;
1270                 }
1271         }
1272
1273         if (!found) {
1274                 pr_debug("No selected events with CoreSight Trace data\n");
1275                 return 0;
1276         }
1277
1278         memset(&attr, 0, sizeof(struct perf_event_attr));
1279         attr.size = sizeof(struct perf_event_attr);
1280         attr.type = PERF_TYPE_HARDWARE;
1281         attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1282         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1283                             PERF_SAMPLE_PERIOD;
1284         if (etm->timeless_decoding)
1285                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1286         else
1287                 attr.sample_type |= PERF_SAMPLE_TIME;
1288
1289         attr.exclude_user = evsel->core.attr.exclude_user;
1290         attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1291         attr.exclude_hv = evsel->core.attr.exclude_hv;
1292         attr.exclude_host = evsel->core.attr.exclude_host;
1293         attr.exclude_guest = evsel->core.attr.exclude_guest;
1294         attr.sample_id_all = evsel->core.attr.sample_id_all;
1295         attr.read_format = evsel->core.attr.read_format;
1296
1297         /* create new id val to be a fixed offset from evsel id */
1298         id = evsel->id[0] + 1000000000;
1299
1300         if (!id)
1301                 id = 1;
1302
1303         if (etm->synth_opts.branches) {
1304                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1305                 attr.sample_period = 1;
1306                 attr.sample_type |= PERF_SAMPLE_ADDR;
1307                 err = cs_etm__synth_event(session, &attr, id);
1308                 if (err)
1309                         return err;
1310                 etm->sample_branches = true;
1311                 etm->branches_sample_type = attr.sample_type;
1312                 etm->branches_id = id;
1313                 id += 1;
1314                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1315         }
1316
1317         if (etm->synth_opts.last_branch)
1318                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1319
1320         if (etm->synth_opts.instructions) {
1321                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1322                 attr.sample_period = etm->synth_opts.period;
1323                 etm->instructions_sample_period = attr.sample_period;
1324                 err = cs_etm__synth_event(session, &attr, id);
1325                 if (err)
1326                         return err;
1327                 etm->sample_instructions = true;
1328                 etm->instructions_sample_type = attr.sample_type;
1329                 etm->instructions_id = id;
1330                 id += 1;
1331         }
1332
1333         return 0;
1334 }
1335
1336 static int cs_etm__sample(struct cs_etm_queue *etmq,
1337                           struct cs_etm_traceid_queue *tidq)
1338 {
1339         struct cs_etm_auxtrace *etm = etmq->etm;
1340         struct cs_etm_packet *tmp;
1341         int ret;
1342         u8 trace_chan_id = tidq->trace_chan_id;
1343         u64 instrs_executed = tidq->packet->instr_count;
1344
1345         tidq->period_instructions += instrs_executed;
1346
1347         /*
1348          * Record a branch when the last instruction in
1349          * PREV_PACKET is a branch.
1350          */
1351         if (etm->synth_opts.last_branch &&
1352             tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1353             tidq->prev_packet->last_instr_taken_branch)
1354                 cs_etm__update_last_branch_rb(etmq, tidq);
1355
1356         if (etm->sample_instructions &&
1357             tidq->period_instructions >= etm->instructions_sample_period) {
1358                 /*
1359                  * Emit instruction sample periodically
1360                  * TODO: allow period to be defined in cycles and clock time
1361                  */
1362
1363                 /* Get number of instructions executed after the sample point */
1364                 u64 instrs_over = tidq->period_instructions -
1365                         etm->instructions_sample_period;
1366
1367                 /*
1368                  * Calculate the address of the sampled instruction (-1 as
1369                  * sample is reported as though instruction has just been
1370                  * executed, but PC has not advanced to next instruction)
1371                  */
1372                 u64 offset = (instrs_executed - instrs_over - 1);
1373                 u64 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1374                                               tidq->packet, offset);
1375
1376                 ret = cs_etm__synth_instruction_sample(
1377                         etmq, tidq, addr, etm->instructions_sample_period);
1378                 if (ret)
1379                         return ret;
1380
1381                 /* Carry remaining instructions into next sample period */
1382                 tidq->period_instructions = instrs_over;
1383         }
1384
1385         if (etm->sample_branches) {
1386                 bool generate_sample = false;
1387
1388                 /* Generate sample for tracing on packet */
1389                 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1390                         generate_sample = true;
1391
1392                 /* Generate sample for branch taken packet */
1393                 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1394                     tidq->prev_packet->last_instr_taken_branch)
1395                         generate_sample = true;
1396
1397                 if (generate_sample) {
1398                         ret = cs_etm__synth_branch_sample(etmq, tidq);
1399                         if (ret)
1400                                 return ret;
1401                 }
1402         }
1403
1404         if (etm->sample_branches || etm->synth_opts.last_branch) {
1405                 /*
1406                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1407                  * the next incoming packet.
1408                  */
1409                 tmp = tidq->packet;
1410                 tidq->packet = tidq->prev_packet;
1411                 tidq->prev_packet = tmp;
1412         }
1413
1414         return 0;
1415 }
1416
1417 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1418 {
1419         /*
1420          * When the exception packet is inserted, whether the last instruction
1421          * in previous range packet is taken branch or not, we need to force
1422          * to set 'prev_packet->last_instr_taken_branch' to true.  This ensures
1423          * to generate branch sample for the instruction range before the
1424          * exception is trapped to kernel or before the exception returning.
1425          *
1426          * The exception packet includes the dummy address values, so don't
1427          * swap PACKET with PREV_PACKET.  This keeps PREV_PACKET to be useful
1428          * for generating instruction and branch samples.
1429          */
1430         if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1431                 tidq->prev_packet->last_instr_taken_branch = true;
1432
1433         return 0;
1434 }
1435
1436 static int cs_etm__flush(struct cs_etm_queue *etmq,
1437                          struct cs_etm_traceid_queue *tidq)
1438 {
1439         int err = 0;
1440         struct cs_etm_auxtrace *etm = etmq->etm;
1441         struct cs_etm_packet *tmp;
1442
1443         /* Handle start tracing packet */
1444         if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1445                 goto swap_packet;
1446
1447         if (etmq->etm->synth_opts.last_branch &&
1448             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1449                 /*
1450                  * Generate a last branch event for the branches left in the
1451                  * circular buffer at the end of the trace.
1452                  *
1453                  * Use the address of the end of the last reported execution
1454                  * range
1455                  */
1456                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1457
1458                 err = cs_etm__synth_instruction_sample(
1459                         etmq, tidq, addr,
1460                         tidq->period_instructions);
1461                 if (err)
1462                         return err;
1463
1464                 tidq->period_instructions = 0;
1465
1466         }
1467
1468         if (etm->sample_branches &&
1469             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1470                 err = cs_etm__synth_branch_sample(etmq, tidq);
1471                 if (err)
1472                         return err;
1473         }
1474
1475 swap_packet:
1476         if (etm->sample_branches || etm->synth_opts.last_branch) {
1477                 /*
1478                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1479                  * the next incoming packet.
1480                  */
1481                 tmp = tidq->packet;
1482                 tidq->packet = tidq->prev_packet;
1483                 tidq->prev_packet = tmp;
1484         }
1485
1486         return err;
1487 }
1488
1489 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1490                              struct cs_etm_traceid_queue *tidq)
1491 {
1492         int err;
1493
1494         /*
1495          * It has no new packet coming and 'etmq->packet' contains the stale
1496          * packet which was set at the previous time with packets swapping;
1497          * so skip to generate branch sample to avoid stale packet.
1498          *
1499          * For this case only flush branch stack and generate a last branch
1500          * event for the branches left in the circular buffer at the end of
1501          * the trace.
1502          */
1503         if (etmq->etm->synth_opts.last_branch &&
1504             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1505                 /*
1506                  * Use the address of the end of the last reported execution
1507                  * range.
1508                  */
1509                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1510
1511                 err = cs_etm__synth_instruction_sample(
1512                         etmq, tidq, addr,
1513                         tidq->period_instructions);
1514                 if (err)
1515                         return err;
1516
1517                 tidq->period_instructions = 0;
1518         }
1519
1520         return 0;
1521 }
1522 /*
1523  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1524  *                         if need be.
1525  * Returns:     < 0     if error
1526  *              = 0     if no more auxtrace_buffer to read
1527  *              > 0     if the current buffer isn't empty yet
1528  */
1529 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1530 {
1531         int ret;
1532
1533         if (!etmq->buf_len) {
1534                 ret = cs_etm__get_trace(etmq);
1535                 if (ret <= 0)
1536                         return ret;
1537                 /*
1538                  * We cannot assume consecutive blocks in the data file
1539                  * are contiguous, reset the decoder to force re-sync.
1540                  */
1541                 ret = cs_etm_decoder__reset(etmq->decoder);
1542                 if (ret)
1543                         return ret;
1544         }
1545
1546         return etmq->buf_len;
1547 }
1548
1549 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1550                                  struct cs_etm_packet *packet,
1551                                  u64 end_addr)
1552 {
1553         /* Initialise to keep compiler happy */
1554         u16 instr16 = 0;
1555         u32 instr32 = 0;
1556         u64 addr;
1557
1558         switch (packet->isa) {
1559         case CS_ETM_ISA_T32:
1560                 /*
1561                  * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1562                  *
1563                  *  b'15         b'8
1564                  * +-----------------+--------+
1565                  * | 1 1 0 1 1 1 1 1 |  imm8  |
1566                  * +-----------------+--------+
1567                  *
1568                  * According to the specifiction, it only defines SVC for T32
1569                  * with 16 bits instruction and has no definition for 32bits;
1570                  * so below only read 2 bytes as instruction size for T32.
1571                  */
1572                 addr = end_addr - 2;
1573                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1574                                    sizeof(instr16), (u8 *)&instr16);
1575                 if ((instr16 & 0xFF00) == 0xDF00)
1576                         return true;
1577
1578                 break;
1579         case CS_ETM_ISA_A32:
1580                 /*
1581                  * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1582                  *
1583                  *  b'31 b'28 b'27 b'24
1584                  * +---------+---------+-------------------------+
1585                  * |  !1111  | 1 1 1 1 |        imm24            |
1586                  * +---------+---------+-------------------------+
1587                  */
1588                 addr = end_addr - 4;
1589                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1590                                    sizeof(instr32), (u8 *)&instr32);
1591                 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1592                     (instr32 & 0xF0000000) != 0xF0000000)
1593                         return true;
1594
1595                 break;
1596         case CS_ETM_ISA_A64:
1597                 /*
1598                  * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1599                  *
1600                  *  b'31               b'21           b'4     b'0
1601                  * +-----------------------+---------+-----------+
1602                  * | 1 1 0 1 0 1 0 0 0 0 0 |  imm16  | 0 0 0 0 1 |
1603                  * +-----------------------+---------+-----------+
1604                  */
1605                 addr = end_addr - 4;
1606                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1607                                    sizeof(instr32), (u8 *)&instr32);
1608                 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1609                         return true;
1610
1611                 break;
1612         case CS_ETM_ISA_UNKNOWN:
1613         default:
1614                 break;
1615         }
1616
1617         return false;
1618 }
1619
1620 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1621                                struct cs_etm_traceid_queue *tidq, u64 magic)
1622 {
1623         u8 trace_chan_id = tidq->trace_chan_id;
1624         struct cs_etm_packet *packet = tidq->packet;
1625         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1626
1627         if (magic == __perf_cs_etmv3_magic)
1628                 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1629                         return true;
1630
1631         /*
1632          * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1633          * HVC cases; need to check if it's SVC instruction based on
1634          * packet address.
1635          */
1636         if (magic == __perf_cs_etmv4_magic) {
1637                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1638                     cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1639                                          prev_packet->end_addr))
1640                         return true;
1641         }
1642
1643         return false;
1644 }
1645
1646 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1647                                        u64 magic)
1648 {
1649         struct cs_etm_packet *packet = tidq->packet;
1650
1651         if (magic == __perf_cs_etmv3_magic)
1652                 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1653                     packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1654                     packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1655                     packet->exception_number == CS_ETMV3_EXC_IRQ ||
1656                     packet->exception_number == CS_ETMV3_EXC_FIQ)
1657                         return true;
1658
1659         if (magic == __perf_cs_etmv4_magic)
1660                 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1661                     packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1662                     packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1663                     packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1664                     packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1665                     packet->exception_number == CS_ETMV4_EXC_IRQ ||
1666                     packet->exception_number == CS_ETMV4_EXC_FIQ)
1667                         return true;
1668
1669         return false;
1670 }
1671
1672 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
1673                                       struct cs_etm_traceid_queue *tidq,
1674                                       u64 magic)
1675 {
1676         u8 trace_chan_id = tidq->trace_chan_id;
1677         struct cs_etm_packet *packet = tidq->packet;
1678         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1679
1680         if (magic == __perf_cs_etmv3_magic)
1681                 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
1682                     packet->exception_number == CS_ETMV3_EXC_HYP ||
1683                     packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
1684                     packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
1685                     packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
1686                     packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
1687                     packet->exception_number == CS_ETMV3_EXC_GENERIC)
1688                         return true;
1689
1690         if (magic == __perf_cs_etmv4_magic) {
1691                 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
1692                     packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
1693                     packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
1694                     packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
1695                         return true;
1696
1697                 /*
1698                  * For CS_ETMV4_EXC_CALL, except SVC other instructions
1699                  * (SMC, HVC) are taken as sync exceptions.
1700                  */
1701                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1702                     !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1703                                           prev_packet->end_addr))
1704                         return true;
1705
1706                 /*
1707                  * ETMv4 has 5 bits for exception number; if the numbers
1708                  * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
1709                  * they are implementation defined exceptions.
1710                  *
1711                  * For this case, simply take it as sync exception.
1712                  */
1713                 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
1714                     packet->exception_number <= CS_ETMV4_EXC_END)
1715                         return true;
1716         }
1717
1718         return false;
1719 }
1720
1721 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
1722                                     struct cs_etm_traceid_queue *tidq)
1723 {
1724         struct cs_etm_packet *packet = tidq->packet;
1725         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1726         u8 trace_chan_id = tidq->trace_chan_id;
1727         u64 magic;
1728         int ret;
1729
1730         switch (packet->sample_type) {
1731         case CS_ETM_RANGE:
1732                 /*
1733                  * Immediate branch instruction without neither link nor
1734                  * return flag, it's normal branch instruction within
1735                  * the function.
1736                  */
1737                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1738                     packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
1739                         packet->flags = PERF_IP_FLAG_BRANCH;
1740
1741                         if (packet->last_instr_cond)
1742                                 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
1743                 }
1744
1745                 /*
1746                  * Immediate branch instruction with link (e.g. BL), this is
1747                  * branch instruction for function call.
1748                  */
1749                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1750                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1751                         packet->flags = PERF_IP_FLAG_BRANCH |
1752                                         PERF_IP_FLAG_CALL;
1753
1754                 /*
1755                  * Indirect branch instruction with link (e.g. BLR), this is
1756                  * branch instruction for function call.
1757                  */
1758                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1759                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1760                         packet->flags = PERF_IP_FLAG_BRANCH |
1761                                         PERF_IP_FLAG_CALL;
1762
1763                 /*
1764                  * Indirect branch instruction with subtype of
1765                  * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
1766                  * function return for A32/T32.
1767                  */
1768                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1769                     packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
1770                         packet->flags = PERF_IP_FLAG_BRANCH |
1771                                         PERF_IP_FLAG_RETURN;
1772
1773                 /*
1774                  * Indirect branch instruction without link (e.g. BR), usually
1775                  * this is used for function return, especially for functions
1776                  * within dynamic link lib.
1777                  */
1778                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1779                     packet->last_instr_subtype == OCSD_S_INSTR_NONE)
1780                         packet->flags = PERF_IP_FLAG_BRANCH |
1781                                         PERF_IP_FLAG_RETURN;
1782
1783                 /* Return instruction for function return. */
1784                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1785                     packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
1786                         packet->flags = PERF_IP_FLAG_BRANCH |
1787                                         PERF_IP_FLAG_RETURN;
1788
1789                 /*
1790                  * Decoder might insert a discontinuity in the middle of
1791                  * instruction packets, fixup prev_packet with flag
1792                  * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
1793                  */
1794                 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1795                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1796                                               PERF_IP_FLAG_TRACE_BEGIN;
1797
1798                 /*
1799                  * If the previous packet is an exception return packet
1800                  * and the return address just follows SVC instuction,
1801                  * it needs to calibrate the previous packet sample flags
1802                  * as PERF_IP_FLAG_SYSCALLRET.
1803                  */
1804                 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
1805                                            PERF_IP_FLAG_RETURN |
1806                                            PERF_IP_FLAG_INTERRUPT) &&
1807                     cs_etm__is_svc_instr(etmq, trace_chan_id,
1808                                          packet, packet->start_addr))
1809                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1810                                              PERF_IP_FLAG_RETURN |
1811                                              PERF_IP_FLAG_SYSCALLRET;
1812                 break;
1813         case CS_ETM_DISCONTINUITY:
1814                 /*
1815                  * The trace is discontinuous, if the previous packet is
1816                  * instruction packet, set flag PERF_IP_FLAG_TRACE_END
1817                  * for previous packet.
1818                  */
1819                 if (prev_packet->sample_type == CS_ETM_RANGE)
1820                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1821                                               PERF_IP_FLAG_TRACE_END;
1822                 break;
1823         case CS_ETM_EXCEPTION:
1824                 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
1825                 if (ret)
1826                         return ret;
1827
1828                 /* The exception is for system call. */
1829                 if (cs_etm__is_syscall(etmq, tidq, magic))
1830                         packet->flags = PERF_IP_FLAG_BRANCH |
1831                                         PERF_IP_FLAG_CALL |
1832                                         PERF_IP_FLAG_SYSCALLRET;
1833                 /*
1834                  * The exceptions are triggered by external signals from bus,
1835                  * interrupt controller, debug module, PE reset or halt.
1836                  */
1837                 else if (cs_etm__is_async_exception(tidq, magic))
1838                         packet->flags = PERF_IP_FLAG_BRANCH |
1839                                         PERF_IP_FLAG_CALL |
1840                                         PERF_IP_FLAG_ASYNC |
1841                                         PERF_IP_FLAG_INTERRUPT;
1842                 /*
1843                  * Otherwise, exception is caused by trap, instruction &
1844                  * data fault, or alignment errors.
1845                  */
1846                 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
1847                         packet->flags = PERF_IP_FLAG_BRANCH |
1848                                         PERF_IP_FLAG_CALL |
1849                                         PERF_IP_FLAG_INTERRUPT;
1850
1851                 /*
1852                  * When the exception packet is inserted, since exception
1853                  * packet is not used standalone for generating samples
1854                  * and it's affiliation to the previous instruction range
1855                  * packet; so set previous range packet flags to tell perf
1856                  * it is an exception taken branch.
1857                  */
1858                 if (prev_packet->sample_type == CS_ETM_RANGE)
1859                         prev_packet->flags = packet->flags;
1860                 break;
1861         case CS_ETM_EXCEPTION_RET:
1862                 /*
1863                  * When the exception return packet is inserted, since
1864                  * exception return packet is not used standalone for
1865                  * generating samples and it's affiliation to the previous
1866                  * instruction range packet; so set previous range packet
1867                  * flags to tell perf it is an exception return branch.
1868                  *
1869                  * The exception return can be for either system call or
1870                  * other exception types; unfortunately the packet doesn't
1871                  * contain exception type related info so we cannot decide
1872                  * the exception type purely based on exception return packet.
1873                  * If we record the exception number from exception packet and
1874                  * reuse it for excpetion return packet, this is not reliable
1875                  * due the trace can be discontinuity or the interrupt can
1876                  * be nested, thus the recorded exception number cannot be
1877                  * used for exception return packet for these two cases.
1878                  *
1879                  * For exception return packet, we only need to distinguish the
1880                  * packet is for system call or for other types.  Thus the
1881                  * decision can be deferred when receive the next packet which
1882                  * contains the return address, based on the return address we
1883                  * can read out the previous instruction and check if it's a
1884                  * system call instruction and then calibrate the sample flag
1885                  * as needed.
1886                  */
1887                 if (prev_packet->sample_type == CS_ETM_RANGE)
1888                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1889                                              PERF_IP_FLAG_RETURN |
1890                                              PERF_IP_FLAG_INTERRUPT;
1891                 break;
1892         case CS_ETM_EMPTY:
1893         default:
1894                 break;
1895         }
1896
1897         return 0;
1898 }
1899
1900 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
1901 {
1902         int ret = 0;
1903         size_t processed = 0;
1904
1905         /*
1906          * Packets are decoded and added to the decoder's packet queue
1907          * until the decoder packet processing callback has requested that
1908          * processing stops or there is nothing left in the buffer.  Normal
1909          * operations that stop processing are a timestamp packet or a full
1910          * decoder buffer queue.
1911          */
1912         ret = cs_etm_decoder__process_data_block(etmq->decoder,
1913                                                  etmq->offset,
1914                                                  &etmq->buf[etmq->buf_used],
1915                                                  etmq->buf_len,
1916                                                  &processed);
1917         if (ret)
1918                 goto out;
1919
1920         etmq->offset += processed;
1921         etmq->buf_used += processed;
1922         etmq->buf_len -= processed;
1923
1924 out:
1925         return ret;
1926 }
1927
1928 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
1929                                          struct cs_etm_traceid_queue *tidq)
1930 {
1931         int ret;
1932         struct cs_etm_packet_queue *packet_queue;
1933
1934         packet_queue = &tidq->packet_queue;
1935
1936         /* Process each packet in this chunk */
1937         while (1) {
1938                 ret = cs_etm_decoder__get_packet(packet_queue,
1939                                                  tidq->packet);
1940                 if (ret <= 0)
1941                         /*
1942                          * Stop processing this chunk on
1943                          * end of data or error
1944                          */
1945                         break;
1946
1947                 /*
1948                  * Since packet addresses are swapped in packet
1949                  * handling within below switch() statements,
1950                  * thus setting sample flags must be called
1951                  * prior to switch() statement to use address
1952                  * information before packets swapping.
1953                  */
1954                 ret = cs_etm__set_sample_flags(etmq, tidq);
1955                 if (ret < 0)
1956                         break;
1957
1958                 switch (tidq->packet->sample_type) {
1959                 case CS_ETM_RANGE:
1960                         /*
1961                          * If the packet contains an instruction
1962                          * range, generate instruction sequence
1963                          * events.
1964                          */
1965                         cs_etm__sample(etmq, tidq);
1966                         break;
1967                 case CS_ETM_EXCEPTION:
1968                 case CS_ETM_EXCEPTION_RET:
1969                         /*
1970                          * If the exception packet is coming,
1971                          * make sure the previous instruction
1972                          * range packet to be handled properly.
1973                          */
1974                         cs_etm__exception(tidq);
1975                         break;
1976                 case CS_ETM_DISCONTINUITY:
1977                         /*
1978                          * Discontinuity in trace, flush
1979                          * previous branch stack
1980                          */
1981                         cs_etm__flush(etmq, tidq);
1982                         break;
1983                 case CS_ETM_EMPTY:
1984                         /*
1985                          * Should not receive empty packet,
1986                          * report error.
1987                          */
1988                         pr_err("CS ETM Trace: empty packet\n");
1989                         return -EINVAL;
1990                 default:
1991                         break;
1992                 }
1993         }
1994
1995         return ret;
1996 }
1997
1998 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
1999 {
2000         int idx;
2001         struct int_node *inode;
2002         struct cs_etm_traceid_queue *tidq;
2003         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2004
2005         intlist__for_each_entry(inode, traceid_queues_list) {
2006                 idx = (int)(intptr_t)inode->priv;
2007                 tidq = etmq->traceid_queues[idx];
2008
2009                 /* Ignore return value */
2010                 cs_etm__process_traceid_queue(etmq, tidq);
2011
2012                 /*
2013                  * Generate an instruction sample with the remaining
2014                  * branchstack entries.
2015                  */
2016                 cs_etm__flush(etmq, tidq);
2017         }
2018 }
2019
2020 static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
2021 {
2022         int err = 0;
2023         struct cs_etm_traceid_queue *tidq;
2024
2025         tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2026         if (!tidq)
2027                 return -EINVAL;
2028
2029         /* Go through each buffer in the queue and decode them one by one */
2030         while (1) {
2031                 err = cs_etm__get_data_block(etmq);
2032                 if (err <= 0)
2033                         return err;
2034
2035                 /* Run trace decoder until buffer consumed or end of trace */
2036                 do {
2037                         err = cs_etm__decode_data_block(etmq);
2038                         if (err)
2039                                 return err;
2040
2041                         /*
2042                          * Process each packet in this chunk, nothing to do if
2043                          * an error occurs other than hoping the next one will
2044                          * be better.
2045                          */
2046                         err = cs_etm__process_traceid_queue(etmq, tidq);
2047
2048                 } while (etmq->buf_len);
2049
2050                 if (err == 0)
2051                         /* Flush any remaining branch stack entries */
2052                         err = cs_etm__end_block(etmq, tidq);
2053         }
2054
2055         return err;
2056 }
2057
2058 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2059                                            pid_t tid)
2060 {
2061         unsigned int i;
2062         struct auxtrace_queues *queues = &etm->queues;
2063
2064         for (i = 0; i < queues->nr_queues; i++) {
2065                 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2066                 struct cs_etm_queue *etmq = queue->priv;
2067                 struct cs_etm_traceid_queue *tidq;
2068
2069                 if (!etmq)
2070                         continue;
2071
2072                 tidq = cs_etm__etmq_get_traceid_queue(etmq,
2073                                                 CS_ETM_PER_THREAD_TRACEID);
2074
2075                 if (!tidq)
2076                         continue;
2077
2078                 if ((tid == -1) || (tidq->tid == tid)) {
2079                         cs_etm__set_pid_tid_cpu(etm, tidq);
2080                         cs_etm__run_decoder(etmq);
2081                 }
2082         }
2083
2084         return 0;
2085 }
2086
2087 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm)
2088 {
2089         int ret = 0;
2090         unsigned int cs_queue_nr, queue_nr;
2091         u8 trace_chan_id;
2092         u64 timestamp;
2093         struct auxtrace_queue *queue;
2094         struct cs_etm_queue *etmq;
2095         struct cs_etm_traceid_queue *tidq;
2096
2097         while (1) {
2098                 if (!etm->heap.heap_cnt)
2099                         goto out;
2100
2101                 /* Take the entry at the top of the min heap */
2102                 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2103                 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2104                 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2105                 queue = &etm->queues.queue_array[queue_nr];
2106                 etmq = queue->priv;
2107
2108                 /*
2109                  * Remove the top entry from the heap since we are about
2110                  * to process it.
2111                  */
2112                 auxtrace_heap__pop(&etm->heap);
2113
2114                 tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2115                 if (!tidq) {
2116                         /*
2117                          * No traceID queue has been allocated for this traceID,
2118                          * which means something somewhere went very wrong.  No
2119                          * other choice than simply exit.
2120                          */
2121                         ret = -EINVAL;
2122                         goto out;
2123                 }
2124
2125                 /*
2126                  * Packets associated with this timestamp are already in
2127                  * the etmq's traceID queue, so process them.
2128                  */
2129                 ret = cs_etm__process_traceid_queue(etmq, tidq);
2130                 if (ret < 0)
2131                         goto out;
2132
2133                 /*
2134                  * Packets for this timestamp have been processed, time to
2135                  * move on to the next timestamp, fetching a new auxtrace_buffer
2136                  * if need be.
2137                  */
2138 refetch:
2139                 ret = cs_etm__get_data_block(etmq);
2140                 if (ret < 0)
2141                         goto out;
2142
2143                 /*
2144                  * No more auxtrace_buffers to process in this etmq, simply
2145                  * move on to another entry in the auxtrace_heap.
2146                  */
2147                 if (!ret)
2148                         continue;
2149
2150                 ret = cs_etm__decode_data_block(etmq);
2151                 if (ret)
2152                         goto out;
2153
2154                 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2155
2156                 if (!timestamp) {
2157                         /*
2158                          * Function cs_etm__decode_data_block() returns when
2159                          * there is no more traces to decode in the current
2160                          * auxtrace_buffer OR when a timestamp has been
2161                          * encountered on any of the traceID queues.  Since we
2162                          * did not get a timestamp, there is no more traces to
2163                          * process in this auxtrace_buffer.  As such empty and
2164                          * flush all traceID queues.
2165                          */
2166                         cs_etm__clear_all_traceid_queues(etmq);
2167
2168                         /* Fetch another auxtrace_buffer for this etmq */
2169                         goto refetch;
2170                 }
2171
2172                 /*
2173                  * Add to the min heap the timestamp for packets that have
2174                  * just been decoded.  They will be processed and synthesized
2175                  * during the next call to cs_etm__process_traceid_queue() for
2176                  * this queue/traceID.
2177                  */
2178                 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2179                 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
2180         }
2181
2182 out:
2183         return ret;
2184 }
2185
2186 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2187                                         union perf_event *event)
2188 {
2189         struct thread *th;
2190
2191         if (etm->timeless_decoding)
2192                 return 0;
2193
2194         /*
2195          * Add the tid/pid to the log so that we can get a match when
2196          * we get a contextID from the decoder.
2197          */
2198         th = machine__findnew_thread(etm->machine,
2199                                      event->itrace_start.pid,
2200                                      event->itrace_start.tid);
2201         if (!th)
2202                 return -ENOMEM;
2203
2204         thread__put(th);
2205
2206         return 0;
2207 }
2208
2209 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2210                                            union perf_event *event)
2211 {
2212         struct thread *th;
2213         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2214
2215         /*
2216          * Context switch in per-thread mode are irrelevant since perf
2217          * will start/stop tracing as the process is scheduled.
2218          */
2219         if (etm->timeless_decoding)
2220                 return 0;
2221
2222         /*
2223          * SWITCH_IN events carry the next process to be switched out while
2224          * SWITCH_OUT events carry the process to be switched in.  As such
2225          * we don't care about IN events.
2226          */
2227         if (!out)
2228                 return 0;
2229
2230         /*
2231          * Add the tid/pid to the log so that we can get a match when
2232          * we get a contextID from the decoder.
2233          */
2234         th = machine__findnew_thread(etm->machine,
2235                                      event->context_switch.next_prev_pid,
2236                                      event->context_switch.next_prev_tid);
2237         if (!th)
2238                 return -ENOMEM;
2239
2240         thread__put(th);
2241
2242         return 0;
2243 }
2244
2245 static int cs_etm__process_event(struct perf_session *session,
2246                                  union perf_event *event,
2247                                  struct perf_sample *sample,
2248                                  struct perf_tool *tool)
2249 {
2250         int err = 0;
2251         u64 timestamp;
2252         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2253                                                    struct cs_etm_auxtrace,
2254                                                    auxtrace);
2255
2256         if (dump_trace)
2257                 return 0;
2258
2259         if (!tool->ordered_events) {
2260                 pr_err("CoreSight ETM Trace requires ordered events\n");
2261                 return -EINVAL;
2262         }
2263
2264         if (sample->time && (sample->time != (u64) -1))
2265                 timestamp = sample->time;
2266         else
2267                 timestamp = 0;
2268
2269         if (timestamp || etm->timeless_decoding) {
2270                 err = cs_etm__update_queues(etm);
2271                 if (err)
2272                         return err;
2273         }
2274
2275         if (etm->timeless_decoding &&
2276             event->header.type == PERF_RECORD_EXIT)
2277                 return cs_etm__process_timeless_queues(etm,
2278                                                        event->fork.tid);
2279
2280         if (event->header.type == PERF_RECORD_ITRACE_START)
2281                 return cs_etm__process_itrace_start(etm, event);
2282         else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2283                 return cs_etm__process_switch_cpu_wide(etm, event);
2284
2285         if (!etm->timeless_decoding &&
2286             event->header.type == PERF_RECORD_AUX)
2287                 return cs_etm__process_queues(etm);
2288
2289         return 0;
2290 }
2291
2292 static int cs_etm__process_auxtrace_event(struct perf_session *session,
2293                                           union perf_event *event,
2294                                           struct perf_tool *tool __maybe_unused)
2295 {
2296         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2297                                                    struct cs_etm_auxtrace,
2298                                                    auxtrace);
2299         if (!etm->data_queued) {
2300                 struct auxtrace_buffer *buffer;
2301                 off_t  data_offset;
2302                 int fd = perf_data__fd(session->data);
2303                 bool is_pipe = perf_data__is_pipe(session->data);
2304                 int err;
2305
2306                 if (is_pipe)
2307                         data_offset = 0;
2308                 else {
2309                         data_offset = lseek(fd, 0, SEEK_CUR);
2310                         if (data_offset == -1)
2311                                 return -errno;
2312                 }
2313
2314                 err = auxtrace_queues__add_event(&etm->queues, session,
2315                                                  event, data_offset, &buffer);
2316                 if (err)
2317                         return err;
2318
2319                 if (dump_trace)
2320                         if (auxtrace_buffer__get_data(buffer, fd)) {
2321                                 cs_etm__dump_event(etm, buffer);
2322                                 auxtrace_buffer__put_data(buffer);
2323                         }
2324         }
2325
2326         return 0;
2327 }
2328
2329 static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)
2330 {
2331         struct evsel *evsel;
2332         struct evlist *evlist = etm->session->evlist;
2333         bool timeless_decoding = true;
2334
2335         /*
2336          * Circle through the list of event and complain if we find one
2337          * with the time bit set.
2338          */
2339         evlist__for_each_entry(evlist, evsel) {
2340                 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
2341                         timeless_decoding = false;
2342         }
2343
2344         return timeless_decoding;
2345 }
2346
2347 static const char * const cs_etm_global_header_fmts[] = {
2348         [CS_HEADER_VERSION_0]   = "     Header version                 %llx\n",
2349         [CS_PMU_TYPE_CPUS]      = "     PMU type/num cpus              %llx\n",
2350         [CS_ETM_SNAPSHOT]       = "     Snapshot                       %llx\n",
2351 };
2352
2353 static const char * const cs_etm_priv_fmts[] = {
2354         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
2355         [CS_ETM_CPU]            = "     CPU                            %lld\n",
2356         [CS_ETM_ETMCR]          = "     ETMCR                          %llx\n",
2357         [CS_ETM_ETMTRACEIDR]    = "     ETMTRACEIDR                    %llx\n",
2358         [CS_ETM_ETMCCER]        = "     ETMCCER                        %llx\n",
2359         [CS_ETM_ETMIDR]         = "     ETMIDR                         %llx\n",
2360 };
2361
2362 static const char * const cs_etmv4_priv_fmts[] = {
2363         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
2364         [CS_ETM_CPU]            = "     CPU                            %lld\n",
2365         [CS_ETMV4_TRCCONFIGR]   = "     TRCCONFIGR                     %llx\n",
2366         [CS_ETMV4_TRCTRACEIDR]  = "     TRCTRACEIDR                    %llx\n",
2367         [CS_ETMV4_TRCIDR0]      = "     TRCIDR0                        %llx\n",
2368         [CS_ETMV4_TRCIDR1]      = "     TRCIDR1                        %llx\n",
2369         [CS_ETMV4_TRCIDR2]      = "     TRCIDR2                        %llx\n",
2370         [CS_ETMV4_TRCIDR8]      = "     TRCIDR8                        %llx\n",
2371         [CS_ETMV4_TRCAUTHSTATUS] = "    TRCAUTHSTATUS                  %llx\n",
2372 };
2373
2374 static void cs_etm__print_auxtrace_info(__u64 *val, int num)
2375 {
2376         int i, j, cpu = 0;
2377
2378         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2379                 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]);
2380
2381         for (i = CS_HEADER_VERSION_0_MAX; cpu < num; cpu++) {
2382                 if (val[i] == __perf_cs_etmv3_magic)
2383                         for (j = 0; j < CS_ETM_PRIV_MAX; j++, i++)
2384                                 fprintf(stdout, cs_etm_priv_fmts[j], val[i]);
2385                 else if (val[i] == __perf_cs_etmv4_magic)
2386                         for (j = 0; j < CS_ETMV4_PRIV_MAX; j++, i++)
2387                                 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]);
2388                 else
2389                         /* failure.. return */
2390                         return;
2391         }
2392 }
2393
2394 int cs_etm__process_auxtrace_info(union perf_event *event,
2395                                   struct perf_session *session)
2396 {
2397         struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
2398         struct cs_etm_auxtrace *etm = NULL;
2399         struct int_node *inode;
2400         unsigned int pmu_type;
2401         int event_header_size = sizeof(struct perf_event_header);
2402         int info_header_size;
2403         int total_size = auxtrace_info->header.size;
2404         int priv_size = 0;
2405         int num_cpu;
2406         int err = 0, idx = -1;
2407         int i, j, k;
2408         u64 *ptr, *hdr = NULL;
2409         u64 **metadata = NULL;
2410
2411         /*
2412          * sizeof(auxtrace_info_event::type) +
2413          * sizeof(auxtrace_info_event::reserved) == 8
2414          */
2415         info_header_size = 8;
2416
2417         if (total_size < (event_header_size + info_header_size))
2418                 return -EINVAL;
2419
2420         priv_size = total_size - event_header_size - info_header_size;
2421
2422         /* First the global part */
2423         ptr = (u64 *) auxtrace_info->priv;
2424
2425         /* Look for version '0' of the header */
2426         if (ptr[0] != 0)
2427                 return -EINVAL;
2428
2429         hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_0_MAX);
2430         if (!hdr)
2431                 return -ENOMEM;
2432
2433         /* Extract header information - see cs-etm.h for format */
2434         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2435                 hdr[i] = ptr[i];
2436         num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff;
2437         pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) &
2438                                     0xffffffff);
2439
2440         /*
2441          * Create an RB tree for traceID-metadata tuple.  Since the conversion
2442          * has to be made for each packet that gets decoded, optimizing access
2443          * in anything other than a sequential array is worth doing.
2444          */
2445         traceid_list = intlist__new(NULL);
2446         if (!traceid_list) {
2447                 err = -ENOMEM;
2448                 goto err_free_hdr;
2449         }
2450
2451         metadata = zalloc(sizeof(*metadata) * num_cpu);
2452         if (!metadata) {
2453                 err = -ENOMEM;
2454                 goto err_free_traceid_list;
2455         }
2456
2457         /*
2458          * The metadata is stored in the auxtrace_info section and encodes
2459          * the configuration of the ARM embedded trace macrocell which is
2460          * required by the trace decoder to properly decode the trace due
2461          * to its highly compressed nature.
2462          */
2463         for (j = 0; j < num_cpu; j++) {
2464                 if (ptr[i] == __perf_cs_etmv3_magic) {
2465                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2466                                              CS_ETM_PRIV_MAX);
2467                         if (!metadata[j]) {
2468                                 err = -ENOMEM;
2469                                 goto err_free_metadata;
2470                         }
2471                         for (k = 0; k < CS_ETM_PRIV_MAX; k++)
2472                                 metadata[j][k] = ptr[i + k];
2473
2474                         /* The traceID is our handle */
2475                         idx = metadata[j][CS_ETM_ETMTRACEIDR];
2476                         i += CS_ETM_PRIV_MAX;
2477                 } else if (ptr[i] == __perf_cs_etmv4_magic) {
2478                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2479                                              CS_ETMV4_PRIV_MAX);
2480                         if (!metadata[j]) {
2481                                 err = -ENOMEM;
2482                                 goto err_free_metadata;
2483                         }
2484                         for (k = 0; k < CS_ETMV4_PRIV_MAX; k++)
2485                                 metadata[j][k] = ptr[i + k];
2486
2487                         /* The traceID is our handle */
2488                         idx = metadata[j][CS_ETMV4_TRCTRACEIDR];
2489                         i += CS_ETMV4_PRIV_MAX;
2490                 }
2491
2492                 /* Get an RB node for this CPU */
2493                 inode = intlist__findnew(traceid_list, idx);
2494
2495                 /* Something went wrong, no need to continue */
2496                 if (!inode) {
2497                         err = -ENOMEM;
2498                         goto err_free_metadata;
2499                 }
2500
2501                 /*
2502                  * The node for that CPU should not be taken.
2503                  * Back out if that's the case.
2504                  */
2505                 if (inode->priv) {
2506                         err = -EINVAL;
2507                         goto err_free_metadata;
2508                 }
2509                 /* All good, associate the traceID with the metadata pointer */
2510                 inode->priv = metadata[j];
2511         }
2512
2513         /*
2514          * Each of CS_HEADER_VERSION_0_MAX, CS_ETM_PRIV_MAX and
2515          * CS_ETMV4_PRIV_MAX mark how many double words are in the
2516          * global metadata, and each cpu's metadata respectively.
2517          * The following tests if the correct number of double words was
2518          * present in the auxtrace info section.
2519          */
2520         if (i * 8 != priv_size) {
2521                 err = -EINVAL;
2522                 goto err_free_metadata;
2523         }
2524
2525         etm = zalloc(sizeof(*etm));
2526
2527         if (!etm) {
2528                 err = -ENOMEM;
2529                 goto err_free_metadata;
2530         }
2531
2532         err = auxtrace_queues__init(&etm->queues);
2533         if (err)
2534                 goto err_free_etm;
2535
2536         etm->session = session;
2537         etm->machine = &session->machines.host;
2538
2539         etm->num_cpu = num_cpu;
2540         etm->pmu_type = pmu_type;
2541         etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);
2542         etm->metadata = metadata;
2543         etm->auxtrace_type = auxtrace_info->type;
2544         etm->timeless_decoding = cs_etm__is_timeless_decoding(etm);
2545
2546         etm->auxtrace.process_event = cs_etm__process_event;
2547         etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
2548         etm->auxtrace.flush_events = cs_etm__flush_events;
2549         etm->auxtrace.free_events = cs_etm__free_events;
2550         etm->auxtrace.free = cs_etm__free;
2551         session->auxtrace = &etm->auxtrace;
2552
2553         etm->unknown_thread = thread__new(999999999, 999999999);
2554         if (!etm->unknown_thread) {
2555                 err = -ENOMEM;
2556                 goto err_free_queues;
2557         }
2558
2559         /*
2560          * Initialize list node so that at thread__zput() we can avoid
2561          * segmentation fault at list_del_init().
2562          */
2563         INIT_LIST_HEAD(&etm->unknown_thread->node);
2564
2565         err = thread__set_comm(etm->unknown_thread, "unknown", 0);
2566         if (err)
2567                 goto err_delete_thread;
2568
2569         if (thread__init_map_groups(etm->unknown_thread, etm->machine)) {
2570                 err = -ENOMEM;
2571                 goto err_delete_thread;
2572         }
2573
2574         if (dump_trace) {
2575                 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu);
2576                 return 0;
2577         }
2578
2579         if (session->itrace_synth_opts->set) {
2580                 etm->synth_opts = *session->itrace_synth_opts;
2581         } else {
2582                 itrace_synth_opts__set_default(&etm->synth_opts,
2583                                 session->itrace_synth_opts->default_no_sample);
2584                 etm->synth_opts.callchain = false;
2585         }
2586
2587         err = cs_etm__synth_events(etm, session);
2588         if (err)
2589                 goto err_delete_thread;
2590
2591         err = auxtrace_queues__process_index(&etm->queues, session);
2592         if (err)
2593                 goto err_delete_thread;
2594
2595         etm->data_queued = etm->queues.populated;
2596
2597         return 0;
2598
2599 err_delete_thread:
2600         thread__zput(etm->unknown_thread);
2601 err_free_queues:
2602         auxtrace_queues__free(&etm->queues);
2603         session->auxtrace = NULL;
2604 err_free_etm:
2605         zfree(&etm);
2606 err_free_metadata:
2607         /* No need to check @metadata[j], free(NULL) is supported */
2608         for (j = 0; j < num_cpu; j++)
2609                 zfree(&metadata[j]);
2610         zfree(&metadata);
2611 err_free_traceid_list:
2612         intlist__delete(traceid_list);
2613 err_free_hdr:
2614         zfree(&hdr);
2615
2616         return err;
2617 }