tools/perf/arch/arm/util/cs-etm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015 Linaro Limited. All rights reserved.
   4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   5  */
   6
   7 #include <api/fs/fs.h>
   8 #include <linux/bits.h>
   9 #include <linux/bitops.h>
  10 #include <linux/compiler.h>
  11 #include <linux/coresight-pmu.h>
  12 #include <linux/kernel.h>
  13 #include <linux/log2.h>
  14 #include <linux/string.h>
  15 #include <linux/types.h>
  16 #include <linux/zalloc.h>
  17
  18 #include "cs-etm.h"
  19 #include "../../util/debug.h"
  20 #include "../../util/record.h"
  21 #include "../../util/auxtrace.h"
  22 #include "../../util/cpumap.h"
  23 #include "../../util/event.h"
  24 #include "../../util/evlist.h"
  25 #include "../../util/evsel.h"
  26 #include "../../util/perf_api_probe.h"
  27 #include "../../util/evsel_config.h"
  28 #include "../../util/pmu.h"
  29 #include "../../util/cs-etm.h"
  30 #include <internal/lib.h> // page_size
  31 #include "../../util/session.h"
  32
  33 #include <errno.h>
  34 #include <stdlib.h>
  35 #include <sys/stat.h>
  36
  37 struct cs_etm_recording {
  38         struct auxtrace_record  itr;
  39         struct perf_pmu         *cs_etm_pmu;
  40         struct evlist           *evlist;
  41         int                     wrapped_cnt;
  42         bool                    *wrapped;
  43         bool                    snapshot_mode;
  44         size_t                  snapshot_size;
  45 };
  46
  47 static const char *metadata_etmv3_ro[CS_ETM_PRIV_MAX] = {
  48         [CS_ETM_ETMCCER]        = "mgmt/etmccer",
  49         [CS_ETM_ETMIDR]         = "mgmt/etmidr",
  50 };
  51
  52 static const char *metadata_etmv4_ro[CS_ETMV4_PRIV_MAX] = {
  53         [CS_ETMV4_TRCIDR0]              = "trcidr/trcidr0",
  54         [CS_ETMV4_TRCIDR1]              = "trcidr/trcidr1",
  55         [CS_ETMV4_TRCIDR2]              = "trcidr/trcidr2",
  56         [CS_ETMV4_TRCIDR8]              = "trcidr/trcidr8",
  57         [CS_ETMV4_TRCAUTHSTATUS]        = "mgmt/trcauthstatus",
  58 };
  59
  60 static bool cs_etm_is_etmv4(struct auxtrace_record *itr, int cpu);
  61
  62 static int cs_etm_set_context_id(struct auxtrace_record *itr,
  63                                  struct evsel *evsel, int cpu)
  64 {
  65         struct cs_etm_recording *ptr;
  66         struct perf_pmu *cs_etm_pmu;
  67         char path[PATH_MAX];
  68         int err = -EINVAL;
  69         u32 val;
  70         u64 contextid;
  71
  72         ptr = container_of(itr, struct cs_etm_recording, itr);
  73         cs_etm_pmu = ptr->cs_etm_pmu;
  74
  75         if (!cs_etm_is_etmv4(itr, cpu))
  76                 goto out;
  77
  78         /* Get a handle on TRCIRD2 */
  79         snprintf(path, PATH_MAX, "cpu%d/%s",
  80                  cpu, metadata_etmv4_ro[CS_ETMV4_TRCIDR2]);
  81         err = perf_pmu__scan_file(cs_etm_pmu, path, "%x", &val);
  82
  83         /* There was a problem reading the file, bailing out */
  84         if (err != 1) {
  85                 pr_err("%s: can't read file %s\n",
  86                        CORESIGHT_ETM_PMU_NAME, path);
  87                 goto out;
  88         }
  89
  90         /* User has configured for PID tracing, respects it. */
  91         contextid = evsel->core.attr.config &
  92                         (BIT(ETM_OPT_CTXTID) | BIT(ETM_OPT_CTXTID2));
  93
  94         /*
  95          * If user doesn't configure the contextid format, parse PMU format and
  96          * enable PID tracing according to the "contextid" format bits:
  97          *
  98          *   If bit ETM_OPT_CTXTID is set, trace CONTEXTIDR_EL1;
  99          *   If bit ETM_OPT_CTXTID2 is set, trace CONTEXTIDR_EL2.
 100          */
 101         if (!contextid)
 102                 contextid = perf_pmu__format_bits(&cs_etm_pmu->format,
 103                                                   "contextid");
 104
 105         if (contextid & BIT(ETM_OPT_CTXTID)) {
 106                 /*
 107                  * TRCIDR2.CIDSIZE, bit [9-5], indicates whether contextID
 108                  * tracing is supported:
 109                  *  0b00000 Context ID tracing is not supported.
 110                  *  0b00100 Maximum of 32-bit Context ID size.
 111                  *  All other values are reserved.
 112                  */
 113                 val = BMVAL(val, 5, 9);
 114                 if (!val || val != 0x4) {
 115                         pr_err("%s: CONTEXTIDR_EL1 isn't supported\n",
 116                                CORESIGHT_ETM_PMU_NAME);
 117                         err = -EINVAL;
 118                         goto out;
 119                 }
 120         }
 121
 122         if (contextid & BIT(ETM_OPT_CTXTID2)) {
 123                 /*
 124                  * TRCIDR2.VMIDOPT[30:29] != 0 and
 125                  * TRCIDR2.VMIDSIZE[14:10] == 0b00100 (32bit virtual contextid)
 126                  * We can't support CONTEXTIDR in VMID if the size of the
 127                  * virtual context id is < 32bit.
 128                  * Any value of VMIDSIZE >= 4 (i.e, > 32bit) is fine for us.
 129                  */
 130                 if (!BMVAL(val, 29, 30) || BMVAL(val, 10, 14) < 4) {
 131                         pr_err("%s: CONTEXTIDR_EL2 isn't supported\n",
 132                                CORESIGHT_ETM_PMU_NAME);
 133                         err = -EINVAL;
 134                         goto out;
 135                 }
 136         }
 137
 138         /* All good, let the kernel know */
 139         evsel->core.attr.config |= contextid;
 140         err = 0;
 141
 142 out:
 143         return err;
 144 }
 145
 146 static int cs_etm_set_timestamp(struct auxtrace_record *itr,
 147                                 struct evsel *evsel, int cpu)
 148 {
 149         struct cs_etm_recording *ptr;
 150         struct perf_pmu *cs_etm_pmu;
 151         char path[PATH_MAX];
 152         int err = -EINVAL;
 153         u32 val;
 154
 155         ptr = container_of(itr, struct cs_etm_recording, itr);
 156         cs_etm_pmu = ptr->cs_etm_pmu;
 157
 158         if (!cs_etm_is_etmv4(itr, cpu))
 159                 goto out;
 160
 161         /* Get a handle on TRCIRD0 */
 162         snprintf(path, PATH_MAX, "cpu%d/%s",
 163                  cpu, metadata_etmv4_ro[CS_ETMV4_TRCIDR0]);
 164         err = perf_pmu__scan_file(cs_etm_pmu, path, "%x", &val);
 165
 166         /* There was a problem reading the file, bailing out */
 167         if (err != 1) {
 168                 pr_err("%s: can't read file %s\n",
 169                        CORESIGHT_ETM_PMU_NAME, path);
 170                 goto out;
 171         }
 172
 173         /*
 174          * TRCIDR0.TSSIZE, bit [28-24], indicates whether global timestamping
 175          * is supported:
 176          *  0b00000 Global timestamping is not implemented
 177          *  0b00110 Implementation supports a maximum timestamp of 48bits.
 178          *  0b01000 Implementation supports a maximum timestamp of 64bits.
 179          */
 180         val &= GENMASK(28, 24);
 181         if (!val) {
 182                 err = -EINVAL;
 183                 goto out;
 184         }
 185
 186         /* All good, let the kernel know */
 187         evsel->core.attr.config |= (1 << ETM_OPT_TS);
 188         err = 0;
 189
 190 out:
 191         return err;
 192 }
 193
 194 #define ETM_SET_OPT_CTXTID      (1 << 0)
 195 #define ETM_SET_OPT_TS          (1 << 1)
 196 #define ETM_SET_OPT_MASK        (ETM_SET_OPT_CTXTID | ETM_SET_OPT_TS)
 197
 198 static int cs_etm_set_option(struct auxtrace_record *itr,
 199                              struct evsel *evsel, u32 option)
 200 {
 201         int i, err = -EINVAL;
 202         struct perf_cpu_map *event_cpus = evsel->evlist->core.cpus;
 203         struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL);
 204
 205         /* Set option of each CPU we have */
 206         for (i = 0; i < cpu__max_cpu(); i++) {
 207                 if (!cpu_map__has(event_cpus, i) ||
 208                     !cpu_map__has(online_cpus, i))
 209                         continue;
 210
 211                 if (option & BIT(ETM_OPT_CTXTID)) {
 212                         err = cs_etm_set_context_id(itr, evsel, i);
 213                         if (err)
 214                                 goto out;
 215                 }
 216                 if (option & BIT(ETM_OPT_TS)) {
 217                         err = cs_etm_set_timestamp(itr, evsel, i);
 218                         if (err)
 219                                 goto out;
 220                 }
 221                 if (option & ~(BIT(ETM_OPT_CTXTID) | BIT(ETM_OPT_TS)))
 222                         /* Nothing else is currently supported */
 223                         goto out;
 224         }
 225
 226         err = 0;
 227 out:
 228         perf_cpu_map__put(online_cpus);
 229         return err;
 230 }
 231
 232 static int cs_etm_parse_snapshot_options(struct auxtrace_record *itr,
 233                                          struct record_opts *opts,
 234                                          const char *str)
 235 {
 236         struct cs_etm_recording *ptr =
 237                                 container_of(itr, struct cs_etm_recording, itr);
 238         unsigned long long snapshot_size = 0;
 239         char *endptr;
 240
 241         if (str) {
 242                 snapshot_size = strtoull(str, &endptr, 0);
 243                 if (*endptr || snapshot_size > SIZE_MAX)
 244                         return -1;
 245         }
 246
 247         opts->auxtrace_snapshot_mode = true;
 248         opts->auxtrace_snapshot_size = snapshot_size;
 249         ptr->snapshot_size = snapshot_size;
 250
 251         return 0;
 252 }
 253
 254 static int cs_etm_set_sink_attr(struct perf_pmu *pmu,
 255                                 struct evsel *evsel)
 256 {
 257         char msg[BUFSIZ], path[PATH_MAX], *sink;
 258         struct evsel_config_term *term;
 259         int ret = -EINVAL;
 260         u32 hash;
 261
 262         if (evsel->core.attr.config2 & GENMASK(31, 0))
 263                 return 0;
 264
 265         list_for_each_entry(term, &evsel->config_terms, list) {
 266                 if (term->type != EVSEL__CONFIG_TERM_DRV_CFG)
 267                         continue;
 268
 269                 sink = term->val.str;
 270                 snprintf(path, PATH_MAX, "sinks/%s", sink);
 271
 272                 ret = perf_pmu__scan_file(pmu, path, "%x", &hash);
 273                 if (ret != 1) {
 274                         pr_err("failed to set sink \"%s\" on event %s with %d (%s)\n",
 275                                sink, evsel__name(evsel), errno,
 276                                str_error_r(errno, msg, sizeof(msg)));
 277                         return ret;
 278                 }
 279
 280                 evsel->core.attr.config2 |= hash;
 281                 return 0;
 282         }
 283
 284         /*
 285          * No sink was provided on the command line - allow the CoreSight
 286          * system to look for a default
 287          */
 288         return 0;
 289 }
 290
 291 static int cs_etm_recording_options(struct auxtrace_record *itr,
 292                                     struct evlist *evlist,
 293                                     struct record_opts *opts)
 294 {
 295         int ret;
 296         struct cs_etm_recording *ptr =
 297                                 container_of(itr, struct cs_etm_recording, itr);
 298         struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
 299         struct evsel *evsel, *cs_etm_evsel = NULL;
 300         struct perf_cpu_map *cpus = evlist->core.cpus;
 301         bool privileged = perf_event_paranoid_check(-1);
 302         int err = 0;
 303
 304         ptr->evlist = evlist;
 305         ptr->snapshot_mode = opts->auxtrace_snapshot_mode;
 306
 307         if (!record_opts__no_switch_events(opts) &&
 308             perf_can_record_switch_events())
 309                 opts->record_switch_events = true;
 310
 311         evlist__for_each_entry(evlist, evsel) {
 312                 if (evsel->core.attr.type == cs_etm_pmu->type) {
 313                         if (cs_etm_evsel) {
 314                                 pr_err("There may be only one %s event\n",
 315                                        CORESIGHT_ETM_PMU_NAME);
 316                                 return -EINVAL;
 317                         }
 318                         evsel->core.attr.freq = 0;
 319                         evsel->core.attr.sample_period = 1;
 320                         cs_etm_evsel = evsel;
 321                         opts->full_auxtrace = true;
 322                 }
 323         }
 324
 325         /* no need to continue if at least one event of interest was found */
 326         if (!cs_etm_evsel)
 327                 return 0;
 328
 329         ret = cs_etm_set_sink_attr(cs_etm_pmu, cs_etm_evsel);
 330         if (ret)
 331                 return ret;
 332
 333         if (opts->use_clockid) {
 334                 pr_err("Cannot use clockid (-k option) with %s\n",
 335                        CORESIGHT_ETM_PMU_NAME);
 336                 return -EINVAL;
 337         }
 338
 339         /* we are in snapshot mode */
 340         if (opts->auxtrace_snapshot_mode) {
 341                 /*
 342                  * No size were given to '-S' or '-m,', so go with
 343                  * the default
 344                  */
 345                 if (!opts->auxtrace_snapshot_size &&
 346                     !opts->auxtrace_mmap_pages) {
 347                         if (privileged) {
 348                                 opts->auxtrace_mmap_pages = MiB(4) / page_size;
 349                         } else {
 350                                 opts->auxtrace_mmap_pages =
 351                                                         KiB(128) / page_size;
 352                                 if (opts->mmap_pages == UINT_MAX)
 353                                         opts->mmap_pages = KiB(256) / page_size;
 354                         }
 355                 } else if (!opts->auxtrace_mmap_pages && !privileged &&
 356                                                 opts->mmap_pages == UINT_MAX) {
 357                         opts->mmap_pages = KiB(256) / page_size;
 358                 }
 359
 360                 /*
 361                  * '-m,xyz' was specified but no snapshot size, so make the
 362                  * snapshot size as big as the auxtrace mmap area.
 363                  */
 364                 if (!opts->auxtrace_snapshot_size) {
 365                         opts->auxtrace_snapshot_size =
 366                                 opts->auxtrace_mmap_pages * (size_t)page_size;
 367                 }
 368
 369                 /*
 370                  * -Sxyz was specified but no auxtrace mmap area, so make the
 371                  * auxtrace mmap area big enough to fit the requested snapshot
 372                  * size.
 373                  */
 374                 if (!opts->auxtrace_mmap_pages) {
 375                         size_t sz = opts->auxtrace_snapshot_size;
 376
 377                         sz = round_up(sz, page_size) / page_size;
 378                         opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
 379                 }
 380
 381                 /* Snapshost size can't be bigger than the auxtrace area */
 382                 if (opts->auxtrace_snapshot_size >
 383                                 opts->auxtrace_mmap_pages * (size_t)page_size) {
 384                         pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
 385                                opts->auxtrace_snapshot_size,
 386                                opts->auxtrace_mmap_pages * (size_t)page_size);
 387                         return -EINVAL;
 388                 }
 389
 390                 /* Something went wrong somewhere - this shouldn't happen */
 391                 if (!opts->auxtrace_snapshot_size ||
 392                     !opts->auxtrace_mmap_pages) {
 393                         pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
 394                         return -EINVAL;
 395                 }
 396         }
 397
 398         /* We are in full trace mode but '-m,xyz' wasn't specified */
 399         if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
 400                 if (privileged) {
 401                         opts->auxtrace_mmap_pages = MiB(4) / page_size;
 402                 } else {
 403                         opts->auxtrace_mmap_pages = KiB(128) / page_size;
 404                         if (opts->mmap_pages == UINT_MAX)
 405                                 opts->mmap_pages = KiB(256) / page_size;
 406                 }
 407
 408         }
 409
 410         /* Validate auxtrace_mmap_pages provided by user */
 411         if (opts->auxtrace_mmap_pages) {
 412                 unsigned int max_page = (KiB(128) / page_size);
 413                 size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
 414
 415                 if (!privileged &&
 416                     opts->auxtrace_mmap_pages > max_page) {
 417                         opts->auxtrace_mmap_pages = max_page;
 418                         pr_err("auxtrace too big, truncating to %d\n",
 419                                max_page);
 420                 }
 421
 422                 if (!is_power_of_2(sz)) {
 423                         pr_err("Invalid mmap size for %s: must be a power of 2\n",
 424                                CORESIGHT_ETM_PMU_NAME);
 425                         return -EINVAL;
 426                 }
 427         }
 428
 429         if (opts->auxtrace_snapshot_mode)
 430                 pr_debug2("%s snapshot size: %zu\n", CORESIGHT_ETM_PMU_NAME,
 431                           opts->auxtrace_snapshot_size);
 432
 433         /*
 434          * To obtain the auxtrace buffer file descriptor, the auxtrace
 435          * event must come first.
 436          */
 437         evlist__to_front(evlist, cs_etm_evsel);
 438
 439         /*
 440          * In the case of per-cpu mmaps, we need the CPU on the
 441          * AUX event.  We also need the contextID in order to be notified
 442          * when a context switch happened.
 443          */
 444         if (!perf_cpu_map__empty(cpus)) {
 445                 evsel__set_sample_bit(cs_etm_evsel, CPU);
 446
 447                 err = cs_etm_set_option(itr, cs_etm_evsel,
 448                                         BIT(ETM_OPT_CTXTID) | BIT(ETM_OPT_TS));
 449                 if (err)
 450                         goto out;
 451         }
 452
 453         /* Add dummy event to keep tracking */
 454         if (opts->full_auxtrace) {
 455                 struct evsel *tracking_evsel;
 456
 457                 err = parse_events(evlist, "dummy:u", NULL);
 458                 if (err)
 459                         goto out;
 460
 461                 tracking_evsel = evlist__last(evlist);
 462                 evlist__set_tracking_event(evlist, tracking_evsel);
 463
 464                 tracking_evsel->core.attr.freq = 0;
 465                 tracking_evsel->core.attr.sample_period = 1;
 466
 467                 /* In per-cpu case, always need the time of mmap events etc */
 468                 if (!perf_cpu_map__empty(cpus))
 469                         evsel__set_sample_bit(tracking_evsel, TIME);
 470         }
 471
 472 out:
 473         return err;
 474 }
 475
 476 static u64 cs_etm_get_config(struct auxtrace_record *itr)
 477 {
 478         u64 config = 0;
 479         struct cs_etm_recording *ptr =
 480                         container_of(itr, struct cs_etm_recording, itr);
 481         struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
 482         struct evlist *evlist = ptr->evlist;
 483         struct evsel *evsel;
 484
 485         evlist__for_each_entry(evlist, evsel) {
 486                 if (evsel->core.attr.type == cs_etm_pmu->type) {
 487                         /*
 488                          * Variable perf_event_attr::config is assigned to
 489                          * ETMv3/PTM.  The bit fields have been made to match
 490                          * the ETMv3.5 ETRMCR register specification.  See the
 491                          * PMU_FORMAT_ATTR() declarations in
 492                          * drivers/hwtracing/coresight/coresight-perf.c for
 493                          * details.
 494                          */
 495                         config = evsel->core.attr.config;
 496                         break;
 497                 }
 498         }
 499
 500         return config;
 501 }
 502
 503 #ifndef BIT
 504 #define BIT(N) (1UL << (N))
 505 #endif
 506
 507 static u64 cs_etmv4_get_config(struct auxtrace_record *itr)
 508 {
 509         u64 config = 0;
 510         u64 config_opts = 0;
 511
 512         /*
 513          * The perf event variable config bits represent both
 514          * the command line options and register programming
 515          * bits in ETMv3/PTM. For ETMv4 we must remap options
 516          * to real bits
 517          */
 518         config_opts = cs_etm_get_config(itr);
 519         if (config_opts & BIT(ETM_OPT_CYCACC))
 520                 config |= BIT(ETM4_CFG_BIT_CYCACC);
 521         if (config_opts & BIT(ETM_OPT_CTXTID))
 522                 config |= BIT(ETM4_CFG_BIT_CTXTID);
 523         if (config_opts & BIT(ETM_OPT_TS))
 524                 config |= BIT(ETM4_CFG_BIT_TS);
 525         if (config_opts & BIT(ETM_OPT_RETSTK))
 526                 config |= BIT(ETM4_CFG_BIT_RETSTK);
 527         if (config_opts & BIT(ETM_OPT_CTXTID2))
 528                 config |= BIT(ETM4_CFG_BIT_VMID) |
 529                           BIT(ETM4_CFG_BIT_VMID_OPT);
 530         return config;
 531 }
 532
 533 static size_t
 534 cs_etm_info_priv_size(struct auxtrace_record *itr __maybe_unused,
 535                       struct evlist *evlist __maybe_unused)
 536 {
 537         int i;
 538         int etmv3 = 0, etmv4 = 0;
 539         struct perf_cpu_map *event_cpus = evlist->core.cpus;
 540         struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL);
 541
 542         /* cpu map is not empty, we have specific CPUs to work with */
 543         if (!perf_cpu_map__empty(event_cpus)) {
 544                 for (i = 0; i < cpu__max_cpu(); i++) {
 545                         if (!cpu_map__has(event_cpus, i) ||
 546                             !cpu_map__has(online_cpus, i))
 547                                 continue;
 548
 549                         if (cs_etm_is_etmv4(itr, i))
 550                                 etmv4++;
 551                         else
 552                                 etmv3++;
 553                 }
 554         } else {
 555                 /* get configuration for all CPUs in the system */
 556                 for (i = 0; i < cpu__max_cpu(); i++) {
 557                         if (!cpu_map__has(online_cpus, i))
 558                                 continue;
 559
 560                         if (cs_etm_is_etmv4(itr, i))
 561                                 etmv4++;
 562                         else
 563                                 etmv3++;
 564                 }
 565         }
 566
 567         perf_cpu_map__put(online_cpus);
 568
 569         return (CS_ETM_HEADER_SIZE +
 570                (etmv4 * CS_ETMV4_PRIV_SIZE) +
 571                (etmv3 * CS_ETMV3_PRIV_SIZE));
 572 }
 573
 574 static bool cs_etm_is_etmv4(struct auxtrace_record *itr, int cpu)
 575 {
 576         bool ret = false;
 577         char path[PATH_MAX];
 578         int scan;
 579         unsigned int val;
 580         struct cs_etm_recording *ptr =
 581                         container_of(itr, struct cs_etm_recording, itr);
 582         struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
 583
 584         /* Take any of the RO files for ETMv4 and see if it present */
 585         snprintf(path, PATH_MAX, "cpu%d/%s",
 586                  cpu, metadata_etmv4_ro[CS_ETMV4_TRCIDR0]);
 587         scan = perf_pmu__scan_file(cs_etm_pmu, path, "%x", &val);
 588
 589         /* The file was read successfully, we have a winner */
 590         if (scan == 1)
 591                 ret = true;
 592
 593         return ret;
 594 }
 595
 596 static int cs_etm_get_ro(struct perf_pmu *pmu, int cpu, const char *path)
 597 {
 598         char pmu_path[PATH_MAX];
 599         int scan;
 600         unsigned int val = 0;
 601
 602         /* Get RO metadata from sysfs */
 603         snprintf(pmu_path, PATH_MAX, "cpu%d/%s", cpu, path);
 604
 605         scan = perf_pmu__scan_file(pmu, pmu_path, "%x", &val);
 606         if (scan != 1)
 607                 pr_err("%s: error reading: %s\n", __func__, pmu_path);
 608
 609         return val;
 610 }
 611
 612 static void cs_etm_get_metadata(int cpu, u32 *offset,
 613                                 struct auxtrace_record *itr,
 614                                 struct perf_record_auxtrace_info *info)
 615 {
 616         u32 increment, nr_trc_params;
 617         u64 magic;
 618         struct cs_etm_recording *ptr =
 619                         container_of(itr, struct cs_etm_recording, itr);
 620         struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
 621
 622         /* first see what kind of tracer this cpu is affined to */
 623         if (cs_etm_is_etmv4(itr, cpu)) {
 624                 magic = __perf_cs_etmv4_magic;
 625                 /* Get trace configuration register */
 626                 info->priv[*offset + CS_ETMV4_TRCCONFIGR] =
 627                                                 cs_etmv4_get_config(itr);
 628                 /* Get traceID from the framework */
 629                 info->priv[*offset + CS_ETMV4_TRCTRACEIDR] =
 630                                                 coresight_get_trace_id(cpu);
 631                 /* Get read-only information from sysFS */
 632                 info->priv[*offset + CS_ETMV4_TRCIDR0] =
 633                         cs_etm_get_ro(cs_etm_pmu, cpu,
 634                                       metadata_etmv4_ro[CS_ETMV4_TRCIDR0]);
 635                 info->priv[*offset + CS_ETMV4_TRCIDR1] =
 636                         cs_etm_get_ro(cs_etm_pmu, cpu,
 637                                       metadata_etmv4_ro[CS_ETMV4_TRCIDR1]);
 638                 info->priv[*offset + CS_ETMV4_TRCIDR2] =
 639                         cs_etm_get_ro(cs_etm_pmu, cpu,
 640                                       metadata_etmv4_ro[CS_ETMV4_TRCIDR2]);
 641                 info->priv[*offset + CS_ETMV4_TRCIDR8] =
 642                         cs_etm_get_ro(cs_etm_pmu, cpu,
 643                                       metadata_etmv4_ro[CS_ETMV4_TRCIDR8]);
 644                 info->priv[*offset + CS_ETMV4_TRCAUTHSTATUS] =
 645                         cs_etm_get_ro(cs_etm_pmu, cpu,
 646                                       metadata_etmv4_ro
 647                                       [CS_ETMV4_TRCAUTHSTATUS]);
 648
 649                 /* How much space was used */
 650                 increment = CS_ETMV4_PRIV_MAX;
 651                 nr_trc_params = CS_ETMV4_PRIV_MAX - CS_ETMV4_TRCCONFIGR;
 652         } else {
 653                 magic = __perf_cs_etmv3_magic;
 654                 /* Get configuration register */
 655                 info->priv[*offset + CS_ETM_ETMCR] = cs_etm_get_config(itr);
 656                 /* Get traceID from the framework */
 657                 info->priv[*offset + CS_ETM_ETMTRACEIDR] =
 658                                                 coresight_get_trace_id(cpu);
 659                 /* Get read-only information from sysFS */
 660                 info->priv[*offset + CS_ETM_ETMCCER] =
 661                         cs_etm_get_ro(cs_etm_pmu, cpu,
 662                                       metadata_etmv3_ro[CS_ETM_ETMCCER]);
 663                 info->priv[*offset + CS_ETM_ETMIDR] =
 664                         cs_etm_get_ro(cs_etm_pmu, cpu,
 665                                       metadata_etmv3_ro[CS_ETM_ETMIDR]);
 666
 667                 /* How much space was used */
 668                 increment = CS_ETM_PRIV_MAX;
 669                 nr_trc_params = CS_ETM_PRIV_MAX - CS_ETM_ETMCR;
 670         }
 671
 672         /* Build generic header portion */
 673         info->priv[*offset + CS_ETM_MAGIC] = magic;
 674         info->priv[*offset + CS_ETM_CPU] = cpu;
 675         info->priv[*offset + CS_ETM_NR_TRC_PARAMS] = nr_trc_params;
 676         /* Where the next CPU entry should start from */
 677         *offset += increment;
 678 }
 679
 680 static int cs_etm_info_fill(struct auxtrace_record *itr,
 681                             struct perf_session *session,
 682                             struct perf_record_auxtrace_info *info,
 683                             size_t priv_size)
 684 {
 685         int i;
 686         u32 offset;
 687         u64 nr_cpu, type;
 688         struct perf_cpu_map *cpu_map;
 689         struct perf_cpu_map *event_cpus = session->evlist->core.cpus;
 690         struct perf_cpu_map *online_cpus = perf_cpu_map__new(NULL);
 691         struct cs_etm_recording *ptr =
 692                         container_of(itr, struct cs_etm_recording, itr);
 693         struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
 694
 695         if (priv_size != cs_etm_info_priv_size(itr, session->evlist))
 696                 return -EINVAL;
 697
 698         if (!session->evlist->core.nr_mmaps)
 699                 return -EINVAL;
 700
 701         /* If the cpu_map is empty all online CPUs are involved */
 702         if (perf_cpu_map__empty(event_cpus)) {
 703                 cpu_map = online_cpus;
 704         } else {
 705                 /* Make sure all specified CPUs are online */
 706                 for (i = 0; i < perf_cpu_map__nr(event_cpus); i++) {
 707                         if (cpu_map__has(event_cpus, i) &&
 708                             !cpu_map__has(online_cpus, i))
 709                                 return -EINVAL;
 710                 }
 711
 712                 cpu_map = event_cpus;
 713         }
 714
 715         nr_cpu = perf_cpu_map__nr(cpu_map);
 716         /* Get PMU type as dynamically assigned by the core */
 717         type = cs_etm_pmu->type;
 718
 719         /* First fill out the session header */
 720         info->type = PERF_AUXTRACE_CS_ETM;
 721         info->priv[CS_HEADER_VERSION] = CS_HEADER_CURRENT_VERSION;
 722         info->priv[CS_PMU_TYPE_CPUS] = type << 32;
 723         info->priv[CS_PMU_TYPE_CPUS] |= nr_cpu;
 724         info->priv[CS_ETM_SNAPSHOT] = ptr->snapshot_mode;
 725
 726         offset = CS_ETM_SNAPSHOT + 1;
 727
 728         for (i = 0; i < cpu__max_cpu() && offset < priv_size; i++)
 729                 if (cpu_map__has(cpu_map, i))
 730                         cs_etm_get_metadata(i, &offset, itr, info);
 731
 732         perf_cpu_map__put(online_cpus);
 733
 734         return 0;
 735 }
 736
 737 static int cs_etm_alloc_wrapped_array(struct cs_etm_recording *ptr, int idx)
 738 {
 739         bool *wrapped;
 740         int cnt = ptr->wrapped_cnt;
 741
 742         /* Make @ptr->wrapped as big as @idx */
 743         while (cnt <= idx)
 744                 cnt++;
 745
 746         /*
 747          * Free'ed in cs_etm_recording_free().  Using realloc() to avoid
 748          * cross compilation problems where the host's system supports
 749          * reallocarray() but not the target.
 750          */
 751         wrapped = realloc(ptr->wrapped, cnt * sizeof(bool));
 752         if (!wrapped)
 753                 return -ENOMEM;
 754
 755         wrapped[cnt - 1] = false;
 756         ptr->wrapped_cnt = cnt;
 757         ptr->wrapped = wrapped;
 758
 759         return 0;
 760 }
 761
 762 static bool cs_etm_buffer_has_wrapped(unsigned char *buffer,
 763                                       size_t buffer_size, u64 head)
 764 {
 765         u64 i, watermark;
 766         u64 *buf = (u64 *)buffer;
 767         size_t buf_size = buffer_size;
 768
 769         /*
 770          * We want to look the very last 512 byte (chosen arbitrarily) in
 771          * the ring buffer.
 772          */
 773         watermark = buf_size - 512;
 774
 775         /*
 776          * @head is continuously increasing - if its value is equal or greater
 777          * than the size of the ring buffer, it has wrapped around.
 778          */
 779         if (head >= buffer_size)
 780                 return true;
 781
 782         /*
 783          * The value of @head is somewhere within the size of the ring buffer.
 784          * This can be that there hasn't been enough data to fill the ring
 785          * buffer yet or the trace time was so long that @head has numerically
 786          * wrapped around.  To find we need to check if we have data at the very
 787          * end of the ring buffer.  We can reliably do this because mmap'ed
 788          * pages are zeroed out and there is a fresh mapping with every new
 789          * session.
 790          */
 791
 792         /* @head is less than 512 byte from the end of the ring buffer */
 793         if (head > watermark)
 794                 watermark = head;
 795
 796         /*
 797          * Speed things up by using 64 bit transactions (see "u64 *buf" above)
 798          */
 799         watermark >>= 3;
 800         buf_size >>= 3;
 801
 802         /*
 803          * If we find trace data at the end of the ring buffer, @head has
 804          * been there and has numerically wrapped around at least once.
 805          */
 806         for (i = watermark; i < buf_size; i++)
 807                 if (buf[i])
 808                         return true;
 809
 810         return false;
 811 }
 812
 813 static int cs_etm_find_snapshot(struct auxtrace_record *itr,
 814                                 int idx, struct auxtrace_mmap *mm,
 815                                 unsigned char *data,
 816                                 u64 *head, u64 *old)
 817 {
 818         int err;
 819         bool wrapped;
 820         struct cs_etm_recording *ptr =
 821                         container_of(itr, struct cs_etm_recording, itr);
 822
 823         /*
 824          * Allocate memory to keep track of wrapping if this is the first
 825          * time we deal with this *mm.
 826          */
 827         if (idx >= ptr->wrapped_cnt) {
 828                 err = cs_etm_alloc_wrapped_array(ptr, idx);
 829                 if (err)
 830                         return err;
 831         }
 832
 833         /*
 834          * Check to see if *head has wrapped around.  If it hasn't only the
 835          * amount of data between *head and *old is snapshot'ed to avoid
 836          * bloating the perf.data file with zeros.  But as soon as *head has
 837          * wrapped around the entire size of the AUX ring buffer it taken.
 838          */
 839         wrapped = ptr->wrapped[idx];
 840         if (!wrapped && cs_etm_buffer_has_wrapped(data, mm->len, *head)) {
 841                 wrapped = true;
 842                 ptr->wrapped[idx] = true;
 843         }
 844
 845         pr_debug3("%s: mmap index %d old head %zu new head %zu size %zu\n",
 846                   __func__, idx, (size_t)*old, (size_t)*head, mm->len);
 847
 848         /* No wrap has occurred, we can just use *head and *old. */
 849         if (!wrapped)
 850                 return 0;
 851
 852         /*
 853          * *head has wrapped around - adjust *head and *old to pickup the
 854          * entire content of the AUX buffer.
 855          */
 856         if (*head >= mm->len) {
 857                 *old = *head - mm->len;
 858         } else {
 859                 *head += mm->len;
 860                 *old = *head - mm->len;
 861         }
 862
 863         return 0;
 864 }
 865
 866 static int cs_etm_snapshot_start(struct auxtrace_record *itr)
 867 {
 868         struct cs_etm_recording *ptr =
 869                         container_of(itr, struct cs_etm_recording, itr);
 870         struct evsel *evsel;
 871
 872         evlist__for_each_entry(ptr->evlist, evsel) {
 873                 if (evsel->core.attr.type == ptr->cs_etm_pmu->type)
 874                         return evsel__disable(evsel);
 875         }
 876         return -EINVAL;
 877 }
 878
 879 static int cs_etm_snapshot_finish(struct auxtrace_record *itr)
 880 {
 881         struct cs_etm_recording *ptr =
 882                         container_of(itr, struct cs_etm_recording, itr);
 883         struct evsel *evsel;
 884
 885         evlist__for_each_entry(ptr->evlist, evsel) {
 886                 if (evsel->core.attr.type == ptr->cs_etm_pmu->type)
 887                         return evsel__enable(evsel);
 888         }
 889         return -EINVAL;
 890 }
 891
 892 static u64 cs_etm_reference(struct auxtrace_record *itr __maybe_unused)
 893 {
 894         return (((u64) rand() <<  0) & 0x00000000FFFFFFFFull) |
 895                 (((u64) rand() << 32) & 0xFFFFFFFF00000000ull);
 896 }
 897
 898 static void cs_etm_recording_free(struct auxtrace_record *itr)
 899 {
 900         struct cs_etm_recording *ptr =
 901                         container_of(itr, struct cs_etm_recording, itr);
 902
 903         zfree(&ptr->wrapped);
 904         free(ptr);
 905 }
 906
 907 struct auxtrace_record *cs_etm_record_init(int *err)
 908 {
 909         struct perf_pmu *cs_etm_pmu;
 910         struct cs_etm_recording *ptr;
 911
 912         cs_etm_pmu = perf_pmu__find(CORESIGHT_ETM_PMU_NAME);
 913
 914         if (!cs_etm_pmu) {
 915                 *err = -EINVAL;
 916                 goto out;
 917         }
 918
 919         ptr = zalloc(sizeof(struct cs_etm_recording));
 920         if (!ptr) {
 921                 *err = -ENOMEM;
 922                 goto out;
 923         }
 924
 925         ptr->cs_etm_pmu                 = cs_etm_pmu;
 926         ptr->itr.pmu                    = cs_etm_pmu;
 927         ptr->itr.parse_snapshot_options = cs_etm_parse_snapshot_options;
 928         ptr->itr.recording_options      = cs_etm_recording_options;
 929         ptr->itr.info_priv_size         = cs_etm_info_priv_size;
 930         ptr->itr.info_fill              = cs_etm_info_fill;
 931         ptr->itr.find_snapshot          = cs_etm_find_snapshot;
 932         ptr->itr.snapshot_start         = cs_etm_snapshot_start;
 933         ptr->itr.snapshot_finish        = cs_etm_snapshot_finish;
 934         ptr->itr.reference              = cs_etm_reference;
 935         ptr->itr.free                   = cs_etm_recording_free;
 936         ptr->itr.read_finish            = auxtrace_record__read_finish;
 937
 938         *err = 0;
 939         return &ptr->itr;
 940 out:
 941         return NULL;
 942 }