drivers/hwtracing/coresight/coresight-etm4x-core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2014, The Linux Foundation. All rights reserved.
   4  */
   5
   6 #include <linux/bitops.h>
   7 #include <linux/kernel.h>
   8 #include <linux/moduleparam.h>
   9 #include <linux/init.h>
  10 #include <linux/types.h>
  11 #include <linux/device.h>
  12 #include <linux/io.h>
  13 #include <linux/err.h>
  14 #include <linux/fs.h>
  15 #include <linux/slab.h>
  16 #include <linux/delay.h>
  17 #include <linux/smp.h>
  18 #include <linux/sysfs.h>
  19 #include <linux/stat.h>
  20 #include <linux/clk.h>
  21 #include <linux/cpu.h>
  22 #include <linux/cpu_pm.h>
  23 #include <linux/coresight.h>
  24 #include <linux/coresight-pmu.h>
  25 #include <linux/pm_wakeup.h>
  26 #include <linux/amba/bus.h>
  27 #include <linux/seq_file.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/perf_event.h>
  30 #include <linux/platform_device.h>
  31 #include <linux/pm_runtime.h>
  32 #include <linux/property.h>
  33
  34 #include <asm/barrier.h>
  35 #include <asm/sections.h>
  36 #include <asm/sysreg.h>
  37 #include <asm/local.h>
  38 #include <asm/virt.h>
  39
  40 #include "coresight-etm4x.h"
  41 #include "coresight-etm-perf.h"
  42
  43 static int boot_enable;
  44 module_param(boot_enable, int, 0444);
  45 MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
  46
  47 #define PARAM_PM_SAVE_FIRMWARE    0 /* save self-hosted state as per firmware */
  48 #define PARAM_PM_SAVE_NEVER       1 /* never save any state */
  49 #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
  50
  51 static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
  52 module_param(pm_save_enable, int, 0444);
  53 MODULE_PARM_DESC(pm_save_enable,
  54         "Save/restore state on power down: 1 = never, 2 = self-hosted");
  55
  56 static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
  57 static void etm4_set_default_config(struct etmv4_config *config);
  58 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
  59                                   struct perf_event *event);
  60 static u64 etm4_get_access_type(struct etmv4_config *config);
  61
  62 static enum cpuhp_state hp_online;
  63
  64 struct etm4_init_arg {
  65         unsigned int            pid;
  66         struct etmv4_drvdata    *drvdata;
  67         struct csdev_access     *csa;
  68 };
  69
  70 /*
  71  * Check if TRCSSPCICRn(i) is implemented for a given instance.
  72  *
  73  * TRCSSPCICRn is implemented only if :
  74  *      TRCSSPCICR<n> is present only if all of the following are true:
  75  *              TRCIDR4.NUMSSCC > n.
  76  *              TRCIDR4.NUMPC > 0b0000 .
  77  *              TRCSSCSR<n>.PC == 0b1
  78  */
  79 static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n)
  80 {
  81         return (n < drvdata->nr_ss_cmp) &&
  82                drvdata->nr_pe &&
  83                (drvdata->config.ss_status[n] & TRCSSCSRn_PC);
  84 }
  85
  86 u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit)
  87 {
  88         u64 res = 0;
  89
  90         switch (offset) {
  91         ETM4x_READ_SYSREG_CASES(res)
  92         default :
  93                 pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n",
  94                          offset);
  95         }
  96
  97         if (!_relaxed)
  98                 __iormb(res);   /* Imitate the !relaxed I/O helpers */
  99
 100         return res;
 101 }
 102
 103 void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit)
 104 {
 105         if (!_relaxed)
 106                 __iowmb();      /* Imitate the !relaxed I/O helpers */
 107         if (!_64bit)
 108                 val &= GENMASK(31, 0);
 109
 110         switch (offset) {
 111         ETM4x_WRITE_SYSREG_CASES(val)
 112         default :
 113                 pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n",
 114                         offset);
 115         }
 116 }
 117
 118 static void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, struct csdev_access *csa)
 119 {
 120         /* Writing 0 to TRCOSLAR unlocks the trace registers */
 121         etm4x_relaxed_write32(csa, 0x0, TRCOSLAR);
 122         drvdata->os_unlock = true;
 123         isb();
 124 }
 125
 126 static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
 127 {
 128         if (!WARN_ON(!drvdata->csdev))
 129                 etm4_os_unlock_csa(drvdata, &drvdata->csdev->access);
 130
 131 }
 132
 133 static void etm4_os_lock(struct etmv4_drvdata *drvdata)
 134 {
 135         if (WARN_ON(!drvdata->csdev))
 136                 return;
 137
 138         /* Writing 0x1 to TRCOSLAR locks the trace registers */
 139         etm4x_relaxed_write32(&drvdata->csdev->access, 0x1, TRCOSLAR);
 140         drvdata->os_unlock = false;
 141         isb();
 142 }
 143
 144 static void etm4_cs_lock(struct etmv4_drvdata *drvdata,
 145                          struct csdev_access *csa)
 146 {
 147         /* Software Lock is only accessible via memory mapped interface */
 148         if (csa->io_mem)
 149                 CS_LOCK(csa->base);
 150 }
 151
 152 static void etm4_cs_unlock(struct etmv4_drvdata *drvdata,
 153                            struct csdev_access *csa)
 154 {
 155         if (csa->io_mem)
 156                 CS_UNLOCK(csa->base);
 157 }
 158
 159 static int etm4_cpu_id(struct coresight_device *csdev)
 160 {
 161         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 162
 163         return drvdata->cpu;
 164 }
 165
 166 static int etm4_trace_id(struct coresight_device *csdev)
 167 {
 168         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 169
 170         return drvdata->trcid;
 171 }
 172
 173 struct etm4_enable_arg {
 174         struct etmv4_drvdata *drvdata;
 175         int rc;
 176 };
 177
 178 #ifdef CONFIG_ETM4X_IMPDEF_FEATURE
 179
 180 #define HISI_HIP08_AMBA_ID              0x000b6d01
 181 #define ETM4_AMBA_MASK                  0xfffff
 182 #define HISI_HIP08_CORE_COMMIT_MASK     0x3000
 183 #define HISI_HIP08_CORE_COMMIT_SHIFT    12
 184 #define HISI_HIP08_CORE_COMMIT_FULL     0b00
 185 #define HISI_HIP08_CORE_COMMIT_LVL_1    0b01
 186 #define HISI_HIP08_CORE_COMMIT_REG      sys_reg(3, 1, 15, 2, 5)
 187
 188 struct etm4_arch_features {
 189         void (*arch_callback)(bool enable);
 190 };
 191
 192 static bool etm4_hisi_match_pid(unsigned int id)
 193 {
 194         return (id & ETM4_AMBA_MASK) == HISI_HIP08_AMBA_ID;
 195 }
 196
 197 static void etm4_hisi_config_core_commit(bool enable)
 198 {
 199         u8 commit = enable ? HISI_HIP08_CORE_COMMIT_LVL_1 :
 200                     HISI_HIP08_CORE_COMMIT_FULL;
 201         u64 val;
 202
 203         /*
 204          * bit 12 and 13 of HISI_HIP08_CORE_COMMIT_REG are used together
 205          * to set core-commit, 2'b00 means cpu is at full speed, 2'b01,
 206          * 2'b10, 2'b11 mean reduce pipeline speed, and 2'b01 means level-1
 207          * speed(minimun value). So bit 12 and 13 should be cleared together.
 208          */
 209         val = read_sysreg_s(HISI_HIP08_CORE_COMMIT_REG);
 210         val &= ~HISI_HIP08_CORE_COMMIT_MASK;
 211         val |= commit << HISI_HIP08_CORE_COMMIT_SHIFT;
 212         write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG);
 213 }
 214
 215 static struct etm4_arch_features etm4_features[] = {
 216         [ETM4_IMPDEF_HISI_CORE_COMMIT] = {
 217                 .arch_callback = etm4_hisi_config_core_commit,
 218         },
 219         {},
 220 };
 221
 222 static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
 223 {
 224         struct etm4_arch_features *ftr;
 225         int bit;
 226
 227         for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {
 228                 ftr = &etm4_features[bit];
 229
 230                 if (ftr->arch_callback)
 231                         ftr->arch_callback(true);
 232         }
 233 }
 234
 235 static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)
 236 {
 237         struct etm4_arch_features *ftr;
 238         int bit;
 239
 240         for_each_set_bit(bit, drvdata->arch_features, ETM4_IMPDEF_FEATURE_MAX) {
 241                 ftr = &etm4_features[bit];
 242
 243                 if (ftr->arch_callback)
 244                         ftr->arch_callback(false);
 245         }
 246 }
 247
 248 static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,
 249                                       unsigned int id)
 250 {
 251         if (etm4_hisi_match_pid(id))
 252                 set_bit(ETM4_IMPDEF_HISI_CORE_COMMIT, drvdata->arch_features);
 253 }
 254 #else
 255 static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
 256 {
 257 }
 258
 259 static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata)
 260 {
 261 }
 262
 263 static void etm4_check_arch_features(struct etmv4_drvdata *drvdata,
 264                                      unsigned int id)
 265 {
 266 }
 267 #endif /* CONFIG_ETM4X_IMPDEF_FEATURE */
 268
 269 static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
 270 {
 271         int i, rc;
 272         struct etmv4_config *config = &drvdata->config;
 273         struct coresight_device *csdev = drvdata->csdev;
 274         struct device *etm_dev = &csdev->dev;
 275         struct csdev_access *csa = &csdev->access;
 276
 277
 278         etm4_cs_unlock(drvdata, csa);
 279         etm4_enable_arch_specific(drvdata);
 280
 281         etm4_os_unlock(drvdata);
 282
 283         rc = coresight_claim_device_unlocked(csdev);
 284         if (rc)
 285                 goto done;
 286
 287         /* Disable the trace unit before programming trace registers */
 288         etm4x_relaxed_write32(csa, 0, TRCPRGCTLR);
 289
 290         /*
 291          * If we use system instructions, we need to synchronize the
 292          * write to the TRCPRGCTLR, before accessing the TRCSTATR.
 293          * See ARM IHI0064F, section
 294          * "4.3.7 Synchronization of register updates"
 295          */
 296         if (!csa->io_mem)
 297                 isb();
 298
 299         /* wait for TRCSTATR.IDLE to go up */
 300         if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
 301                 dev_err(etm_dev,
 302                         "timeout while waiting for Idle Trace Status\n");
 303         if (drvdata->nr_pe)
 304                 etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR);
 305         etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR);
 306         /* nothing specific implemented */
 307         etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR);
 308         etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R);
 309         etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R);
 310         if (drvdata->stallctl)
 311                 etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR);
 312         etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR);
 313         etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR);
 314         etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR);
 315         etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR);
 316         etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR);
 317         etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR);
 318         etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR);
 319         etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR);
 320         if (drvdata->nr_pe_cmp)
 321                 etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR);
 322         for (i = 0; i < drvdata->nrseqstate - 1; i++)
 323                 etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i));
 324         etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR);
 325         etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR);
 326         etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR);
 327         for (i = 0; i < drvdata->nr_cntr; i++) {
 328                 etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i));
 329                 etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i));
 330                 etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i));
 331         }
 332
 333         /*
 334          * Resource selector pair 0 is always implemented and reserved.  As
 335          * such start at 2.
 336          */
 337         for (i = 2; i < drvdata->nr_resource * 2; i++)
 338                 etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i));
 339
 340         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
 341                 /* always clear status bit on restart if using single-shot */
 342                 if (config->ss_ctrl[i] || config->ss_pe_cmp[i])
 343                         config->ss_status[i] &= ~BIT(31);
 344                 etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i));
 345                 etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i));
 346                 if (etm4x_sspcicrn_present(drvdata, i))
 347                         etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i));
 348         }
 349         for (i = 0; i < drvdata->nr_addr_cmp; i++) {
 350                 etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i));
 351                 etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i));
 352         }
 353         for (i = 0; i < drvdata->numcidc; i++)
 354                 etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i));
 355         etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0);
 356         if (drvdata->numcidc > 4)
 357                 etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1);
 358
 359         for (i = 0; i < drvdata->numvmidc; i++)
 360                 etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i));
 361         etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0);
 362         if (drvdata->numvmidc > 4)
 363                 etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1);
 364
 365         if (!drvdata->skip_power_up) {
 366                 u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR);
 367
 368                 /*
 369                  * Request to keep the trace unit powered and also
 370                  * emulation of powerdown
 371                  */
 372                 etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR);
 373         }
 374
 375         /* Enable the trace unit */
 376         etm4x_relaxed_write32(csa, 1, TRCPRGCTLR);
 377
 378         /* Synchronize the register updates for sysreg access */
 379         if (!csa->io_mem)
 380                 isb();
 381
 382         /* wait for TRCSTATR.IDLE to go back down to '0' */
 383         if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
 384                 dev_err(etm_dev,
 385                         "timeout while waiting for Idle Trace Status\n");
 386
 387         /*
 388          * As recommended by section 4.3.7 ("Synchronization when using the
 389          * memory-mapped interface") of ARM IHI 0064D
 390          */
 391         dsb(sy);
 392         isb();
 393
 394 done:
 395         etm4_cs_lock(drvdata, csa);
 396
 397         dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n",
 398                 drvdata->cpu, rc);
 399         return rc;
 400 }
 401
 402 static void etm4_enable_hw_smp_call(void *info)
 403 {
 404         struct etm4_enable_arg *arg = info;
 405
 406         if (WARN_ON(!arg))
 407                 return;
 408         arg->rc = etm4_enable_hw(arg->drvdata);
 409 }
 410
 411 /*
 412  * The goal of function etm4_config_timestamp_event() is to configure a
 413  * counter that will tell the tracer to emit a timestamp packet when it
 414  * reaches zero.  This is done in order to get a more fine grained idea
 415  * of when instructions are executed so that they can be correlated
 416  * with execution on other CPUs.
 417  *
 418  * To do this the counter itself is configured to self reload and
 419  * TRCRSCTLR1 (always true) used to get the counter to decrement.  From
 420  * there a resource selector is configured with the counter and the
 421  * timestamp control register to use the resource selector to trigger the
 422  * event that will insert a timestamp packet in the stream.
 423  */
 424 static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata)
 425 {
 426         int ctridx, ret = -EINVAL;
 427         int counter, rselector;
 428         u32 val = 0;
 429         struct etmv4_config *config = &drvdata->config;
 430
 431         /* No point in trying if we don't have at least one counter */
 432         if (!drvdata->nr_cntr)
 433                 goto out;
 434
 435         /* Find a counter that hasn't been initialised */
 436         for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++)
 437                 if (config->cntr_val[ctridx] == 0)
 438                         break;
 439
 440         /* All the counters have been configured already, bail out */
 441         if (ctridx == drvdata->nr_cntr) {
 442                 pr_debug("%s: no available counter found\n", __func__);
 443                 ret = -ENOSPC;
 444                 goto out;
 445         }
 446
 447         /*
 448          * Searching for an available resource selector to use, starting at
 449          * '2' since every implementation has at least 2 resource selector.
 450          * ETMIDR4 gives the number of resource selector _pairs_,
 451          * hence multiply by 2.
 452          */
 453         for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++)
 454                 if (!config->res_ctrl[rselector])
 455                         break;
 456
 457         if (rselector == drvdata->nr_resource * 2) {
 458                 pr_debug("%s: no available resource selector found\n",
 459                          __func__);
 460                 ret = -ENOSPC;
 461                 goto out;
 462         }
 463
 464         /* Remember what counter we used */
 465         counter = 1 << ctridx;
 466
 467         /*
 468          * Initialise original and reload counter value to the smallest
 469          * possible value in order to get as much precision as we can.
 470          */
 471         config->cntr_val[ctridx] = 1;
 472         config->cntrldvr[ctridx] = 1;
 473
 474         /* Set the trace counter control register */
 475         val =  0x1 << 16        |  /* Bit 16, reload counter automatically */
 476                0x0 << 7         |  /* Select single resource selector */
 477                0x1;                /* Resource selector 1, i.e always true */
 478
 479         config->cntr_ctrl[ctridx] = val;
 480
 481         val = 0x2 << 16         | /* Group 0b0010 - Counter and sequencers */
 482               counter << 0;       /* Counter to use */
 483
 484         config->res_ctrl[rselector] = val;
 485
 486         val = 0x0 << 7          | /* Select single resource selector */
 487               rselector;          /* Resource selector */
 488
 489         config->ts_ctrl = val;
 490
 491         ret = 0;
 492 out:
 493         return ret;
 494 }
 495
 496 static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
 497                                    struct perf_event *event)
 498 {
 499         int ret = 0;
 500         struct etmv4_config *config = &drvdata->config;
 501         struct perf_event_attr *attr = &event->attr;
 502
 503         if (!attr) {
 504                 ret = -EINVAL;
 505                 goto out;
 506         }
 507
 508         /* Clear configuration from previous run */
 509         memset(config, 0, sizeof(struct etmv4_config));
 510
 511         if (attr->exclude_kernel)
 512                 config->mode = ETM_MODE_EXCL_KERN;
 513
 514         if (attr->exclude_user)
 515                 config->mode = ETM_MODE_EXCL_USER;
 516
 517         /* Always start from the default config */
 518         etm4_set_default_config(config);
 519
 520         /* Configure filters specified on the perf cmd line, if any. */
 521         ret = etm4_set_event_filters(drvdata, event);
 522         if (ret)
 523                 goto out;
 524
 525         /* Go from generic option to ETMv4 specifics */
 526         if (attr->config & BIT(ETM_OPT_CYCACC)) {
 527                 config->cfg |= BIT(4);
 528                 /* TRM: Must program this for cycacc to work */
 529                 config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
 530         }
 531         if (attr->config & BIT(ETM_OPT_TS)) {
 532                 /*
 533                  * Configure timestamps to be emitted at regular intervals in
 534                  * order to correlate instructions executed on different CPUs
 535                  * (CPU-wide trace scenarios).
 536                  */
 537                 ret = etm4_config_timestamp_event(drvdata);
 538
 539                 /*
 540                  * No need to go further if timestamp intervals can't
 541                  * be configured.
 542                  */
 543                 if (ret)
 544                         goto out;
 545
 546                 /* bit[11], Global timestamp tracing bit */
 547                 config->cfg |= BIT(11);
 548         }
 549
 550         if (attr->config & BIT(ETM_OPT_CTXTID))
 551                 /* bit[6], Context ID tracing bit */
 552                 config->cfg |= BIT(ETM4_CFG_BIT_CTXTID);
 553
 554         /*
 555          * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID
 556          * for recording CONTEXTIDR_EL2.  Do not enable VMID tracing if the
 557          * kernel is not running in EL2.
 558          */
 559         if (attr->config & BIT(ETM_OPT_CTXTID2)) {
 560                 if (!is_kernel_in_hyp_mode()) {
 561                         ret = -EINVAL;
 562                         goto out;
 563                 }
 564                 config->cfg |= BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT);
 565         }
 566
 567         /* return stack - enable if selected and supported */
 568         if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
 569                 /* bit[12], Return stack enable bit */
 570                 config->cfg |= BIT(12);
 571
 572 out:
 573         return ret;
 574 }
 575
 576 static int etm4_enable_perf(struct coresight_device *csdev,
 577                             struct perf_event *event)
 578 {
 579         int ret = 0;
 580         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 581
 582         if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
 583                 ret = -EINVAL;
 584                 goto out;
 585         }
 586
 587         /* Configure the tracer based on the session's specifics */
 588         ret = etm4_parse_event_config(drvdata, event);
 589         if (ret)
 590                 goto out;
 591         /* And enable it */
 592         ret = etm4_enable_hw(drvdata);
 593
 594 out:
 595         return ret;
 596 }
 597
 598 static int etm4_enable_sysfs(struct coresight_device *csdev)
 599 {
 600         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 601         struct etm4_enable_arg arg = { };
 602         int ret;
 603
 604         spin_lock(&drvdata->spinlock);
 605
 606         /*
 607          * Executing etm4_enable_hw on the cpu whose ETM is being enabled
 608          * ensures that register writes occur when cpu is powered.
 609          */
 610         arg.drvdata = drvdata;
 611         ret = smp_call_function_single(drvdata->cpu,
 612                                        etm4_enable_hw_smp_call, &arg, 1);
 613         if (!ret)
 614                 ret = arg.rc;
 615         if (!ret)
 616                 drvdata->sticky_enable = true;
 617         spin_unlock(&drvdata->spinlock);
 618
 619         if (!ret)
 620                 dev_dbg(&csdev->dev, "ETM tracing enabled\n");
 621         return ret;
 622 }
 623
 624 static int etm4_enable(struct coresight_device *csdev,
 625                        struct perf_event *event, u32 mode)
 626 {
 627         int ret;
 628         u32 val;
 629         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 630
 631         val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode);
 632
 633         /* Someone is already using the tracer */
 634         if (val)
 635                 return -EBUSY;
 636
 637         switch (mode) {
 638         case CS_MODE_SYSFS:
 639                 ret = etm4_enable_sysfs(csdev);
 640                 break;
 641         case CS_MODE_PERF:
 642                 ret = etm4_enable_perf(csdev, event);
 643                 break;
 644         default:
 645                 ret = -EINVAL;
 646         }
 647
 648         /* The tracer didn't start */
 649         if (ret)
 650                 local_set(&drvdata->mode, CS_MODE_DISABLED);
 651
 652         return ret;
 653 }
 654
 655 static void etm4_disable_hw(void *info)
 656 {
 657         u32 control;
 658         u64 trfcr;
 659         struct etmv4_drvdata *drvdata = info;
 660         struct etmv4_config *config = &drvdata->config;
 661         struct coresight_device *csdev = drvdata->csdev;
 662         struct device *etm_dev = &csdev->dev;
 663         struct csdev_access *csa = &csdev->access;
 664         int i;
 665
 666         etm4_cs_unlock(drvdata, csa);
 667         etm4_disable_arch_specific(drvdata);
 668
 669         if (!drvdata->skip_power_up) {
 670                 /* power can be removed from the trace unit now */
 671                 control = etm4x_relaxed_read32(csa, TRCPDCR);
 672                 control &= ~TRCPDCR_PU;
 673                 etm4x_relaxed_write32(csa, control, TRCPDCR);
 674         }
 675
 676         control = etm4x_relaxed_read32(csa, TRCPRGCTLR);
 677
 678         /* EN, bit[0] Trace unit enable bit */
 679         control &= ~0x1;
 680
 681         /*
 682          * If the CPU supports v8.4 Trace filter Control,
 683          * set the ETM to trace prohibited region.
 684          */
 685         if (drvdata->trfc) {
 686                 trfcr = read_sysreg_s(SYS_TRFCR_EL1);
 687                 write_sysreg_s(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE),
 688                                SYS_TRFCR_EL1);
 689                 isb();
 690         }
 691         /*
 692          * Make sure everything completes before disabling, as recommended
 693          * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register,
 694          * SSTATUS") of ARM IHI 0064D
 695          */
 696         dsb(sy);
 697         isb();
 698         /* Trace synchronization barrier, is a nop if not supported */
 699         tsb_csync();
 700         etm4x_relaxed_write32(csa, control, TRCPRGCTLR);
 701
 702         /* wait for TRCSTATR.PMSTABLE to go to '1' */
 703         if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1))
 704                 dev_err(etm_dev,
 705                         "timeout while waiting for PM stable Trace Status\n");
 706         if (drvdata->trfc)
 707                 write_sysreg_s(trfcr, SYS_TRFCR_EL1);
 708
 709         /* read the status of the single shot comparators */
 710         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
 711                 config->ss_status[i] =
 712                         etm4x_relaxed_read32(csa, TRCSSCSRn(i));
 713         }
 714
 715         /* read back the current counter values */
 716         for (i = 0; i < drvdata->nr_cntr; i++) {
 717                 config->cntr_val[i] =
 718                         etm4x_relaxed_read32(csa, TRCCNTVRn(i));
 719         }
 720
 721         coresight_disclaim_device_unlocked(csdev);
 722         etm4_cs_lock(drvdata, csa);
 723
 724         dev_dbg(&drvdata->csdev->dev,
 725                 "cpu: %d disable smp call done\n", drvdata->cpu);
 726 }
 727
 728 static int etm4_disable_perf(struct coresight_device *csdev,
 729                              struct perf_event *event)
 730 {
 731         u32 control;
 732         struct etm_filters *filters = event->hw.addr_filters;
 733         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 734
 735         if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
 736                 return -EINVAL;
 737
 738         etm4_disable_hw(drvdata);
 739
 740         /*
 741          * Check if the start/stop logic was active when the unit was stopped.
 742          * That way we can re-enable the start/stop logic when the process is
 743          * scheduled again.  Configuration of the start/stop logic happens in
 744          * function etm4_set_event_filters().
 745          */
 746         control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR);
 747         /* TRCVICTLR::SSSTATUS, bit[9] */
 748         filters->ssstatus = (control & BIT(9));
 749
 750         return 0;
 751 }
 752
 753 static void etm4_disable_sysfs(struct coresight_device *csdev)
 754 {
 755         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 756
 757         /*
 758          * Taking hotplug lock here protects from clocks getting disabled
 759          * with tracing being left on (crash scenario) if user disable occurs
 760          * after cpu online mask indicates the cpu is offline but before the
 761          * DYING hotplug callback is serviced by the ETM driver.
 762          */
 763         cpus_read_lock();
 764         spin_lock(&drvdata->spinlock);
 765
 766         /*
 767          * Executing etm4_disable_hw on the cpu whose ETM is being disabled
 768          * ensures that register writes occur when cpu is powered.
 769          */
 770         smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);
 771
 772         spin_unlock(&drvdata->spinlock);
 773         cpus_read_unlock();
 774
 775         dev_dbg(&csdev->dev, "ETM tracing disabled\n");
 776 }
 777
 778 static void etm4_disable(struct coresight_device *csdev,
 779                          struct perf_event *event)
 780 {
 781         u32 mode;
 782         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 783
 784         /*
 785          * For as long as the tracer isn't disabled another entity can't
 786          * change its status.  As such we can read the status here without
 787          * fearing it will change under us.
 788          */
 789         mode = local_read(&drvdata->mode);
 790
 791         switch (mode) {
 792         case CS_MODE_DISABLED:
 793                 break;
 794         case CS_MODE_SYSFS:
 795                 etm4_disable_sysfs(csdev);
 796                 break;
 797         case CS_MODE_PERF:
 798                 etm4_disable_perf(csdev, event);
 799                 break;
 800         }
 801
 802         if (mode)
 803                 local_set(&drvdata->mode, CS_MODE_DISABLED);
 804 }
 805
 806 static const struct coresight_ops_source etm4_source_ops = {
 807         .cpu_id         = etm4_cpu_id,
 808         .trace_id       = etm4_trace_id,
 809         .enable         = etm4_enable,
 810         .disable        = etm4_disable,
 811 };
 812
 813 static const struct coresight_ops etm4_cs_ops = {
 814         .source_ops     = &etm4_source_ops,
 815 };
 816
 817 static inline bool cpu_supports_sysreg_trace(void)
 818 {
 819         u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
 820
 821         return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0;
 822 }
 823
 824 static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata,
 825                                     struct csdev_access *csa)
 826 {
 827         u32 devarch;
 828
 829         if (!cpu_supports_sysreg_trace())
 830                 return false;
 831
 832         /*
 833          * ETMs implementing sysreg access must implement TRCDEVARCH.
 834          */
 835         devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH);
 836         if ((devarch & ETM_DEVARCH_ID_MASK) != ETM_DEVARCH_ETMv4x_ARCH)
 837                 return false;
 838         *csa = (struct csdev_access) {
 839                 .io_mem = false,
 840                 .read   = etm4x_sysreg_read,
 841                 .write  = etm4x_sysreg_write,
 842         };
 843
 844         drvdata->arch = etm_devarch_to_arch(devarch);
 845         return true;
 846 }
 847
 848 static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata,
 849                                    struct csdev_access *csa)
 850 {
 851         u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH);
 852         u32 idr1 = readl_relaxed(drvdata->base + TRCIDR1);
 853
 854         /*
 855          * All ETMs must implement TRCDEVARCH to indicate that
 856          * the component is an ETMv4. To support any broken
 857          * implementations we fall back to TRCIDR1 check, which
 858          * is not really reliable.
 859          */
 860         if ((devarch & ETM_DEVARCH_ID_MASK) == ETM_DEVARCH_ETMv4x_ARCH) {
 861                 drvdata->arch = etm_devarch_to_arch(devarch);
 862         } else {
 863                 pr_warn("CPU%d: ETM4x incompatible TRCDEVARCH: %x, falling back to TRCIDR1\n",
 864                         smp_processor_id(), devarch);
 865
 866                 if (ETM_TRCIDR1_ARCH_MAJOR(idr1) != ETM_TRCIDR1_ARCH_ETMv4)
 867                         return false;
 868                 drvdata->arch = etm_trcidr_to_arch(idr1);
 869         }
 870
 871         *csa = CSDEV_ACCESS_IOMEM(drvdata->base);
 872         return true;
 873 }
 874
 875 static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata,
 876                                    struct csdev_access *csa)
 877 {
 878         /*
 879          * Always choose the memory mapped io, if there is
 880          * a memory map to prevent sysreg access on broken
 881          * systems.
 882          */
 883         if (drvdata->base)
 884                 return etm4_init_iomem_access(drvdata, csa);
 885
 886         if (etm4_init_sysreg_access(drvdata, csa))
 887                 return true;
 888
 889         return false;
 890 }
 891
 892 static void cpu_enable_tracing(struct etmv4_drvdata *drvdata)
 893 {
 894         u64 dfr0 = read_sysreg(id_aa64dfr0_el1);
 895         u64 trfcr;
 896
 897         if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT))
 898                 return;
 899
 900         drvdata->trfc = true;
 901         /*
 902          * If the CPU supports v8.4 SelfHosted Tracing, enable
 903          * tracing at the kernel EL and EL0, forcing to use the
 904          * virtual time as the timestamp.
 905          */
 906         trfcr = (TRFCR_ELx_TS_VIRTUAL |
 907                  TRFCR_ELx_ExTRE |
 908                  TRFCR_ELx_E0TRE);
 909
 910         /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */
 911         if (is_kernel_in_hyp_mode())
 912                 trfcr |= TRFCR_EL2_CX;
 913
 914         write_sysreg_s(trfcr, SYS_TRFCR_EL1);
 915 }
 916
 917 static void etm4_init_arch_data(void *info)
 918 {
 919         u32 etmidr0;
 920         u32 etmidr2;
 921         u32 etmidr3;
 922         u32 etmidr4;
 923         u32 etmidr5;
 924         struct etm4_init_arg *init_arg = info;
 925         struct etmv4_drvdata *drvdata;
 926         struct csdev_access *csa;
 927         int i;
 928
 929         drvdata = init_arg->drvdata;
 930         csa = init_arg->csa;
 931
 932         /*
 933          * If we are unable to detect the access mechanism,
 934          * or unable to detect the trace unit type, fail
 935          * early.
 936          */
 937         if (!etm4_init_csdev_access(drvdata, csa))
 938                 return;
 939
 940         /* Make sure all registers are accessible */
 941         etm4_os_unlock_csa(drvdata, csa);
 942         etm4_cs_unlock(drvdata, csa);
 943
 944         etm4_check_arch_features(drvdata, init_arg->pid);
 945
 946         /* find all capabilities of the tracing unit */
 947         etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0);
 948
 949         /* INSTP0, bits[2:1] P0 tracing support field */
 950         if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2))
 951                 drvdata->instrp0 = true;
 952         else
 953                 drvdata->instrp0 = false;
 954
 955         /* TRCBB, bit[5] Branch broadcast tracing support bit */
 956         if (BMVAL(etmidr0, 5, 5))
 957                 drvdata->trcbb = true;
 958         else
 959                 drvdata->trcbb = false;
 960
 961         /* TRCCOND, bit[6] Conditional instruction tracing support bit */
 962         if (BMVAL(etmidr0, 6, 6))
 963                 drvdata->trccond = true;
 964         else
 965                 drvdata->trccond = false;
 966
 967         /* TRCCCI, bit[7] Cycle counting instruction bit */
 968         if (BMVAL(etmidr0, 7, 7))
 969                 drvdata->trccci = true;
 970         else
 971                 drvdata->trccci = false;
 972
 973         /* RETSTACK, bit[9] Return stack bit */
 974         if (BMVAL(etmidr0, 9, 9))
 975                 drvdata->retstack = true;
 976         else
 977                 drvdata->retstack = false;
 978
 979         /* NUMEVENT, bits[11:10] Number of events field */
 980         drvdata->nr_event = BMVAL(etmidr0, 10, 11);
 981         /* QSUPP, bits[16:15] Q element support field */
 982         drvdata->q_support = BMVAL(etmidr0, 15, 16);
 983         /* TSSIZE, bits[28:24] Global timestamp size field */
 984         drvdata->ts_size = BMVAL(etmidr0, 24, 28);
 985
 986         /* maximum size of resources */
 987         etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2);
 988         /* CIDSIZE, bits[9:5] Indicates the Context ID size */
 989         drvdata->ctxid_size = BMVAL(etmidr2, 5, 9);
 990         /* VMIDSIZE, bits[14:10] Indicates the VMID size */
 991         drvdata->vmid_size = BMVAL(etmidr2, 10, 14);
 992         /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */
 993         drvdata->ccsize = BMVAL(etmidr2, 25, 28);
 994
 995         etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3);
 996         /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */
 997         drvdata->ccitmin = BMVAL(etmidr3, 0, 11);
 998         /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */
 999         drvdata->s_ex_level = BMVAL(etmidr3, 16, 19);
1000         drvdata->config.s_ex_level = drvdata->s_ex_level;
1001         /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */
1002         drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23);
1003
1004         /*
1005          * TRCERR, bit[24] whether a trace unit can trace a
1006          * system error exception.
1007          */
1008         if (BMVAL(etmidr3, 24, 24))
1009                 drvdata->trc_error = true;
1010         else
1011                 drvdata->trc_error = false;
1012
1013         /* SYNCPR, bit[25] implementation has a fixed synchronization period? */
1014         if (BMVAL(etmidr3, 25, 25))
1015                 drvdata->syncpr = true;
1016         else
1017                 drvdata->syncpr = false;
1018
1019         /* STALLCTL, bit[26] is stall control implemented? */
1020         if (BMVAL(etmidr3, 26, 26))
1021                 drvdata->stallctl = true;
1022         else
1023                 drvdata->stallctl = false;
1024
1025         /* SYSSTALL, bit[27] implementation can support stall control? */
1026         if (BMVAL(etmidr3, 27, 27))
1027                 drvdata->sysstall = true;
1028         else
1029                 drvdata->sysstall = false;
1030
1031         /*
1032          * NUMPROC - the number of PEs available for tracing, 5bits
1033          *         = TRCIDR3.bits[13:12]bits[30:28]
1034          *  bits[4:3] = TRCIDR3.bits[13:12] (since etm-v4.2, otherwise RES0)
1035          *  bits[3:0] = TRCIDR3.bits[30:28]
1036          */
1037         drvdata->nr_pe = (BMVAL(etmidr3, 12, 13) << 3) | BMVAL(etmidr3, 28, 30);
1038
1039         /* NOOVERFLOW, bit[31] is trace overflow prevention supported */
1040         if (BMVAL(etmidr3, 31, 31))
1041                 drvdata->nooverflow = true;
1042         else
1043                 drvdata->nooverflow = false;
1044
1045         /* number of resources trace unit supports */
1046         etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4);
1047         /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */
1048         drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3);
1049         /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */
1050         drvdata->nr_pe_cmp = BMVAL(etmidr4, 12, 15);
1051         /*
1052          * NUMRSPAIR, bits[19:16]
1053          * The number of resource pairs conveyed by the HW starts at 0, i.e a
1054          * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on.
1055          * As such add 1 to the value of NUMRSPAIR for a better representation.
1056          *
1057          * For ETM v4.3 and later, 0x0 means 0, and no pairs are available -
1058          * the default TRUE and FALSE resource selectors are omitted.
1059          * Otherwise for values 0x1 and above the number is N + 1 as per v4.2.
1060          */
1061         drvdata->nr_resource = BMVAL(etmidr4, 16, 19);
1062         if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0))
1063                 drvdata->nr_resource += 1;
1064         /*
1065          * NUMSSCC, bits[23:20] the number of single-shot
1066          * comparator control for tracing. Read any status regs as these
1067          * also contain RO capability data.
1068          */
1069         drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23);
1070         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1071                 drvdata->config.ss_status[i] =
1072                         etm4x_relaxed_read32(csa, TRCSSCSRn(i));
1073         }
1074         /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */
1075         drvdata->numcidc = BMVAL(etmidr4, 24, 27);
1076         /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */
1077         drvdata->numvmidc = BMVAL(etmidr4, 28, 31);
1078
1079         etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5);
1080         /* NUMEXTIN, bits[8:0] number of external inputs implemented */
1081         drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8);
1082         /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */
1083         drvdata->trcid_size = BMVAL(etmidr5, 16, 21);
1084         /* ATBTRIG, bit[22] implementation can support ATB triggers? */
1085         if (BMVAL(etmidr5, 22, 22))
1086                 drvdata->atbtrig = true;
1087         else
1088                 drvdata->atbtrig = false;
1089         /*
1090          * LPOVERRIDE, bit[23] implementation supports
1091          * low-power state override
1092          */
1093         if (BMVAL(etmidr5, 23, 23) && (!drvdata->skip_power_up))
1094                 drvdata->lpoverride = true;
1095         else
1096                 drvdata->lpoverride = false;
1097         /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */
1098         drvdata->nrseqstate = BMVAL(etmidr5, 25, 27);
1099         /* NUMCNTR, bits[30:28] number of counters available for tracing */
1100         drvdata->nr_cntr = BMVAL(etmidr5, 28, 30);
1101         etm4_cs_lock(drvdata, csa);
1102         cpu_enable_tracing(drvdata);
1103 }
1104
1105 static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config)
1106 {
1107         return etm4_get_access_type(config) << TRCVICTLR_EXLEVEL_SHIFT;
1108 }
1109
1110 /* Set ELx trace filter access in the TRCVICTLR register */
1111 static void etm4_set_victlr_access(struct etmv4_config *config)
1112 {
1113         config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK;
1114         config->vinst_ctrl |= etm4_get_victlr_access_type(config);
1115 }
1116
1117 static void etm4_set_default_config(struct etmv4_config *config)
1118 {
1119         /* disable all events tracing */
1120         config->eventctrl0 = 0x0;
1121         config->eventctrl1 = 0x0;
1122
1123         /* disable stalling */
1124         config->stall_ctrl = 0x0;
1125
1126         /* enable trace synchronization every 4096 bytes, if available */
1127         config->syncfreq = 0xC;
1128
1129         /* disable timestamp event */
1130         config->ts_ctrl = 0x0;
1131
1132         /* TRCVICTLR::EVENT = 0x01, select the always on logic */
1133         config->vinst_ctrl = BIT(0);
1134
1135         /* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */
1136         etm4_set_victlr_access(config);
1137 }
1138
1139 static u64 etm4_get_ns_access_type(struct etmv4_config *config)
1140 {
1141         u64 access_type = 0;
1142
1143         /*
1144          * EXLEVEL_NS, for NonSecure Exception levels.
1145          * The mask here is a generic value and must be
1146          * shifted to the corresponding field for the registers
1147          */
1148         if (!is_kernel_in_hyp_mode()) {
1149                 /* Stay away from hypervisor mode for non-VHE */
1150                 access_type =  ETM_EXLEVEL_NS_HYP;
1151                 if (config->mode & ETM_MODE_EXCL_KERN)
1152                         access_type |= ETM_EXLEVEL_NS_OS;
1153         } else if (config->mode & ETM_MODE_EXCL_KERN) {
1154                 access_type = ETM_EXLEVEL_NS_HYP;
1155         }
1156
1157         if (config->mode & ETM_MODE_EXCL_USER)
1158                 access_type |= ETM_EXLEVEL_NS_APP;
1159
1160         return access_type;
1161 }
1162
1163 /*
1164  * Construct the exception level masks for a given config.
1165  * This must be shifted to the corresponding register field
1166  * for usage.
1167  */
1168 static u64 etm4_get_access_type(struct etmv4_config *config)
1169 {
1170         /* All Secure exception levels are excluded from the trace */
1171         return etm4_get_ns_access_type(config) | (u64)config->s_ex_level;
1172 }
1173
1174 static u64 etm4_get_comparator_access_type(struct etmv4_config *config)
1175 {
1176         return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT;
1177 }
1178
1179 static void etm4_set_comparator_filter(struct etmv4_config *config,
1180                                        u64 start, u64 stop, int comparator)
1181 {
1182         u64 access_type = etm4_get_comparator_access_type(config);
1183
1184         /* First half of default address comparator */
1185         config->addr_val[comparator] = start;
1186         config->addr_acc[comparator] = access_type;
1187         config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
1188
1189         /* Second half of default address comparator */
1190         config->addr_val[comparator + 1] = stop;
1191         config->addr_acc[comparator + 1] = access_type;
1192         config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;
1193
1194         /*
1195          * Configure the ViewInst function to include this address range
1196          * comparator.
1197          *
1198          * @comparator is divided by two since it is the index in the
1199          * etmv4_config::addr_val array but register TRCVIIECTLR deals with
1200          * address range comparator _pairs_.
1201          *
1202          * Therefore:
1203          *      index 0 -> compatator pair 0
1204          *      index 2 -> comparator pair 1
1205          *      index 4 -> comparator pair 2
1206          *      ...
1207          *      index 14 -> comparator pair 7
1208          */
1209         config->viiectlr |= BIT(comparator / 2);
1210 }
1211
1212 static void etm4_set_start_stop_filter(struct etmv4_config *config,
1213                                        u64 address, int comparator,
1214                                        enum etm_addr_type type)
1215 {
1216         int shift;
1217         u64 access_type = etm4_get_comparator_access_type(config);
1218
1219         /* Configure the comparator */
1220         config->addr_val[comparator] = address;
1221         config->addr_acc[comparator] = access_type;
1222         config->addr_type[comparator] = type;
1223
1224         /*
1225          * Configure ViewInst Start-Stop control register.
1226          * Addresses configured to start tracing go from bit 0 to n-1,
1227          * while those configured to stop tracing from 16 to 16 + n-1.
1228          */
1229         shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
1230         config->vissctlr |= BIT(shift + comparator);
1231 }
1232
1233 static void etm4_set_default_filter(struct etmv4_config *config)
1234 {
1235         /* Trace everything 'default' filter achieved by no filtering */
1236         config->viiectlr = 0x0;
1237
1238         /*
1239          * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
1240          * in the started state
1241          */
1242         config->vinst_ctrl |= BIT(9);
1243         config->mode |= ETM_MODE_VIEWINST_STARTSTOP;
1244
1245         /* No start-stop filtering for ViewInst */
1246         config->vissctlr = 0x0;
1247 }
1248
1249 static void etm4_set_default(struct etmv4_config *config)
1250 {
1251         if (WARN_ON_ONCE(!config))
1252                 return;
1253
1254         /*
1255          * Make default initialisation trace everything
1256          *
1257          * This is done by a minimum default config sufficient to enable
1258          * full instruction trace - with a default filter for trace all
1259          * achieved by having no filtering.
1260          */
1261         etm4_set_default_config(config);
1262         etm4_set_default_filter(config);
1263 }
1264
1265 static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
1266 {
1267         int nr_comparator, index = 0;
1268         struct etmv4_config *config = &drvdata->config;
1269
1270         /*
1271          * nr_addr_cmp holds the number of comparator _pair_, so time 2
1272          * for the total number of comparators.
1273          */
1274         nr_comparator = drvdata->nr_addr_cmp * 2;
1275
1276         /* Go through the tally of comparators looking for a free one. */
1277         while (index < nr_comparator) {
1278                 switch (type) {
1279                 case ETM_ADDR_TYPE_RANGE:
1280                         if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
1281                             config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
1282                                 return index;
1283
1284                         /* Address range comparators go in pairs */
1285                         index += 2;
1286                         break;
1287                 case ETM_ADDR_TYPE_START:
1288                 case ETM_ADDR_TYPE_STOP:
1289                         if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
1290                                 return index;
1291
1292                         /* Start/stop address can have odd indexes */
1293                         index += 1;
1294                         break;
1295                 default:
1296                         return -EINVAL;
1297                 }
1298         }
1299
1300         /* If we are here all the comparators have been used. */
1301         return -ENOSPC;
1302 }
1303
1304 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
1305                                   struct perf_event *event)
1306 {
1307         int i, comparator, ret = 0;
1308         u64 address;
1309         struct etmv4_config *config = &drvdata->config;
1310         struct etm_filters *filters = event->hw.addr_filters;
1311
1312         if (!filters)
1313                 goto default_filter;
1314
1315         /* Sync events with what Perf got */
1316         perf_event_addr_filters_sync(event);
1317
1318         /*
1319          * If there are no filters to deal with simply go ahead with
1320          * the default filter, i.e the entire address range.
1321          */
1322         if (!filters->nr_filters)
1323                 goto default_filter;
1324
1325         for (i = 0; i < filters->nr_filters; i++) {
1326                 struct etm_filter *filter = &filters->etm_filter[i];
1327                 enum etm_addr_type type = filter->type;
1328
1329                 /* See if a comparator is free. */
1330                 comparator = etm4_get_next_comparator(drvdata, type);
1331                 if (comparator < 0) {
1332                         ret = comparator;
1333                         goto out;
1334                 }
1335
1336                 switch (type) {
1337                 case ETM_ADDR_TYPE_RANGE:
1338                         etm4_set_comparator_filter(config,
1339                                                    filter->start_addr,
1340                                                    filter->stop_addr,
1341                                                    comparator);
1342                         /*
1343                          * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
1344                          * in the started state
1345                          */
1346                         config->vinst_ctrl |= BIT(9);
1347
1348                         /* No start-stop filtering for ViewInst */
1349                         config->vissctlr = 0x0;
1350                         break;
1351                 case ETM_ADDR_TYPE_START:
1352                 case ETM_ADDR_TYPE_STOP:
1353                         /* Get the right start or stop address */
1354                         address = (type == ETM_ADDR_TYPE_START ?
1355                                    filter->start_addr :
1356                                    filter->stop_addr);
1357
1358                         /* Configure comparator */
1359                         etm4_set_start_stop_filter(config, address,
1360                                                    comparator, type);
1361
1362                         /*
1363                          * If filters::ssstatus == 1, trace acquisition was
1364                          * started but the process was yanked away before the
1365                          * the stop address was hit.  As such the start/stop
1366                          * logic needs to be re-started so that tracing can
1367                          * resume where it left.
1368                          *
1369                          * The start/stop logic status when a process is
1370                          * scheduled out is checked in function
1371                          * etm4_disable_perf().
1372                          */
1373                         if (filters->ssstatus)
1374                                 config->vinst_ctrl |= BIT(9);
1375
1376                         /* No include/exclude filtering for ViewInst */
1377                         config->viiectlr = 0x0;
1378                         break;
1379                 default:
1380                         ret = -EINVAL;
1381                         goto out;
1382                 }
1383         }
1384
1385         goto out;
1386
1387
1388 default_filter:
1389         etm4_set_default_filter(config);
1390
1391 out:
1392         return ret;
1393 }
1394
1395 void etm4_config_trace_mode(struct etmv4_config *config)
1396 {
1397         u32 mode;
1398
1399         mode = config->mode;
1400         mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);
1401
1402         /* excluding kernel AND user space doesn't make sense */
1403         WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));
1404
1405         /* nothing to do if neither flags are set */
1406         if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))
1407                 return;
1408
1409         etm4_set_victlr_access(config);
1410 }
1411
1412 static int etm4_online_cpu(unsigned int cpu)
1413 {
1414         if (!etmdrvdata[cpu])
1415                 return 0;
1416
1417         if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)
1418                 coresight_enable(etmdrvdata[cpu]->csdev);
1419         return 0;
1420 }
1421
1422 static int etm4_starting_cpu(unsigned int cpu)
1423 {
1424         if (!etmdrvdata[cpu])
1425                 return 0;
1426
1427         spin_lock(&etmdrvdata[cpu]->spinlock);
1428         if (!etmdrvdata[cpu]->os_unlock)
1429                 etm4_os_unlock(etmdrvdata[cpu]);
1430
1431         if (local_read(&etmdrvdata[cpu]->mode))
1432                 etm4_enable_hw(etmdrvdata[cpu]);
1433         spin_unlock(&etmdrvdata[cpu]->spinlock);
1434         return 0;
1435 }
1436
1437 static int etm4_dying_cpu(unsigned int cpu)
1438 {
1439         if (!etmdrvdata[cpu])
1440                 return 0;
1441
1442         spin_lock(&etmdrvdata[cpu]->spinlock);
1443         if (local_read(&etmdrvdata[cpu]->mode))
1444                 etm4_disable_hw(etmdrvdata[cpu]);
1445         spin_unlock(&etmdrvdata[cpu]->spinlock);
1446         return 0;
1447 }
1448
1449 static void etm4_init_trace_id(struct etmv4_drvdata *drvdata)
1450 {
1451         drvdata->trcid = coresight_get_trace_id(drvdata->cpu);
1452 }
1453
1454 static int etm4_cpu_save(struct etmv4_drvdata *drvdata)
1455 {
1456         int i, ret = 0;
1457         struct etmv4_save_state *state;
1458         struct coresight_device *csdev = drvdata->csdev;
1459         struct csdev_access *csa;
1460         struct device *etm_dev;
1461
1462         if (WARN_ON(!csdev))
1463                 return -ENODEV;
1464
1465         etm_dev = &csdev->dev;
1466         csa = &csdev->access;
1467
1468         /*
1469          * As recommended by 3.4.1 ("The procedure when powering down the PE")
1470          * of ARM IHI 0064D
1471          */
1472         dsb(sy);
1473         isb();
1474
1475         etm4_cs_unlock(drvdata, csa);
1476         /* Lock the OS lock to disable trace and external debugger access */
1477         etm4_os_lock(drvdata);
1478
1479         /* wait for TRCSTATR.PMSTABLE to go up */
1480         if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) {
1481                 dev_err(etm_dev,
1482                         "timeout while waiting for PM Stable Status\n");
1483                 etm4_os_unlock(drvdata);
1484                 ret = -EBUSY;
1485                 goto out;
1486         }
1487
1488         state = drvdata->save_state;
1489
1490         state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR);
1491         if (drvdata->nr_pe)
1492                 state->trcprocselr = etm4x_read32(csa, TRCPROCSELR);
1493         state->trcconfigr = etm4x_read32(csa, TRCCONFIGR);
1494         state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR);
1495         state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R);
1496         state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R);
1497         if (drvdata->stallctl)
1498                 state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR);
1499         state->trctsctlr = etm4x_read32(csa, TRCTSCTLR);
1500         state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR);
1501         state->trcccctlr = etm4x_read32(csa, TRCCCCTLR);
1502         state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR);
1503         state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR);
1504         state->trcqctlr = etm4x_read32(csa, TRCQCTLR);
1505
1506         state->trcvictlr = etm4x_read32(csa, TRCVICTLR);
1507         state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR);
1508         state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR);
1509         if (drvdata->nr_pe_cmp)
1510                 state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR);
1511         state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR);
1512         state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR);
1513         state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR);
1514
1515         for (i = 0; i < drvdata->nrseqstate - 1; i++)
1516                 state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i));
1517
1518         state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR);
1519         state->trcseqstr = etm4x_read32(csa, TRCSEQSTR);
1520         state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR);
1521
1522         for (i = 0; i < drvdata->nr_cntr; i++) {
1523                 state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i));
1524                 state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i));
1525                 state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i));
1526         }
1527
1528         for (i = 0; i < drvdata->nr_resource * 2; i++)
1529                 state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i));
1530
1531         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1532                 state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i));
1533                 state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i));
1534                 if (etm4x_sspcicrn_present(drvdata, i))
1535                         state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i));
1536         }
1537
1538         for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1539                 state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i));
1540                 state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i));
1541         }
1542
1543         /*
1544          * Data trace stream is architecturally prohibited for A profile cores
1545          * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per
1546          * section 1.3.4 ("Possible functional configurations of an ETMv4 trace
1547          * unit") of ARM IHI 0064D.
1548          */
1549
1550         for (i = 0; i < drvdata->numcidc; i++)
1551                 state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i));
1552
1553         for (i = 0; i < drvdata->numvmidc; i++)
1554                 state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i));
1555
1556         state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0);
1557         if (drvdata->numcidc > 4)
1558                 state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1);
1559
1560         state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0);
1561         if (drvdata->numvmidc > 4)
1562                 state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1);
1563
1564         state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR);
1565
1566         if (!drvdata->skip_power_up)
1567                 state->trcpdcr = etm4x_read32(csa, TRCPDCR);
1568
1569         /* wait for TRCSTATR.IDLE to go up */
1570         if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) {
1571                 dev_err(etm_dev,
1572                         "timeout while waiting for Idle Trace Status\n");
1573                 etm4_os_unlock(drvdata);
1574                 ret = -EBUSY;
1575                 goto out;
1576         }
1577
1578         drvdata->state_needs_restore = true;
1579
1580         /*
1581          * Power can be removed from the trace unit now. We do this to
1582          * potentially save power on systems that respect the TRCPDCR_PU
1583          * despite requesting software to save/restore state.
1584          */
1585         if (!drvdata->skip_power_up)
1586                 etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU),
1587                                       TRCPDCR);
1588 out:
1589         etm4_cs_lock(drvdata, csa);
1590         return ret;
1591 }
1592
1593 static void etm4_cpu_restore(struct etmv4_drvdata *drvdata)
1594 {
1595         int i;
1596         struct etmv4_save_state *state = drvdata->save_state;
1597         struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base);
1598         struct csdev_access *csa = &tmp_csa;
1599
1600         etm4_cs_unlock(drvdata, csa);
1601         etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
1602
1603         etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR);
1604         if (drvdata->nr_pe)
1605                 etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR);
1606         etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR);
1607         etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR);
1608         etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R);
1609         etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R);
1610         if (drvdata->stallctl)
1611                 etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR);
1612         etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR);
1613         etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR);
1614         etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR);
1615         etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR);
1616         etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR);
1617         etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR);
1618
1619         etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR);
1620         etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR);
1621         etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR);
1622         if (drvdata->nr_pe_cmp)
1623                 etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR);
1624         etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR);
1625         etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR);
1626         etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR);
1627
1628         for (i = 0; i < drvdata->nrseqstate - 1; i++)
1629                 etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i));
1630
1631         etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR);
1632         etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR);
1633         etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR);
1634
1635         for (i = 0; i < drvdata->nr_cntr; i++) {
1636                 etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i));
1637                 etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i));
1638                 etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i));
1639         }
1640
1641         for (i = 0; i < drvdata->nr_resource * 2; i++)
1642                 etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i));
1643
1644         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1645                 etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i));
1646                 etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i));
1647                 if (etm4x_sspcicrn_present(drvdata, i))
1648                         etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i));
1649         }
1650
1651         for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1652                 etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i));
1653                 etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i));
1654         }
1655
1656         for (i = 0; i < drvdata->numcidc; i++)
1657                 etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i));
1658
1659         for (i = 0; i < drvdata->numvmidc; i++)
1660                 etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i));
1661
1662         etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0);
1663         if (drvdata->numcidc > 4)
1664                 etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1);
1665
1666         etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0);
1667         if (drvdata->numvmidc > 4)
1668                 etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1);
1669
1670         etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
1671
1672         if (!drvdata->skip_power_up)
1673                 etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR);
1674
1675         drvdata->state_needs_restore = false;
1676
1677         /*
1678          * As recommended by section 4.3.7 ("Synchronization when using the
1679          * memory-mapped interface") of ARM IHI 0064D
1680          */
1681         dsb(sy);
1682         isb();
1683
1684         /* Unlock the OS lock to re-enable trace and external debug access */
1685         etm4_os_unlock(drvdata);
1686         etm4_cs_lock(drvdata, csa);
1687 }
1688
1689 static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
1690                               void *v)
1691 {
1692         struct etmv4_drvdata *drvdata;
1693         unsigned int cpu = smp_processor_id();
1694
1695         if (!etmdrvdata[cpu])
1696                 return NOTIFY_OK;
1697
1698         drvdata = etmdrvdata[cpu];
1699
1700         if (!drvdata->save_state)
1701                 return NOTIFY_OK;
1702
1703         if (WARN_ON_ONCE(drvdata->cpu != cpu))
1704                 return NOTIFY_BAD;
1705
1706         switch (cmd) {
1707         case CPU_PM_ENTER:
1708                 /* save the state if self-hosted coresight is in use */
1709                 if (local_read(&drvdata->mode))
1710                         if (etm4_cpu_save(drvdata))
1711                                 return NOTIFY_BAD;
1712                 break;
1713         case CPU_PM_EXIT:
1714         case CPU_PM_ENTER_FAILED:
1715                 if (drvdata->state_needs_restore)
1716                         etm4_cpu_restore(drvdata);
1717                 break;
1718         default:
1719                 return NOTIFY_DONE;
1720         }
1721
1722         return NOTIFY_OK;
1723 }
1724
1725 static struct notifier_block etm4_cpu_pm_nb = {
1726         .notifier_call = etm4_cpu_pm_notify,
1727 };
1728
1729 /* Setup PM. Deals with error conditions and counts */
1730 static int __init etm4_pm_setup(void)
1731 {
1732         int ret;
1733
1734         ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb);
1735         if (ret)
1736                 return ret;
1737
1738         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
1739                                         "arm/coresight4:starting",
1740                                         etm4_starting_cpu, etm4_dying_cpu);
1741
1742         if (ret)
1743                 goto unregister_notifier;
1744
1745         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
1746                                         "arm/coresight4:online",
1747                                         etm4_online_cpu, NULL);
1748
1749         /* HP dyn state ID returned in ret on success */
1750         if (ret > 0) {
1751                 hp_online = ret;
1752                 return 0;
1753         }
1754
1755         /* failed dyn state - remove others */
1756         cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1757
1758 unregister_notifier:
1759         cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
1760         return ret;
1761 }
1762
1763 static void etm4_pm_clear(void)
1764 {
1765         cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
1766         cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1767         if (hp_online) {
1768                 cpuhp_remove_state_nocalls(hp_online);
1769                 hp_online = 0;
1770         }
1771 }
1772
1773 static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid)
1774 {
1775         int ret;
1776         struct coresight_platform_data *pdata = NULL;
1777         struct etmv4_drvdata *drvdata;
1778         struct coresight_desc desc = { 0 };
1779         struct etm4_init_arg init_arg = { 0 };
1780
1781         drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1782         if (!drvdata)
1783                 return -ENOMEM;
1784
1785         dev_set_drvdata(dev, drvdata);
1786
1787         if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
1788                 pm_save_enable = coresight_loses_context_with_cpu(dev) ?
1789                                PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER;
1790
1791         if (pm_save_enable != PARAM_PM_SAVE_NEVER) {
1792                 drvdata->save_state = devm_kmalloc(dev,
1793                                 sizeof(struct etmv4_save_state), GFP_KERNEL);
1794                 if (!drvdata->save_state)
1795                         return -ENOMEM;
1796         }
1797
1798         drvdata->base = base;
1799
1800         spin_lock_init(&drvdata->spinlock);
1801
1802         drvdata->cpu = coresight_get_cpu(dev);
1803         if (drvdata->cpu < 0)
1804                 return drvdata->cpu;
1805
1806         desc.name = devm_kasprintf(dev, GFP_KERNEL, "etm%d", drvdata->cpu);
1807         if (!desc.name)
1808                 return -ENOMEM;
1809
1810         init_arg.drvdata = drvdata;
1811         init_arg.csa = &desc.access;
1812         init_arg.pid = etm_pid;
1813
1814         if (smp_call_function_single(drvdata->cpu,
1815                                 etm4_init_arch_data,  &init_arg, 1))
1816                 dev_err(dev, "ETM arch init failed\n");
1817
1818         if (!drvdata->arch)
1819                 return -EINVAL;
1820
1821         /* TRCPDCR is not accessible with system instructions. */
1822         if (!desc.access.io_mem ||
1823             fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up"))
1824                 drvdata->skip_power_up = true;
1825
1826         etm4_init_trace_id(drvdata);
1827         etm4_set_default(&drvdata->config);
1828
1829         pdata = coresight_get_platform_data(dev);
1830         if (IS_ERR(pdata))
1831                 return PTR_ERR(pdata);
1832
1833         dev->platform_data = pdata;
1834
1835         desc.type = CORESIGHT_DEV_TYPE_SOURCE;
1836         desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
1837         desc.ops = &etm4_cs_ops;
1838         desc.pdata = pdata;
1839         desc.dev = dev;
1840         desc.groups = coresight_etmv4_groups;
1841         drvdata->csdev = coresight_register(&desc);
1842         if (IS_ERR(drvdata->csdev))
1843                 return PTR_ERR(drvdata->csdev);
1844
1845         ret = etm_perf_symlink(drvdata->csdev, true);
1846         if (ret) {
1847                 coresight_unregister(drvdata->csdev);
1848                 return ret;
1849         }
1850
1851         etmdrvdata[drvdata->cpu] = drvdata;
1852
1853         dev_info(&drvdata->csdev->dev, "CPU%d: ETM v%d.%d initialized\n",
1854                  drvdata->cpu, ETM_ARCH_MAJOR_VERSION(drvdata->arch),
1855                  ETM_ARCH_MINOR_VERSION(drvdata->arch));
1856
1857         if (boot_enable) {
1858                 coresight_enable(drvdata->csdev);
1859                 drvdata->boot_enable = true;
1860         }
1861
1862         return 0;
1863 }
1864
1865 static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id)
1866 {
1867         void __iomem *base;
1868         struct device *dev = &adev->dev;
1869         struct resource *res = &adev->res;
1870         int ret;
1871
1872         /* Validity for the resource is already checked by the AMBA core */
1873         base = devm_ioremap_resource(dev, res);
1874         if (IS_ERR(base))
1875                 return PTR_ERR(base);
1876
1877         ret = etm4_probe(dev, base, id->id);
1878         if (!ret)
1879                 pm_runtime_put(&adev->dev);
1880
1881         return ret;
1882 }
1883
1884 static int etm4_probe_platform_dev(struct platform_device *pdev)
1885 {
1886         int ret;
1887
1888         pm_runtime_get_noresume(&pdev->dev);
1889         pm_runtime_set_active(&pdev->dev);
1890         pm_runtime_enable(&pdev->dev);
1891
1892         /*
1893          * System register based devices could match the
1894          * HW by reading appropriate registers on the HW
1895          * and thus we could skip the PID.
1896          */
1897         ret = etm4_probe(&pdev->dev, NULL, 0);
1898
1899         pm_runtime_put(&pdev->dev);
1900         return ret;
1901 }
1902
1903 static struct amba_cs_uci_id uci_id_etm4[] = {
1904         {
1905                 /*  ETMv4 UCI data */
1906                 .devarch        = ETM_DEVARCH_ETMv4x_ARCH,
1907                 .devarch_mask   = ETM_DEVARCH_ID_MASK,
1908                 .devtype        = 0x00000013,
1909         }
1910 };
1911
1912 static void clear_etmdrvdata(void *info)
1913 {
1914         int cpu = *(int *)info;
1915
1916         etmdrvdata[cpu] = NULL;
1917 }
1918
1919 static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata)
1920 {
1921         etm_perf_symlink(drvdata->csdev, false);
1922         /*
1923          * Taking hotplug lock here to avoid racing between etm4_remove_dev()
1924          * and CPU hotplug call backs.
1925          */
1926         cpus_read_lock();
1927         /*
1928          * The readers for etmdrvdata[] are CPU hotplug call backs
1929          * and PM notification call backs. Change etmdrvdata[i] on
1930          * CPU i ensures these call backs has consistent view
1931          * inside one call back function.
1932          */
1933         if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1))
1934                 etmdrvdata[drvdata->cpu] = NULL;
1935
1936         cpus_read_unlock();
1937
1938         coresight_unregister(drvdata->csdev);
1939
1940         return 0;
1941 }
1942
1943 static void __exit etm4_remove_amba(struct amba_device *adev)
1944 {
1945         struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev);
1946
1947         if (drvdata)
1948                 etm4_remove_dev(drvdata);
1949 }
1950
1951 static int __exit etm4_remove_platform_dev(struct platform_device *pdev)
1952 {
1953         int ret = 0;
1954         struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev);
1955
1956         if (drvdata)
1957                 ret = etm4_remove_dev(drvdata);
1958         pm_runtime_disable(&pdev->dev);
1959         return ret;
1960 }
1961
1962 static const struct amba_id etm4_ids[] = {
1963         CS_AMBA_ID(0x000bb95d),                 /* Cortex-A53 */
1964         CS_AMBA_ID(0x000bb95e),                 /* Cortex-A57 */
1965         CS_AMBA_ID(0x000bb95a),                 /* Cortex-A72 */
1966         CS_AMBA_ID(0x000bb959),                 /* Cortex-A73 */
1967         CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */
1968         CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */
1969         CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */
1970         CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */
1971         CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */
1972         CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */
1973         CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */
1974         CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */
1975         CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */
1976         CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */
1977         CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */
1978         CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */
1979         CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */
1980         {},
1981 };
1982
1983 MODULE_DEVICE_TABLE(amba, etm4_ids);
1984
1985 static struct amba_driver etm4x_amba_driver = {
1986         .drv = {
1987                 .name   = "coresight-etm4x",
1988                 .owner  = THIS_MODULE,
1989                 .suppress_bind_attrs = true,
1990         },
1991         .probe          = etm4_probe_amba,
1992         .remove         = etm4_remove_amba,
1993         .id_table       = etm4_ids,
1994 };
1995
1996 static const struct of_device_id etm4_sysreg_match[] = {
1997         { .compatible   = "arm,coresight-etm4x-sysreg" },
1998         {}
1999 };
2000
2001 static struct platform_driver etm4_platform_driver = {
2002         .probe          = etm4_probe_platform_dev,
2003         .remove         = etm4_remove_platform_dev,
2004         .driver                 = {
2005                 .name                   = "coresight-etm4x",
2006                 .of_match_table         = etm4_sysreg_match,
2007                 .suppress_bind_attrs    = true,
2008         },
2009 };
2010
2011 static int __init etm4x_init(void)
2012 {
2013         int ret;
2014
2015         ret = etm4_pm_setup();
2016
2017         /* etm4_pm_setup() does its own cleanup - exit on error */
2018         if (ret)
2019                 return ret;
2020
2021         ret = amba_driver_register(&etm4x_amba_driver);
2022         if (ret) {
2023                 pr_err("Error registering etm4x AMBA driver\n");
2024                 goto clear_pm;
2025         }
2026
2027         ret = platform_driver_register(&etm4_platform_driver);
2028         if (!ret)
2029                 return 0;
2030
2031         pr_err("Error registering etm4x platform driver\n");
2032         amba_driver_unregister(&etm4x_amba_driver);
2033
2034 clear_pm:
2035         etm4_pm_clear();
2036         return ret;
2037 }
2038
2039 static void __exit etm4x_exit(void)
2040 {
2041         amba_driver_unregister(&etm4x_amba_driver);
2042         platform_driver_unregister(&etm4_platform_driver);
2043         etm4_pm_clear();
2044 }
2045
2046 module_init(etm4x_init);
2047 module_exit(etm4x_exit);
2048
2049 MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>");
2050 MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>");
2051 MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver");
2052 MODULE_LICENSE("GPL v2");