drivers/hwtracing/coresight/coresight-etm-perf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015 Linaro Limited. All rights reserved.
   4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   5  */
   6
   7 #include <linux/coresight.h>
   8 #include <linux/coresight-pmu.h>
   9 #include <linux/cpumask.h>
  10 #include <linux/device.h>
  11 #include <linux/list.h>
  12 #include <linux/mm.h>
  13 #include <linux/init.h>
  14 #include <linux/perf_event.h>
  15 #include <linux/slab.h>
  16 #include <linux/types.h>
  17 #include <linux/workqueue.h>
  18
  19 #include "coresight-etm-perf.h"
  20 #include "coresight-priv.h"
  21
  22 static struct pmu etm_pmu;
  23 static bool etm_perf_up;
  24
  25 /**
  26  * struct etm_event_data - Coresight specifics associated to an event
  27  * @work:               Handle to free allocated memory outside IRQ context.
  28  * @mask:               Hold the CPU(s) this event was set for.
  29  * @snk_config:         The sink configuration.
  30  * @path:               An array of path, each slot for one CPU.
  31  */
  32 struct etm_event_data {
  33         struct work_struct work;
  34         cpumask_t mask;
  35         void *snk_config;
  36         struct list_head **path;
  37 };
  38
  39 static DEFINE_PER_CPU(struct perf_output_handle, ctx_handle);
  40 static DEFINE_PER_CPU(struct coresight_device *, csdev_src);
  41
  42 /* ETMv3.5/PTM's ETMCR is 'config' */
  43 PMU_FORMAT_ATTR(cycacc,         "config:" __stringify(ETM_OPT_CYCACC));
  44 PMU_FORMAT_ATTR(timestamp,      "config:" __stringify(ETM_OPT_TS));
  45 PMU_FORMAT_ATTR(retstack,       "config:" __stringify(ETM_OPT_RETSTK));
  46
  47 static struct attribute *etm_config_formats_attr[] = {
  48         &format_attr_cycacc.attr,
  49         &format_attr_timestamp.attr,
  50         &format_attr_retstack.attr,
  51         NULL,
  52 };
  53
  54 static const struct attribute_group etm_pmu_format_group = {
  55         .name   = "format",
  56         .attrs  = etm_config_formats_attr,
  57 };
  58
  59 static const struct attribute_group *etm_pmu_attr_groups[] = {
  60         &etm_pmu_format_group,
  61         NULL,
  62 };
  63
  64 static void etm_event_read(struct perf_event *event) {}
  65
  66 static int etm_addr_filters_alloc(struct perf_event *event)
  67 {
  68         struct etm_filters *filters;
  69         int node = event->cpu == -1 ? -1 : cpu_to_node(event->cpu);
  70
  71         filters = kzalloc_node(sizeof(struct etm_filters), GFP_KERNEL, node);
  72         if (!filters)
  73                 return -ENOMEM;
  74
  75         if (event->parent)
  76                 memcpy(filters, event->parent->hw.addr_filters,
  77                        sizeof(*filters));
  78
  79         event->hw.addr_filters = filters;
  80
  81         return 0;
  82 }
  83
  84 static void etm_event_destroy(struct perf_event *event)
  85 {
  86         kfree(event->hw.addr_filters);
  87         event->hw.addr_filters = NULL;
  88 }
  89
  90 static int etm_event_init(struct perf_event *event)
  91 {
  92         int ret = 0;
  93
  94         if (event->attr.type != etm_pmu.type) {
  95                 ret = -ENOENT;
  96                 goto out;
  97         }
  98
  99         ret = etm_addr_filters_alloc(event);
 100         if (ret)
 101                 goto out;
 102
 103         event->destroy = etm_event_destroy;
 104 out:
 105         return ret;
 106 }
 107
 108 static void free_event_data(struct work_struct *work)
 109 {
 110         int cpu;
 111         cpumask_t *mask;
 112         struct etm_event_data *event_data;
 113         struct coresight_device *sink;
 114
 115         event_data = container_of(work, struct etm_event_data, work);
 116         mask = &event_data->mask;
 117         /*
 118          * First deal with the sink configuration.  See comment in
 119          * etm_setup_aux() about why we take the first available path.
 120          */
 121         if (event_data->snk_config) {
 122                 cpu = cpumask_first(mask);
 123                 sink = coresight_get_sink(event_data->path[cpu]);
 124                 if (sink_ops(sink)->free_buffer)
 125                         sink_ops(sink)->free_buffer(event_data->snk_config);
 126         }
 127
 128         for_each_cpu(cpu, mask) {
 129                 if (!(IS_ERR_OR_NULL(event_data->path[cpu])))
 130                         coresight_release_path(event_data->path[cpu]);
 131         }
 132
 133         kfree(event_data->path);
 134         kfree(event_data);
 135 }
 136
 137 static void *alloc_event_data(int cpu)
 138 {
 139         int size;
 140         cpumask_t *mask;
 141         struct etm_event_data *event_data;
 142
 143         /* First get memory for the session's data */
 144         event_data = kzalloc(sizeof(struct etm_event_data), GFP_KERNEL);
 145         if (!event_data)
 146                 return NULL;
 147
 148         /* Make sure nothing disappears under us */
 149         get_online_cpus();
 150         size = num_online_cpus();
 151
 152         mask = &event_data->mask;
 153         if (cpu != -1)
 154                 cpumask_set_cpu(cpu, mask);
 155         else
 156                 cpumask_copy(mask, cpu_online_mask);
 157         put_online_cpus();
 158
 159         /*
 160          * Each CPU has a single path between source and destination.  As such
 161          * allocate an array using CPU numbers as indexes.  That way a path
 162          * for any CPU can easily be accessed at any given time.  We proceed
 163          * the same way for sessions involving a single CPU.  The cost of
 164          * unused memory when dealing with single CPU trace scenarios is small
 165          * compared to the cost of searching through an optimized array.
 166          */
 167         event_data->path = kcalloc(size,
 168                                    sizeof(struct list_head *), GFP_KERNEL);
 169         if (!event_data->path) {
 170                 kfree(event_data);
 171                 return NULL;
 172         }
 173
 174         return event_data;
 175 }
 176
 177 static void etm_free_aux(void *data)
 178 {
 179         struct etm_event_data *event_data = data;
 180
 181         schedule_work(&event_data->work);
 182 }
 183
 184 static void *etm_setup_aux(int event_cpu, void **pages,
 185                            int nr_pages, bool overwrite)
 186 {
 187         int cpu;
 188         cpumask_t *mask;
 189         struct coresight_device *sink;
 190         struct etm_event_data *event_data = NULL;
 191
 192         event_data = alloc_event_data(event_cpu);
 193         if (!event_data)
 194                 return NULL;
 195         INIT_WORK(&event_data->work, free_event_data);
 196
 197         /*
 198          * In theory nothing prevent tracers in a trace session from being
 199          * associated with different sinks, nor having a sink per tracer.  But
 200          * until we have HW with this kind of topology we need to assume tracers
 201          * in a trace session are using the same sink.  Therefore go through
 202          * the coresight bus and pick the first enabled sink.
 203          *
 204          * When operated from sysFS users are responsible to enable the sink
 205          * while from perf, the perf tools will do it based on the choice made
 206          * on the cmd line.  As such the "enable_sink" flag in sysFS is reset.
 207          */
 208         sink = coresight_get_enabled_sink(true);
 209         if (!sink)
 210                 goto err;
 211
 212         mask = &event_data->mask;
 213
 214         /* Setup the path for each CPU in a trace session */
 215         for_each_cpu(cpu, mask) {
 216                 struct coresight_device *csdev;
 217
 218                 csdev = per_cpu(csdev_src, cpu);
 219                 if (!csdev)
 220                         goto err;
 221
 222                 /*
 223                  * Building a path doesn't enable it, it simply builds a
 224                  * list of devices from source to sink that can be
 225                  * referenced later when the path is actually needed.
 226                  */
 227                 event_data->path[cpu] = coresight_build_path(csdev, sink);
 228                 if (IS_ERR(event_data->path[cpu]))
 229                         goto err;
 230         }
 231
 232         if (!sink_ops(sink)->alloc_buffer)
 233                 goto err;
 234
 235         cpu = cpumask_first(mask);
 236         /* Get the AUX specific data from the sink buffer */
 237         event_data->snk_config =
 238                         sink_ops(sink)->alloc_buffer(sink, cpu, pages,
 239                                                      nr_pages, overwrite);
 240         if (!event_data->snk_config)
 241                 goto err;
 242
 243 out:
 244         return event_data;
 245
 246 err:
 247         etm_free_aux(event_data);
 248         event_data = NULL;
 249         goto out;
 250 }
 251
 252 static void etm_event_start(struct perf_event *event, int flags)
 253 {
 254         int cpu = smp_processor_id();
 255         struct etm_event_data *event_data;
 256         struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle);
 257         struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu);
 258
 259         if (!csdev)
 260                 goto fail;
 261
 262         /*
 263          * Deal with the ring buffer API and get a handle on the
 264          * session's information.
 265          */
 266         event_data = perf_aux_output_begin(handle, event);
 267         if (!event_data)
 268                 goto fail;
 269
 270         /* We need a sink, no need to continue without one */
 271         sink = coresight_get_sink(event_data->path[cpu]);
 272         if (WARN_ON_ONCE(!sink || !sink_ops(sink)->set_buffer))
 273                 goto fail_end_stop;
 274
 275         /* Configure the sink */
 276         if (sink_ops(sink)->set_buffer(sink, handle,
 277                                        event_data->snk_config))
 278                 goto fail_end_stop;
 279
 280         /* Nothing will happen without a path */
 281         if (coresight_enable_path(event_data->path[cpu], CS_MODE_PERF))
 282                 goto fail_end_stop;
 283
 284         /* Tell the perf core the event is alive */
 285         event->hw.state = 0;
 286
 287         /* Finally enable the tracer */
 288         if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF))
 289                 goto fail_end_stop;
 290
 291 out:
 292         return;
 293
 294 fail_end_stop:
 295         perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 296         perf_aux_output_end(handle, 0);
 297 fail:
 298         event->hw.state = PERF_HES_STOPPED;
 299         goto out;
 300 }
 301
 302 static void etm_event_stop(struct perf_event *event, int mode)
 303 {
 304         int cpu = smp_processor_id();
 305         unsigned long size;
 306         struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu);
 307         struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle);
 308         struct etm_event_data *event_data = perf_get_aux(handle);
 309
 310         if (event->hw.state == PERF_HES_STOPPED)
 311                 return;
 312
 313         if (!csdev)
 314                 return;
 315
 316         sink = coresight_get_sink(event_data->path[cpu]);
 317         if (!sink)
 318                 return;
 319
 320         /* stop tracer */
 321         source_ops(csdev)->disable(csdev, event);
 322
 323         /* tell the core */
 324         event->hw.state = PERF_HES_STOPPED;
 325
 326         if (mode & PERF_EF_UPDATE) {
 327                 if (WARN_ON_ONCE(handle->event != event))
 328                         return;
 329
 330                 /* update trace information */
 331                 if (!sink_ops(sink)->update_buffer)
 332                         return;
 333
 334                 sink_ops(sink)->update_buffer(sink, handle,
 335                                               event_data->snk_config);
 336
 337                 if (!sink_ops(sink)->reset_buffer)
 338                         return;
 339
 340                 size = sink_ops(sink)->reset_buffer(sink, handle,
 341                                                     event_data->snk_config);
 342
 343                 perf_aux_output_end(handle, size);
 344         }
 345
 346         /* Disabling the path make its elements available to other sessions */
 347         coresight_disable_path(event_data->path[cpu]);
 348 }
 349
 350 static int etm_event_add(struct perf_event *event, int mode)
 351 {
 352         int ret = 0;
 353         struct hw_perf_event *hwc = &event->hw;
 354
 355         if (mode & PERF_EF_START) {
 356                 etm_event_start(event, 0);
 357                 if (hwc->state & PERF_HES_STOPPED)
 358                         ret = -EINVAL;
 359         } else {
 360                 hwc->state = PERF_HES_STOPPED;
 361         }
 362
 363         return ret;
 364 }
 365
 366 static void etm_event_del(struct perf_event *event, int mode)
 367 {
 368         etm_event_stop(event, PERF_EF_UPDATE);
 369 }
 370
 371 static int etm_addr_filters_validate(struct list_head *filters)
 372 {
 373         bool range = false, address = false;
 374         int index = 0;
 375         struct perf_addr_filter *filter;
 376
 377         list_for_each_entry(filter, filters, entry) {
 378                 /*
 379                  * No need to go further if there's no more
 380                  * room for filters.
 381                  */
 382                 if (++index > ETM_ADDR_CMP_MAX)
 383                         return -EOPNOTSUPP;
 384
 385                 /* filter::size==0 means single address trigger */
 386                 if (filter->size) {
 387                         /*
 388                          * The existing code relies on START/STOP filters
 389                          * being address filters.
 390                          */
 391                         if (filter->action == PERF_ADDR_FILTER_ACTION_START ||
 392                             filter->action == PERF_ADDR_FILTER_ACTION_STOP)
 393                                 return -EOPNOTSUPP;
 394
 395                         range = true;
 396                 } else
 397                         address = true;
 398
 399                 /*
 400                  * At this time we don't allow range and start/stop filtering
 401                  * to cohabitate, they have to be mutually exclusive.
 402                  */
 403                 if (range && address)
 404                         return -EOPNOTSUPP;
 405         }
 406
 407         return 0;
 408 }
 409
 410 static void etm_addr_filters_sync(struct perf_event *event)
 411 {
 412         struct perf_addr_filters_head *head = perf_event_addr_filters(event);
 413         unsigned long start, stop, *offs = event->addr_filters_offs;
 414         struct etm_filters *filters = event->hw.addr_filters;
 415         struct etm_filter *etm_filter;
 416         struct perf_addr_filter *filter;
 417         int i = 0;
 418
 419         list_for_each_entry(filter, &head->list, entry) {
 420                 start = filter->offset + offs[i];
 421                 stop = start + filter->size;
 422                 etm_filter = &filters->etm_filter[i];
 423
 424                 switch (filter->action) {
 425                 case PERF_ADDR_FILTER_ACTION_FILTER:
 426                         etm_filter->start_addr = start;
 427                         etm_filter->stop_addr = stop;
 428                         etm_filter->type = ETM_ADDR_TYPE_RANGE;
 429                         break;
 430                 case PERF_ADDR_FILTER_ACTION_START:
 431                         etm_filter->start_addr = start;
 432                         etm_filter->type = ETM_ADDR_TYPE_START;
 433                         break;
 434                 case PERF_ADDR_FILTER_ACTION_STOP:
 435                         etm_filter->stop_addr = stop;
 436                         etm_filter->type = ETM_ADDR_TYPE_STOP;
 437                         break;
 438                 }
 439                 i++;
 440         }
 441
 442         filters->nr_filters = i;
 443 }
 444
 445 int etm_perf_symlink(struct coresight_device *csdev, bool link)
 446 {
 447         char entry[sizeof("cpu9999999")];
 448         int ret = 0, cpu = source_ops(csdev)->cpu_id(csdev);
 449         struct device *pmu_dev = etm_pmu.dev;
 450         struct device *cs_dev = &csdev->dev;
 451
 452         sprintf(entry, "cpu%d", cpu);
 453
 454         if (!etm_perf_up)
 455                 return -EPROBE_DEFER;
 456
 457         if (link) {
 458                 ret = sysfs_create_link(&pmu_dev->kobj, &cs_dev->kobj, entry);
 459                 if (ret)
 460                         return ret;
 461                 per_cpu(csdev_src, cpu) = csdev;
 462         } else {
 463                 sysfs_remove_link(&pmu_dev->kobj, entry);
 464                 per_cpu(csdev_src, cpu) = NULL;
 465         }
 466
 467         return 0;
 468 }
 469
 470 static int __init etm_perf_init(void)
 471 {
 472         int ret;
 473
 474         etm_pmu.capabilities            = PERF_PMU_CAP_EXCLUSIVE;
 475
 476         etm_pmu.attr_groups             = etm_pmu_attr_groups;
 477         etm_pmu.task_ctx_nr             = perf_sw_context;
 478         etm_pmu.read                    = etm_event_read;
 479         etm_pmu.event_init              = etm_event_init;
 480         etm_pmu.setup_aux               = etm_setup_aux;
 481         etm_pmu.free_aux                = etm_free_aux;
 482         etm_pmu.start                   = etm_event_start;
 483         etm_pmu.stop                    = etm_event_stop;
 484         etm_pmu.add                     = etm_event_add;
 485         etm_pmu.del                     = etm_event_del;
 486         etm_pmu.addr_filters_sync       = etm_addr_filters_sync;
 487         etm_pmu.addr_filters_validate   = etm_addr_filters_validate;
 488         etm_pmu.nr_addr_filters         = ETM_ADDR_CMP_MAX;
 489
 490         ret = perf_pmu_register(&etm_pmu, CORESIGHT_ETM_PMU_NAME, -1);
 491         if (ret == 0)
 492                 etm_perf_up = true;
 493
 494         return ret;
 495 }
 496 device_initcall(etm_perf_init);