kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 /* for gfp flag names */
  17 #include <linux/trace_events.h>
  18 #include <trace/events/mmflags.h>
  19
  20 #include "tracing_map.h"
  21 #include "trace.h"
  22 #include "trace_dynevent.h"
  23
  24 #define SYNTH_SYSTEM            "synthetic"
  25 #define SYNTH_FIELDS_MAX        16
  26
  27 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  28
  29 #define ERRORS                                                          \
  30         C(NONE,                 "No error"),                            \
  31         C(DUPLICATE_VAR,        "Variable already defined"),            \
  32         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  33         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  34         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  35         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  36         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  37         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  38         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  39         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  40         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  41         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  42         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  43         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  44         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  45         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  46         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  47         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  48         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  49         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  50         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  51         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  52         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  53         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  54         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  55         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  56         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  57         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  58         C(INVALID_PARAM,        "Invalid action param"),                \
  59         C(ACTION_NOT_FOUND,     "No action found"),                     \
  60         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  61         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  62         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  63         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  64         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  65         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  66         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  67         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  68         C(FIELD_NOT_FOUND,      "Couldn't find field"),
  69
  70 #undef C
  71 #define C(a, b)         HIST_ERR_##a
  72
  73 enum { ERRORS };
  74
  75 #undef C
  76 #define C(a, b)         b
  77
  78 static const char *err_text[] = { ERRORS };
  79
  80 struct hist_field;
  81
  82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  83                                 struct tracing_map_elt *elt,
  84                                 struct ring_buffer_event *rbe,
  85                                 void *event);
  86
  87 #define HIST_FIELD_OPERANDS_MAX 2
  88 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  89 #define HIST_ACTIONS_MAX        8
  90
  91 enum field_op_id {
  92         FIELD_OP_NONE,
  93         FIELD_OP_PLUS,
  94         FIELD_OP_MINUS,
  95         FIELD_OP_UNARY_MINUS,
  96 };
  97
  98 /*
  99  * A hist_var (histogram variable) contains variable information for
 100  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 101  * flag set.  A hist_var has a variable name e.g. ts0, and is
 102  * associated with a given histogram trigger, as specified by
 103  * hist_data.  The hist_var idx is the unique index assigned to the
 104  * variable by the hist trigger's tracing_map.  The idx is what is
 105  * used to set a variable's value and, by a variable reference, to
 106  * retrieve it.
 107  */
 108 struct hist_var {
 109         char                            *name;
 110         struct hist_trigger_data        *hist_data;
 111         unsigned int                    idx;
 112 };
 113
 114 struct hist_field {
 115         struct ftrace_event_field       *field;
 116         unsigned long                   flags;
 117         hist_field_fn_t                 fn;
 118         unsigned int                    size;
 119         unsigned int                    offset;
 120         unsigned int                    is_signed;
 121         const char                      *type;
 122         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 123         struct hist_trigger_data        *hist_data;
 124
 125         /*
 126          * Variable fields contain variable-specific info in var.
 127          */
 128         struct hist_var                 var;
 129         enum field_op_id                operator;
 130         char                            *system;
 131         char                            *event_name;
 132
 133         /*
 134          * The name field is used for EXPR and VAR_REF fields.  VAR
 135          * fields contain the variable name in var.name.
 136          */
 137         char                            *name;
 138
 139         /*
 140          * When a histogram trigger is hit, if it has any references
 141          * to variables, the values of those variables are collected
 142          * into a var_ref_vals array by resolve_var_refs().  The
 143          * current value of each variable is read from the tracing_map
 144          * using the hist field's hist_var.idx and entered into the
 145          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 146          */
 147         unsigned int                    var_ref_idx;
 148         bool                            read_once;
 149 };
 150
 151 static u64 hist_field_none(struct hist_field *field,
 152                            struct tracing_map_elt *elt,
 153                            struct ring_buffer_event *rbe,
 154                            void *event)
 155 {
 156         return 0;
 157 }
 158
 159 static u64 hist_field_counter(struct hist_field *field,
 160                               struct tracing_map_elt *elt,
 161                               struct ring_buffer_event *rbe,
 162                               void *event)
 163 {
 164         return 1;
 165 }
 166
 167 static u64 hist_field_string(struct hist_field *hist_field,
 168                              struct tracing_map_elt *elt,
 169                              struct ring_buffer_event *rbe,
 170                              void *event)
 171 {
 172         char *addr = (char *)(event + hist_field->field->offset);
 173
 174         return (u64)(unsigned long)addr;
 175 }
 176
 177 static u64 hist_field_dynstring(struct hist_field *hist_field,
 178                                 struct tracing_map_elt *elt,
 179                                 struct ring_buffer_event *rbe,
 180                                 void *event)
 181 {
 182         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 183         int str_loc = str_item & 0xffff;
 184         char *addr = (char *)(event + str_loc);
 185
 186         return (u64)(unsigned long)addr;
 187 }
 188
 189 static u64 hist_field_pstring(struct hist_field *hist_field,
 190                               struct tracing_map_elt *elt,
 191                               struct ring_buffer_event *rbe,
 192                               void *event)
 193 {
 194         char **addr = (char **)(event + hist_field->field->offset);
 195
 196         return (u64)(unsigned long)*addr;
 197 }
 198
 199 static u64 hist_field_log2(struct hist_field *hist_field,
 200                            struct tracing_map_elt *elt,
 201                            struct ring_buffer_event *rbe,
 202                            void *event)
 203 {
 204         struct hist_field *operand = hist_field->operands[0];
 205
 206         u64 val = operand->fn(operand, elt, rbe, event);
 207
 208         return (u64) ilog2(roundup_pow_of_two(val));
 209 }
 210
 211 static u64 hist_field_plus(struct hist_field *hist_field,
 212                            struct tracing_map_elt *elt,
 213                            struct ring_buffer_event *rbe,
 214                            void *event)
 215 {
 216         struct hist_field *operand1 = hist_field->operands[0];
 217         struct hist_field *operand2 = hist_field->operands[1];
 218
 219         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 220         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 221
 222         return val1 + val2;
 223 }
 224
 225 static u64 hist_field_minus(struct hist_field *hist_field,
 226                             struct tracing_map_elt *elt,
 227                             struct ring_buffer_event *rbe,
 228                             void *event)
 229 {
 230         struct hist_field *operand1 = hist_field->operands[0];
 231         struct hist_field *operand2 = hist_field->operands[1];
 232
 233         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 234         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 235
 236         return val1 - val2;
 237 }
 238
 239 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 240                                   struct tracing_map_elt *elt,
 241                                   struct ring_buffer_event *rbe,
 242                                   void *event)
 243 {
 244         struct hist_field *operand = hist_field->operands[0];
 245
 246         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 247         u64 val = (u64)-sval;
 248
 249         return val;
 250 }
 251
 252 #define DEFINE_HIST_FIELD_FN(type)                                      \
 253         static u64 hist_field_##type(struct hist_field *hist_field,     \
 254                                      struct tracing_map_elt *elt,       \
 255                                      struct ring_buffer_event *rbe,     \
 256                                      void *event)                       \
 257 {                                                                       \
 258         type *addr = (type *)(event + hist_field->field->offset);       \
 259                                                                         \
 260         return (u64)(unsigned long)*addr;                               \
 261 }
 262
 263 DEFINE_HIST_FIELD_FN(s64);
 264 DEFINE_HIST_FIELD_FN(u64);
 265 DEFINE_HIST_FIELD_FN(s32);
 266 DEFINE_HIST_FIELD_FN(u32);
 267 DEFINE_HIST_FIELD_FN(s16);
 268 DEFINE_HIST_FIELD_FN(u16);
 269 DEFINE_HIST_FIELD_FN(s8);
 270 DEFINE_HIST_FIELD_FN(u8);
 271
 272 #define for_each_hist_field(i, hist_data)       \
 273         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 274
 275 #define for_each_hist_val_field(i, hist_data)   \
 276         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 277
 278 #define for_each_hist_key_field(i, hist_data)   \
 279         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 280
 281 #define HIST_STACKTRACE_DEPTH   16
 282 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 283 #define HIST_STACKTRACE_SKIP    5
 284
 285 #define HITCOUNT_IDX            0
 286 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 287
 288 enum hist_field_flags {
 289         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 290         HIST_FIELD_FL_KEY               = 1 << 1,
 291         HIST_FIELD_FL_STRING            = 1 << 2,
 292         HIST_FIELD_FL_HEX               = 1 << 3,
 293         HIST_FIELD_FL_SYM               = 1 << 4,
 294         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 295         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 296         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 297         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 298         HIST_FIELD_FL_LOG2              = 1 << 9,
 299         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 300         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 301         HIST_FIELD_FL_VAR               = 1 << 12,
 302         HIST_FIELD_FL_EXPR              = 1 << 13,
 303         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 304         HIST_FIELD_FL_CPU               = 1 << 15,
 305         HIST_FIELD_FL_ALIAS             = 1 << 16,
 306 };
 307
 308 struct var_defs {
 309         unsigned int    n_vars;
 310         char            *name[TRACING_MAP_VARS_MAX];
 311         char            *expr[TRACING_MAP_VARS_MAX];
 312 };
 313
 314 struct hist_trigger_attrs {
 315         char            *keys_str;
 316         char            *vals_str;
 317         char            *sort_key_str;
 318         char            *name;
 319         char            *clock;
 320         bool            pause;
 321         bool            cont;
 322         bool            clear;
 323         bool            ts_in_usecs;
 324         unsigned int    map_bits;
 325
 326         char            *assignment_str[TRACING_MAP_VARS_MAX];
 327         unsigned int    n_assignments;
 328
 329         char            *action_str[HIST_ACTIONS_MAX];
 330         unsigned int    n_actions;
 331
 332         struct var_defs var_defs;
 333 };
 334
 335 struct field_var {
 336         struct hist_field       *var;
 337         struct hist_field       *val;
 338 };
 339
 340 struct field_var_hist {
 341         struct hist_trigger_data        *hist_data;
 342         char                            *cmd;
 343 };
 344
 345 struct hist_trigger_data {
 346         struct hist_field               *fields[HIST_FIELDS_MAX];
 347         unsigned int                    n_vals;
 348         unsigned int                    n_keys;
 349         unsigned int                    n_fields;
 350         unsigned int                    n_vars;
 351         unsigned int                    key_size;
 352         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 353         unsigned int                    n_sort_keys;
 354         struct trace_event_file         *event_file;
 355         struct hist_trigger_attrs       *attrs;
 356         struct tracing_map              *map;
 357         bool                            enable_timestamps;
 358         bool                            remove;
 359         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 360         unsigned int                    n_var_refs;
 361
 362         struct action_data              *actions[HIST_ACTIONS_MAX];
 363         unsigned int                    n_actions;
 364
 365         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 366         unsigned int                    n_field_vars;
 367         unsigned int                    n_field_var_str;
 368         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 369         unsigned int                    n_field_var_hists;
 370
 371         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 372         unsigned int                    n_save_vars;
 373         unsigned int                    n_save_var_str;
 374 };
 375
 376 static int synth_event_create(int argc, const char **argv);
 377 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 378 static int synth_event_release(struct dyn_event *ev);
 379 static bool synth_event_is_busy(struct dyn_event *ev);
 380 static bool synth_event_match(const char *system, const char *event,
 381                         int argc, const char **argv, struct dyn_event *ev);
 382
 383 static struct dyn_event_operations synth_event_ops = {
 384         .create = synth_event_create,
 385         .show = synth_event_show,
 386         .is_busy = synth_event_is_busy,
 387         .free = synth_event_release,
 388         .match = synth_event_match,
 389 };
 390
 391 struct synth_field {
 392         char *type;
 393         char *name;
 394         size_t size;
 395         bool is_signed;
 396         bool is_string;
 397 };
 398
 399 struct synth_event {
 400         struct dyn_event                        devent;
 401         int                                     ref;
 402         char                                    *name;
 403         struct synth_field                      **fields;
 404         unsigned int                            n_fields;
 405         unsigned int                            n_u64;
 406         struct trace_event_class                class;
 407         struct trace_event_call                 call;
 408         struct tracepoint                       *tp;
 409 };
 410
 411 static bool is_synth_event(struct dyn_event *ev)
 412 {
 413         return ev->ops == &synth_event_ops;
 414 }
 415
 416 static struct synth_event *to_synth_event(struct dyn_event *ev)
 417 {
 418         return container_of(ev, struct synth_event, devent);
 419 }
 420
 421 static bool synth_event_is_busy(struct dyn_event *ev)
 422 {
 423         struct synth_event *event = to_synth_event(ev);
 424
 425         return event->ref != 0;
 426 }
 427
 428 static bool synth_event_match(const char *system, const char *event,
 429                         int argc, const char **argv, struct dyn_event *ev)
 430 {
 431         struct synth_event *sev = to_synth_event(ev);
 432
 433         return strcmp(sev->name, event) == 0 &&
 434                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 435 }
 436
 437 struct action_data;
 438
 439 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 440                              struct tracing_map_elt *elt, void *rec,
 441                              struct ring_buffer_event *rbe, void *key,
 442                              struct action_data *data, u64 *var_ref_vals);
 443
 444 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 445
 446 enum handler_id {
 447         HANDLER_ONMATCH = 1,
 448         HANDLER_ONMAX,
 449         HANDLER_ONCHANGE,
 450 };
 451
 452 enum action_id {
 453         ACTION_SAVE = 1,
 454         ACTION_TRACE,
 455         ACTION_SNAPSHOT,
 456 };
 457
 458 struct action_data {
 459         enum handler_id         handler;
 460         enum action_id          action;
 461         char                    *action_name;
 462         action_fn_t             fn;
 463
 464         unsigned int            n_params;
 465         char                    *params[SYNTH_FIELDS_MAX];
 466
 467         /*
 468          * When a histogram trigger is hit, the values of any
 469          * references to variables, including variables being passed
 470          * as parameters to synthetic events, are collected into a
 471          * var_ref_vals array.  This var_ref_idx is the index of the
 472          * first param in the array to be passed to the synthetic
 473          * event invocation.
 474          */
 475         unsigned int            var_ref_idx;
 476         struct synth_event      *synth_event;
 477         bool                    use_trace_keyword;
 478         char                    *synth_event_name;
 479
 480         union {
 481                 struct {
 482                         char                    *event;
 483                         char                    *event_system;
 484                 } match_data;
 485
 486                 struct {
 487                         /*
 488                          * var_str contains the $-unstripped variable
 489                          * name referenced by var_ref, and used when
 490                          * printing the action.  Because var_ref
 491                          * creation is deferred to create_actions(),
 492                          * we need a per-action way to save it until
 493                          * then, thus var_str.
 494                          */
 495                         char                    *var_str;
 496
 497                         /*
 498                          * var_ref refers to the variable being
 499                          * tracked e.g onmax($var).
 500                          */
 501                         struct hist_field       *var_ref;
 502
 503                         /*
 504                          * track_var contains the 'invisible' tracking
 505                          * variable created to keep the current
 506                          * e.g. max value.
 507                          */
 508                         struct hist_field       *track_var;
 509
 510                         check_track_val_fn_t    check_val;
 511                         action_fn_t             save_data;
 512                 } track_data;
 513         };
 514 };
 515
 516 struct track_data {
 517         u64                             track_val;
 518         bool                            updated;
 519
 520         unsigned int                    key_len;
 521         void                            *key;
 522         struct tracing_map_elt          elt;
 523
 524         struct action_data              *action_data;
 525         struct hist_trigger_data        *hist_data;
 526 };
 527
 528 struct hist_elt_data {
 529         char *comm;
 530         u64 *var_ref_vals;
 531         char *field_var_str[SYNTH_FIELDS_MAX];
 532 };
 533
 534 struct snapshot_context {
 535         struct tracing_map_elt  *elt;
 536         void                    *key;
 537 };
 538
 539 static void track_data_free(struct track_data *track_data)
 540 {
 541         struct hist_elt_data *elt_data;
 542
 543         if (!track_data)
 544                 return;
 545
 546         kfree(track_data->key);
 547
 548         elt_data = track_data->elt.private_data;
 549         if (elt_data) {
 550                 kfree(elt_data->comm);
 551                 kfree(elt_data);
 552         }
 553
 554         kfree(track_data);
 555 }
 556
 557 static struct track_data *track_data_alloc(unsigned int key_len,
 558                                            struct action_data *action_data,
 559                                            struct hist_trigger_data *hist_data)
 560 {
 561         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 562         struct hist_elt_data *elt_data;
 563
 564         if (!data)
 565                 return ERR_PTR(-ENOMEM);
 566
 567         data->key = kzalloc(key_len, GFP_KERNEL);
 568         if (!data->key) {
 569                 track_data_free(data);
 570                 return ERR_PTR(-ENOMEM);
 571         }
 572
 573         data->key_len = key_len;
 574         data->action_data = action_data;
 575         data->hist_data = hist_data;
 576
 577         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 578         if (!elt_data) {
 579                 track_data_free(data);
 580                 return ERR_PTR(-ENOMEM);
 581         }
 582         data->elt.private_data = elt_data;
 583
 584         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 585         if (!elt_data->comm) {
 586                 track_data_free(data);
 587                 return ERR_PTR(-ENOMEM);
 588         }
 589
 590         return data;
 591 }
 592
 593 static char last_cmd[MAX_FILTER_STR_VAL];
 594 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 595
 596 static int errpos(char *str)
 597 {
 598         return err_pos(last_cmd, str);
 599 }
 600
 601 static void last_cmd_set(struct trace_event_file *file, char *str)
 602 {
 603         const char *system = NULL, *name = NULL;
 604         struct trace_event_call *call;
 605
 606         if (!str)
 607                 return;
 608
 609         strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 610
 611         if (file) {
 612                 call = file->event_call;
 613
 614                 system = call->class->system;
 615                 if (system) {
 616                         name = trace_event_name(call);
 617                         if (!name)
 618                                 system = NULL;
 619                 }
 620         }
 621
 622         if (system)
 623                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 624 }
 625
 626 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 627 {
 628         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 629                         err_type, err_pos);
 630 }
 631
 632 static void hist_err_clear(void)
 633 {
 634         last_cmd[0] = '\0';
 635         last_cmd_loc[0] = '\0';
 636 }
 637
 638 struct synth_trace_event {
 639         struct trace_entry      ent;
 640         u64                     fields[];
 641 };
 642
 643 static int synth_event_define_fields(struct trace_event_call *call)
 644 {
 645         struct synth_trace_event trace;
 646         int offset = offsetof(typeof(trace), fields);
 647         struct synth_event *event = call->data;
 648         unsigned int i, size, n_u64;
 649         char *name, *type;
 650         bool is_signed;
 651         int ret = 0;
 652
 653         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 654                 size = event->fields[i]->size;
 655                 is_signed = event->fields[i]->is_signed;
 656                 type = event->fields[i]->type;
 657                 name = event->fields[i]->name;
 658                 ret = trace_define_field(call, type, name, offset, size,
 659                                          is_signed, FILTER_OTHER);
 660                 if (ret)
 661                         break;
 662
 663                 if (event->fields[i]->is_string) {
 664                         offset += STR_VAR_LEN_MAX;
 665                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 666                 } else {
 667                         offset += sizeof(u64);
 668                         n_u64++;
 669                 }
 670         }
 671
 672         event->n_u64 = n_u64;
 673
 674         return ret;
 675 }
 676
 677 static bool synth_field_signed(char *type)
 678 {
 679         if (str_has_prefix(type, "u"))
 680                 return false;
 681
 682         return true;
 683 }
 684
 685 static int synth_field_is_string(char *type)
 686 {
 687         if (strstr(type, "char[") != NULL)
 688                 return true;
 689
 690         return false;
 691 }
 692
 693 static int synth_field_string_size(char *type)
 694 {
 695         char buf[4], *end, *start;
 696         unsigned int len;
 697         int size, err;
 698
 699         start = strstr(type, "char[");
 700         if (start == NULL)
 701                 return -EINVAL;
 702         start += sizeof("char[") - 1;
 703
 704         end = strchr(type, ']');
 705         if (!end || end < start)
 706                 return -EINVAL;
 707
 708         len = end - start;
 709         if (len > 3)
 710                 return -EINVAL;
 711
 712         strncpy(buf, start, len);
 713         buf[len] = '\0';
 714
 715         err = kstrtouint(buf, 0, &size);
 716         if (err)
 717                 return err;
 718
 719         if (size > STR_VAR_LEN_MAX)
 720                 return -EINVAL;
 721
 722         return size;
 723 }
 724
 725 static int synth_field_size(char *type)
 726 {
 727         int size = 0;
 728
 729         if (strcmp(type, "s64") == 0)
 730                 size = sizeof(s64);
 731         else if (strcmp(type, "u64") == 0)
 732                 size = sizeof(u64);
 733         else if (strcmp(type, "s32") == 0)
 734                 size = sizeof(s32);
 735         else if (strcmp(type, "u32") == 0)
 736                 size = sizeof(u32);
 737         else if (strcmp(type, "s16") == 0)
 738                 size = sizeof(s16);
 739         else if (strcmp(type, "u16") == 0)
 740                 size = sizeof(u16);
 741         else if (strcmp(type, "s8") == 0)
 742                 size = sizeof(s8);
 743         else if (strcmp(type, "u8") == 0)
 744                 size = sizeof(u8);
 745         else if (strcmp(type, "char") == 0)
 746                 size = sizeof(char);
 747         else if (strcmp(type, "unsigned char") == 0)
 748                 size = sizeof(unsigned char);
 749         else if (strcmp(type, "int") == 0)
 750                 size = sizeof(int);
 751         else if (strcmp(type, "unsigned int") == 0)
 752                 size = sizeof(unsigned int);
 753         else if (strcmp(type, "long") == 0)
 754                 size = sizeof(long);
 755         else if (strcmp(type, "unsigned long") == 0)
 756                 size = sizeof(unsigned long);
 757         else if (strcmp(type, "pid_t") == 0)
 758                 size = sizeof(pid_t);
 759         else if (strcmp(type, "gfp_t") == 0)
 760                 size = sizeof(gfp_t);
 761         else if (synth_field_is_string(type))
 762                 size = synth_field_string_size(type);
 763
 764         return size;
 765 }
 766
 767 static const char *synth_field_fmt(char *type)
 768 {
 769         const char *fmt = "%llu";
 770
 771         if (strcmp(type, "s64") == 0)
 772                 fmt = "%lld";
 773         else if (strcmp(type, "u64") == 0)
 774                 fmt = "%llu";
 775         else if (strcmp(type, "s32") == 0)
 776                 fmt = "%d";
 777         else if (strcmp(type, "u32") == 0)
 778                 fmt = "%u";
 779         else if (strcmp(type, "s16") == 0)
 780                 fmt = "%d";
 781         else if (strcmp(type, "u16") == 0)
 782                 fmt = "%u";
 783         else if (strcmp(type, "s8") == 0)
 784                 fmt = "%d";
 785         else if (strcmp(type, "u8") == 0)
 786                 fmt = "%u";
 787         else if (strcmp(type, "char") == 0)
 788                 fmt = "%d";
 789         else if (strcmp(type, "unsigned char") == 0)
 790                 fmt = "%u";
 791         else if (strcmp(type, "int") == 0)
 792                 fmt = "%d";
 793         else if (strcmp(type, "unsigned int") == 0)
 794                 fmt = "%u";
 795         else if (strcmp(type, "long") == 0)
 796                 fmt = "%ld";
 797         else if (strcmp(type, "unsigned long") == 0)
 798                 fmt = "%lu";
 799         else if (strcmp(type, "pid_t") == 0)
 800                 fmt = "%d";
 801         else if (strcmp(type, "gfp_t") == 0)
 802                 fmt = "%x";
 803         else if (synth_field_is_string(type))
 804                 fmt = "%s";
 805
 806         return fmt;
 807 }
 808
 809 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 810                                            int flags,
 811                                            struct trace_event *event)
 812 {
 813         struct trace_array *tr = iter->tr;
 814         struct trace_seq *s = &iter->seq;
 815         struct synth_trace_event *entry;
 816         struct synth_event *se;
 817         unsigned int i, n_u64;
 818         char print_fmt[32];
 819         const char *fmt;
 820
 821         entry = (struct synth_trace_event *)iter->ent;
 822         se = container_of(event, struct synth_event, call.event);
 823
 824         trace_seq_printf(s, "%s: ", se->name);
 825
 826         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 827                 if (trace_seq_has_overflowed(s))
 828                         goto end;
 829
 830                 fmt = synth_field_fmt(se->fields[i]->type);
 831
 832                 /* parameter types */
 833                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 834                         trace_seq_printf(s, "%s ", fmt);
 835
 836                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 837
 838                 /* parameter values */
 839                 if (se->fields[i]->is_string) {
 840                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 841                                          (char *)&entry->fields[n_u64],
 842                                          i == se->n_fields - 1 ? "" : " ");
 843                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 844                 } else {
 845                         struct trace_print_flags __flags[] = {
 846                             __def_gfpflag_names, {-1, NULL} };
 847
 848                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 849                                          entry->fields[n_u64],
 850                                          i == se->n_fields - 1 ? "" : " ");
 851
 852                         if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 853                                 trace_seq_puts(s, " (");
 854                                 trace_print_flags_seq(s, "|",
 855                                                       entry->fields[n_u64],
 856                                                       __flags);
 857                                 trace_seq_putc(s, ')');
 858                         }
 859                         n_u64++;
 860                 }
 861         }
 862 end:
 863         trace_seq_putc(s, '\n');
 864
 865         return trace_handle_return(s);
 866 }
 867
 868 static struct trace_event_functions synth_event_funcs = {
 869         .trace          = print_synth_event
 870 };
 871
 872 static notrace void trace_event_raw_event_synth(void *__data,
 873                                                 u64 *var_ref_vals,
 874                                                 unsigned int var_ref_idx)
 875 {
 876         struct trace_event_file *trace_file = __data;
 877         struct synth_trace_event *entry;
 878         struct trace_event_buffer fbuffer;
 879         struct ring_buffer *buffer;
 880         struct synth_event *event;
 881         unsigned int i, n_u64;
 882         int fields_size = 0;
 883
 884         event = trace_file->event_call->data;
 885
 886         if (trace_trigger_soft_disabled(trace_file))
 887                 return;
 888
 889         fields_size = event->n_u64 * sizeof(u64);
 890
 891         /*
 892          * Avoid ring buffer recursion detection, as this event
 893          * is being performed within another event.
 894          */
 895         buffer = trace_file->tr->trace_buffer.buffer;
 896         ring_buffer_nest_start(buffer);
 897
 898         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 899                                            sizeof(*entry) + fields_size);
 900         if (!entry)
 901                 goto out;
 902
 903         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 904                 if (event->fields[i]->is_string) {
 905                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 906                         char *str_field = (char *)&entry->fields[n_u64];
 907
 908                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 909                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 910                 } else {
 911                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 912                         n_u64++;
 913                 }
 914         }
 915
 916         trace_event_buffer_commit(&fbuffer);
 917 out:
 918         ring_buffer_nest_end(buffer);
 919 }
 920
 921 static void free_synth_event_print_fmt(struct trace_event_call *call)
 922 {
 923         if (call) {
 924                 kfree(call->print_fmt);
 925                 call->print_fmt = NULL;
 926         }
 927 }
 928
 929 static int __set_synth_event_print_fmt(struct synth_event *event,
 930                                        char *buf, int len)
 931 {
 932         const char *fmt;
 933         int pos = 0;
 934         int i;
 935
 936         /* When len=0, we just calculate the needed length */
 937 #define LEN_OR_ZERO (len ? len - pos : 0)
 938
 939         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 940         for (i = 0; i < event->n_fields; i++) {
 941                 fmt = synth_field_fmt(event->fields[i]->type);
 942                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 943                                 event->fields[i]->name, fmt,
 944                                 i == event->n_fields - 1 ? "" : ", ");
 945         }
 946         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 947
 948         for (i = 0; i < event->n_fields; i++) {
 949                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 950                                 ", REC->%s", event->fields[i]->name);
 951         }
 952
 953 #undef LEN_OR_ZERO
 954
 955         /* return the length of print_fmt */
 956         return pos;
 957 }
 958
 959 static int set_synth_event_print_fmt(struct trace_event_call *call)
 960 {
 961         struct synth_event *event = call->data;
 962         char *print_fmt;
 963         int len;
 964
 965         /* First: called with 0 length to calculate the needed length */
 966         len = __set_synth_event_print_fmt(event, NULL, 0);
 967
 968         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 969         if (!print_fmt)
 970                 return -ENOMEM;
 971
 972         /* Second: actually write the @print_fmt */
 973         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 974         call->print_fmt = print_fmt;
 975
 976         return 0;
 977 }
 978
 979 static void free_synth_field(struct synth_field *field)
 980 {
 981         kfree(field->type);
 982         kfree(field->name);
 983         kfree(field);
 984 }
 985
 986 static struct synth_field *parse_synth_field(int argc, const char **argv,
 987                                              int *consumed)
 988 {
 989         struct synth_field *field;
 990         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 991         int len, ret = 0;
 992
 993         if (field_type[0] == ';')
 994                 field_type++;
 995
 996         if (!strcmp(field_type, "unsigned")) {
 997                 if (argc < 3)
 998                         return ERR_PTR(-EINVAL);
 999                 prefix = "unsigned ";
1000                 field_type = argv[1];
1001                 field_name = argv[2];
1002                 *consumed = 3;
1003         } else {
1004                 field_name = argv[1];
1005                 *consumed = 2;
1006         }
1007
1008         field = kzalloc(sizeof(*field), GFP_KERNEL);
1009         if (!field)
1010                 return ERR_PTR(-ENOMEM);
1011
1012         len = strlen(field_name);
1013         array = strchr(field_name, '[');
1014         if (array)
1015                 len -= strlen(array);
1016         else if (field_name[len - 1] == ';')
1017                 len--;
1018
1019         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1020         if (!field->name) {
1021                 ret = -ENOMEM;
1022                 goto free;
1023         }
1024
1025         if (field_type[0] == ';')
1026                 field_type++;
1027         len = strlen(field_type) + 1;
1028         if (array)
1029                 len += strlen(array);
1030         if (prefix)
1031                 len += strlen(prefix);
1032
1033         field->type = kzalloc(len, GFP_KERNEL);
1034         if (!field->type) {
1035                 ret = -ENOMEM;
1036                 goto free;
1037         }
1038         if (prefix)
1039                 strcat(field->type, prefix);
1040         strcat(field->type, field_type);
1041         if (array) {
1042                 strcat(field->type, array);
1043                 if (field->type[len - 1] == ';')
1044                         field->type[len - 1] = '\0';
1045         }
1046
1047         field->size = synth_field_size(field->type);
1048         if (!field->size) {
1049                 ret = -EINVAL;
1050                 goto free;
1051         }
1052
1053         if (synth_field_is_string(field->type))
1054                 field->is_string = true;
1055
1056         field->is_signed = synth_field_signed(field->type);
1057
1058  out:
1059         return field;
1060  free:
1061         free_synth_field(field);
1062         field = ERR_PTR(ret);
1063         goto out;
1064 }
1065
1066 static void free_synth_tracepoint(struct tracepoint *tp)
1067 {
1068         if (!tp)
1069                 return;
1070
1071         kfree(tp->name);
1072         kfree(tp);
1073 }
1074
1075 static struct tracepoint *alloc_synth_tracepoint(char *name)
1076 {
1077         struct tracepoint *tp;
1078
1079         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1080         if (!tp)
1081                 return ERR_PTR(-ENOMEM);
1082
1083         tp->name = kstrdup(name, GFP_KERNEL);
1084         if (!tp->name) {
1085                 kfree(tp);
1086                 return ERR_PTR(-ENOMEM);
1087         }
1088
1089         return tp;
1090 }
1091
1092 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1093                                     unsigned int var_ref_idx);
1094
1095 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1096                                unsigned int var_ref_idx)
1097 {
1098         struct tracepoint *tp = event->tp;
1099
1100         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1101                 struct tracepoint_func *probe_func_ptr;
1102                 synth_probe_func_t probe_func;
1103                 void *__data;
1104
1105                 if (!(cpu_online(raw_smp_processor_id())))
1106                         return;
1107
1108                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1109                 if (probe_func_ptr) {
1110                         do {
1111                                 probe_func = probe_func_ptr->func;
1112                                 __data = probe_func_ptr->data;
1113                                 probe_func(__data, var_ref_vals, var_ref_idx);
1114                         } while ((++probe_func_ptr)->func);
1115                 }
1116         }
1117 }
1118
1119 static struct synth_event *find_synth_event(const char *name)
1120 {
1121         struct dyn_event *pos;
1122         struct synth_event *event;
1123
1124         for_each_dyn_event(pos) {
1125                 if (!is_synth_event(pos))
1126                         continue;
1127                 event = to_synth_event(pos);
1128                 if (strcmp(event->name, name) == 0)
1129                         return event;
1130         }
1131
1132         return NULL;
1133 }
1134
1135 static int register_synth_event(struct synth_event *event)
1136 {
1137         struct trace_event_call *call = &event->call;
1138         int ret = 0;
1139
1140         event->call.class = &event->class;
1141         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1142         if (!event->class.system) {
1143                 ret = -ENOMEM;
1144                 goto out;
1145         }
1146
1147         event->tp = alloc_synth_tracepoint(event->name);
1148         if (IS_ERR(event->tp)) {
1149                 ret = PTR_ERR(event->tp);
1150                 event->tp = NULL;
1151                 goto out;
1152         }
1153
1154         INIT_LIST_HEAD(&call->class->fields);
1155         call->event.funcs = &synth_event_funcs;
1156         call->class->define_fields = synth_event_define_fields;
1157
1158         ret = register_trace_event(&call->event);
1159         if (!ret) {
1160                 ret = -ENODEV;
1161                 goto out;
1162         }
1163         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1164         call->class->reg = trace_event_reg;
1165         call->class->probe = trace_event_raw_event_synth;
1166         call->data = event;
1167         call->tp = event->tp;
1168
1169         ret = trace_add_event_call(call);
1170         if (ret) {
1171                 pr_warn("Failed to register synthetic event: %s\n",
1172                         trace_event_name(call));
1173                 goto err;
1174         }
1175
1176         ret = set_synth_event_print_fmt(call);
1177         if (ret < 0) {
1178                 trace_remove_event_call(call);
1179                 goto err;
1180         }
1181  out:
1182         return ret;
1183  err:
1184         unregister_trace_event(&call->event);
1185         goto out;
1186 }
1187
1188 static int unregister_synth_event(struct synth_event *event)
1189 {
1190         struct trace_event_call *call = &event->call;
1191         int ret;
1192
1193         ret = trace_remove_event_call(call);
1194
1195         return ret;
1196 }
1197
1198 static void free_synth_event(struct synth_event *event)
1199 {
1200         unsigned int i;
1201
1202         if (!event)
1203                 return;
1204
1205         for (i = 0; i < event->n_fields; i++)
1206                 free_synth_field(event->fields[i]);
1207
1208         kfree(event->fields);
1209         kfree(event->name);
1210         kfree(event->class.system);
1211         free_synth_tracepoint(event->tp);
1212         free_synth_event_print_fmt(&event->call);
1213         kfree(event);
1214 }
1215
1216 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1217                                              struct synth_field **fields)
1218 {
1219         struct synth_event *event;
1220         unsigned int i;
1221
1222         event = kzalloc(sizeof(*event), GFP_KERNEL);
1223         if (!event) {
1224                 event = ERR_PTR(-ENOMEM);
1225                 goto out;
1226         }
1227
1228         event->name = kstrdup(name, GFP_KERNEL);
1229         if (!event->name) {
1230                 kfree(event);
1231                 event = ERR_PTR(-ENOMEM);
1232                 goto out;
1233         }
1234
1235         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1236         if (!event->fields) {
1237                 free_synth_event(event);
1238                 event = ERR_PTR(-ENOMEM);
1239                 goto out;
1240         }
1241
1242         dyn_event_init(&event->devent, &synth_event_ops);
1243
1244         for (i = 0; i < n_fields; i++)
1245                 event->fields[i] = fields[i];
1246
1247         event->n_fields = n_fields;
1248  out:
1249         return event;
1250 }
1251
1252 static void action_trace(struct hist_trigger_data *hist_data,
1253                          struct tracing_map_elt *elt, void *rec,
1254                          struct ring_buffer_event *rbe, void *key,
1255                          struct action_data *data, u64 *var_ref_vals)
1256 {
1257         struct synth_event *event = data->synth_event;
1258
1259         trace_synth(event, var_ref_vals, data->var_ref_idx);
1260 }
1261
1262 struct hist_var_data {
1263         struct list_head list;
1264         struct hist_trigger_data *hist_data;
1265 };
1266
1267 static int __create_synth_event(int argc, const char *name, const char **argv)
1268 {
1269         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1270         struct synth_event *event = NULL;
1271         int i, consumed = 0, n_fields = 0, ret = 0;
1272
1273         /*
1274          * Argument syntax:
1275          *  - Add synthetic event: <event_name> field[;field] ...
1276          *  - Remove synthetic event: !<event_name> field[;field] ...
1277          *      where 'field' = type field_name
1278          */
1279
1280         if (name[0] == '\0' || argc < 1)
1281                 return -EINVAL;
1282
1283         mutex_lock(&event_mutex);
1284
1285         event = find_synth_event(name);
1286         if (event) {
1287                 ret = -EEXIST;
1288                 goto out;
1289         }
1290
1291         for (i = 0; i < argc - 1; i++) {
1292                 if (strcmp(argv[i], ";") == 0)
1293                         continue;
1294                 if (n_fields == SYNTH_FIELDS_MAX) {
1295                         ret = -EINVAL;
1296                         goto err;
1297                 }
1298
1299                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1300                 if (IS_ERR(field)) {
1301                         ret = PTR_ERR(field);
1302                         goto err;
1303                 }
1304                 fields[n_fields++] = field;
1305                 i += consumed - 1;
1306         }
1307
1308         if (i < argc && strcmp(argv[i], ";") != 0) {
1309                 ret = -EINVAL;
1310                 goto err;
1311         }
1312
1313         event = alloc_synth_event(name, n_fields, fields);
1314         if (IS_ERR(event)) {
1315                 ret = PTR_ERR(event);
1316                 event = NULL;
1317                 goto err;
1318         }
1319         ret = register_synth_event(event);
1320         if (!ret)
1321                 dyn_event_add(&event->devent);
1322         else
1323                 free_synth_event(event);
1324  out:
1325         mutex_unlock(&event_mutex);
1326
1327         return ret;
1328  err:
1329         for (i = 0; i < n_fields; i++)
1330                 free_synth_field(fields[i]);
1331
1332         goto out;
1333 }
1334
1335 static int create_or_delete_synth_event(int argc, char **argv)
1336 {
1337         const char *name = argv[0];
1338         struct synth_event *event = NULL;
1339         int ret;
1340
1341         /* trace_run_command() ensures argc != 0 */
1342         if (name[0] == '!') {
1343                 mutex_lock(&event_mutex);
1344                 event = find_synth_event(name + 1);
1345                 if (event) {
1346                         if (event->ref)
1347                                 ret = -EBUSY;
1348                         else {
1349                                 ret = unregister_synth_event(event);
1350                                 if (!ret) {
1351                                         dyn_event_remove(&event->devent);
1352                                         free_synth_event(event);
1353                                 }
1354                         }
1355                 } else
1356                         ret = -ENOENT;
1357                 mutex_unlock(&event_mutex);
1358                 return ret;
1359         }
1360
1361         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1362         return ret == -ECANCELED ? -EINVAL : ret;
1363 }
1364
1365 static int synth_event_create(int argc, const char **argv)
1366 {
1367         const char *name = argv[0];
1368         int len;
1369
1370         if (name[0] != 's' || name[1] != ':')
1371                 return -ECANCELED;
1372         name += 2;
1373
1374         /* This interface accepts group name prefix */
1375         if (strchr(name, '/')) {
1376                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1377                 if (len == 0)
1378                         return -EINVAL;
1379                 name += len;
1380         }
1381         return __create_synth_event(argc - 1, name, argv + 1);
1382 }
1383
1384 static int synth_event_release(struct dyn_event *ev)
1385 {
1386         struct synth_event *event = to_synth_event(ev);
1387         int ret;
1388
1389         if (event->ref)
1390                 return -EBUSY;
1391
1392         ret = unregister_synth_event(event);
1393         if (ret)
1394                 return ret;
1395
1396         dyn_event_remove(ev);
1397         free_synth_event(event);
1398         return 0;
1399 }
1400
1401 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1402 {
1403         struct synth_field *field;
1404         unsigned int i;
1405
1406         seq_printf(m, "%s\t", event->name);
1407
1408         for (i = 0; i < event->n_fields; i++) {
1409                 field = event->fields[i];
1410
1411                 /* parameter values */
1412                 seq_printf(m, "%s %s%s", field->type, field->name,
1413                            i == event->n_fields - 1 ? "" : "; ");
1414         }
1415
1416         seq_putc(m, '\n');
1417
1418         return 0;
1419 }
1420
1421 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1422 {
1423         struct synth_event *event = to_synth_event(ev);
1424
1425         seq_printf(m, "s:%s/", event->class.system);
1426
1427         return __synth_event_show(m, event);
1428 }
1429
1430 static int synth_events_seq_show(struct seq_file *m, void *v)
1431 {
1432         struct dyn_event *ev = v;
1433
1434         if (!is_synth_event(ev))
1435                 return 0;
1436
1437         return __synth_event_show(m, to_synth_event(ev));
1438 }
1439
1440 static const struct seq_operations synth_events_seq_op = {
1441         .start  = dyn_event_seq_start,
1442         .next   = dyn_event_seq_next,
1443         .stop   = dyn_event_seq_stop,
1444         .show   = synth_events_seq_show,
1445 };
1446
1447 static int synth_events_open(struct inode *inode, struct file *file)
1448 {
1449         int ret;
1450
1451         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1452                 ret = dyn_events_release_all(&synth_event_ops);
1453                 if (ret < 0)
1454                         return ret;
1455         }
1456
1457         return seq_open(file, &synth_events_seq_op);
1458 }
1459
1460 static ssize_t synth_events_write(struct file *file,
1461                                   const char __user *buffer,
1462                                   size_t count, loff_t *ppos)
1463 {
1464         return trace_parse_run_command(file, buffer, count, ppos,
1465                                        create_or_delete_synth_event);
1466 }
1467
1468 static const struct file_operations synth_events_fops = {
1469         .open           = synth_events_open,
1470         .write          = synth_events_write,
1471         .read           = seq_read,
1472         .llseek         = seq_lseek,
1473         .release        = seq_release,
1474 };
1475
1476 static u64 hist_field_timestamp(struct hist_field *hist_field,
1477                                 struct tracing_map_elt *elt,
1478                                 struct ring_buffer_event *rbe,
1479                                 void *event)
1480 {
1481         struct hist_trigger_data *hist_data = hist_field->hist_data;
1482         struct trace_array *tr = hist_data->event_file->tr;
1483
1484         u64 ts = ring_buffer_event_time_stamp(rbe);
1485
1486         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1487                 ts = ns2usecs(ts);
1488
1489         return ts;
1490 }
1491
1492 static u64 hist_field_cpu(struct hist_field *hist_field,
1493                           struct tracing_map_elt *elt,
1494                           struct ring_buffer_event *rbe,
1495                           void *event)
1496 {
1497         int cpu = smp_processor_id();
1498
1499         return cpu;
1500 }
1501
1502 /**
1503  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1504  * @hist_field: The VAR_REF field to check
1505  * @var_data: The hist trigger that owns the variable
1506  * @var_idx: The trigger variable identifier
1507  *
1508  * Check the given VAR_REF field to see whether or not it references
1509  * the given variable associated with the given trigger.
1510  *
1511  * Return: The VAR_REF field if it does reference the variable, NULL if not
1512  */
1513 static struct hist_field *
1514 check_field_for_var_ref(struct hist_field *hist_field,
1515                         struct hist_trigger_data *var_data,
1516                         unsigned int var_idx)
1517 {
1518         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1519
1520         if (hist_field && hist_field->var.idx == var_idx &&
1521             hist_field->var.hist_data == var_data)
1522                 return hist_field;
1523
1524         return NULL;
1525 }
1526
1527 /**
1528  * find_var_ref - Check if a trigger has a reference to a trigger variable
1529  * @hist_data: The hist trigger that might have a reference to the variable
1530  * @var_data: The hist trigger that owns the variable
1531  * @var_idx: The trigger variable identifier
1532  *
1533  * Check the list of var_refs[] on the first hist trigger to see
1534  * whether any of them are references to the variable on the second
1535  * trigger.
1536  *
1537  * Return: The VAR_REF field referencing the variable if so, NULL if not
1538  */
1539 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1540                                        struct hist_trigger_data *var_data,
1541                                        unsigned int var_idx)
1542 {
1543         struct hist_field *hist_field;
1544         unsigned int i;
1545
1546         for (i = 0; i < hist_data->n_var_refs; i++) {
1547                 hist_field = hist_data->var_refs[i];
1548                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1549                         return hist_field;
1550         }
1551
1552         return NULL;
1553 }
1554
1555 /**
1556  * find_any_var_ref - Check if there is a reference to a given trigger variable
1557  * @hist_data: The hist trigger
1558  * @var_idx: The trigger variable identifier
1559  *
1560  * Check to see whether the given variable is currently referenced by
1561  * any other trigger.
1562  *
1563  * The trigger the variable is defined on is explicitly excluded - the
1564  * assumption being that a self-reference doesn't prevent a trigger
1565  * from being removed.
1566  *
1567  * Return: The VAR_REF field referencing the variable if so, NULL if not
1568  */
1569 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1570                                            unsigned int var_idx)
1571 {
1572         struct trace_array *tr = hist_data->event_file->tr;
1573         struct hist_field *found = NULL;
1574         struct hist_var_data *var_data;
1575
1576         list_for_each_entry(var_data, &tr->hist_vars, list) {
1577                 if (var_data->hist_data == hist_data)
1578                         continue;
1579                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1580                 if (found)
1581                         break;
1582         }
1583
1584         return found;
1585 }
1586
1587 /**
1588  * check_var_refs - Check if there is a reference to any of trigger's variables
1589  * @hist_data: The hist trigger
1590  *
1591  * A trigger can define one or more variables.  If any one of them is
1592  * currently referenced by any other trigger, this function will
1593  * determine that.
1594
1595  * Typically used to determine whether or not a trigger can be removed
1596  * - if there are any references to a trigger's variables, it cannot.
1597  *
1598  * Return: True if there is a reference to any of trigger's variables
1599  */
1600 static bool check_var_refs(struct hist_trigger_data *hist_data)
1601 {
1602         struct hist_field *field;
1603         bool found = false;
1604         int i;
1605
1606         for_each_hist_field(i, hist_data) {
1607                 field = hist_data->fields[i];
1608                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1609                         if (find_any_var_ref(hist_data, field->var.idx)) {
1610                                 found = true;
1611                                 break;
1612                         }
1613                 }
1614         }
1615
1616         return found;
1617 }
1618
1619 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1620 {
1621         struct trace_array *tr = hist_data->event_file->tr;
1622         struct hist_var_data *var_data, *found = NULL;
1623
1624         list_for_each_entry(var_data, &tr->hist_vars, list) {
1625                 if (var_data->hist_data == hist_data) {
1626                         found = var_data;
1627                         break;
1628                 }
1629         }
1630
1631         return found;
1632 }
1633
1634 static bool field_has_hist_vars(struct hist_field *hist_field,
1635                                 unsigned int level)
1636 {
1637         int i;
1638
1639         if (level > 3)
1640                 return false;
1641
1642         if (!hist_field)
1643                 return false;
1644
1645         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1646             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1647                 return true;
1648
1649         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1650                 struct hist_field *operand;
1651
1652                 operand = hist_field->operands[i];
1653                 if (field_has_hist_vars(operand, level + 1))
1654                         return true;
1655         }
1656
1657         return false;
1658 }
1659
1660 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1661 {
1662         struct hist_field *hist_field;
1663         int i;
1664
1665         for_each_hist_field(i, hist_data) {
1666                 hist_field = hist_data->fields[i];
1667                 if (field_has_hist_vars(hist_field, 0))
1668                         return true;
1669         }
1670
1671         return false;
1672 }
1673
1674 static int save_hist_vars(struct hist_trigger_data *hist_data)
1675 {
1676         struct trace_array *tr = hist_data->event_file->tr;
1677         struct hist_var_data *var_data;
1678
1679         var_data = find_hist_vars(hist_data);
1680         if (var_data)
1681                 return 0;
1682
1683         if (trace_array_get(tr) < 0)
1684                 return -ENODEV;
1685
1686         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1687         if (!var_data) {
1688                 trace_array_put(tr);
1689                 return -ENOMEM;
1690         }
1691
1692         var_data->hist_data = hist_data;
1693         list_add(&var_data->list, &tr->hist_vars);
1694
1695         return 0;
1696 }
1697
1698 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1699 {
1700         struct trace_array *tr = hist_data->event_file->tr;
1701         struct hist_var_data *var_data;
1702
1703         var_data = find_hist_vars(hist_data);
1704         if (!var_data)
1705                 return;
1706
1707         if (WARN_ON(check_var_refs(hist_data)))
1708                 return;
1709
1710         list_del(&var_data->list);
1711
1712         kfree(var_data);
1713
1714         trace_array_put(tr);
1715 }
1716
1717 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1718                                          const char *var_name)
1719 {
1720         struct hist_field *hist_field, *found = NULL;
1721         int i;
1722
1723         for_each_hist_field(i, hist_data) {
1724                 hist_field = hist_data->fields[i];
1725                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1726                     strcmp(hist_field->var.name, var_name) == 0) {
1727                         found = hist_field;
1728                         break;
1729                 }
1730         }
1731
1732         return found;
1733 }
1734
1735 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1736                                    struct trace_event_file *file,
1737                                    const char *var_name)
1738 {
1739         struct hist_trigger_data *test_data;
1740         struct event_trigger_data *test;
1741         struct hist_field *hist_field;
1742
1743         hist_field = find_var_field(hist_data, var_name);
1744         if (hist_field)
1745                 return hist_field;
1746
1747         list_for_each_entry_rcu(test, &file->triggers, list) {
1748                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1749                         test_data = test->private_data;
1750                         hist_field = find_var_field(test_data, var_name);
1751                         if (hist_field)
1752                                 return hist_field;
1753                 }
1754         }
1755
1756         return NULL;
1757 }
1758
1759 static struct trace_event_file *find_var_file(struct trace_array *tr,
1760                                               char *system,
1761                                               char *event_name,
1762                                               char *var_name)
1763 {
1764         struct hist_trigger_data *var_hist_data;
1765         struct hist_var_data *var_data;
1766         struct trace_event_file *file, *found = NULL;
1767
1768         if (system)
1769                 return find_event_file(tr, system, event_name);
1770
1771         list_for_each_entry(var_data, &tr->hist_vars, list) {
1772                 var_hist_data = var_data->hist_data;
1773                 file = var_hist_data->event_file;
1774                 if (file == found)
1775                         continue;
1776
1777                 if (find_var_field(var_hist_data, var_name)) {
1778                         if (found) {
1779                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1780                                 return NULL;
1781                         }
1782
1783                         found = file;
1784                 }
1785         }
1786
1787         return found;
1788 }
1789
1790 static struct hist_field *find_file_var(struct trace_event_file *file,
1791                                         const char *var_name)
1792 {
1793         struct hist_trigger_data *test_data;
1794         struct event_trigger_data *test;
1795         struct hist_field *hist_field;
1796
1797         list_for_each_entry_rcu(test, &file->triggers, list) {
1798                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1799                         test_data = test->private_data;
1800                         hist_field = find_var_field(test_data, var_name);
1801                         if (hist_field)
1802                                 return hist_field;
1803                 }
1804         }
1805
1806         return NULL;
1807 }
1808
1809 static struct hist_field *
1810 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1811 {
1812         struct trace_array *tr = hist_data->event_file->tr;
1813         struct hist_field *hist_field, *found = NULL;
1814         struct trace_event_file *file;
1815         unsigned int i;
1816
1817         for (i = 0; i < hist_data->n_actions; i++) {
1818                 struct action_data *data = hist_data->actions[i];
1819
1820                 if (data->handler == HANDLER_ONMATCH) {
1821                         char *system = data->match_data.event_system;
1822                         char *event_name = data->match_data.event;
1823
1824                         file = find_var_file(tr, system, event_name, var_name);
1825                         if (!file)
1826                                 continue;
1827                         hist_field = find_file_var(file, var_name);
1828                         if (hist_field) {
1829                                 if (found) {
1830                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1831                                                  errpos(var_name));
1832                                         return ERR_PTR(-EINVAL);
1833                                 }
1834
1835                                 found = hist_field;
1836                         }
1837                 }
1838         }
1839         return found;
1840 }
1841
1842 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1843                                          char *system,
1844                                          char *event_name,
1845                                          char *var_name)
1846 {
1847         struct trace_array *tr = hist_data->event_file->tr;
1848         struct hist_field *hist_field = NULL;
1849         struct trace_event_file *file;
1850
1851         if (!system || !event_name) {
1852                 hist_field = find_match_var(hist_data, var_name);
1853                 if (IS_ERR(hist_field))
1854                         return NULL;
1855                 if (hist_field)
1856                         return hist_field;
1857         }
1858
1859         file = find_var_file(tr, system, event_name, var_name);
1860         if (!file)
1861                 return NULL;
1862
1863         hist_field = find_file_var(file, var_name);
1864
1865         return hist_field;
1866 }
1867
1868 static u64 hist_field_var_ref(struct hist_field *hist_field,
1869                               struct tracing_map_elt *elt,
1870                               struct ring_buffer_event *rbe,
1871                               void *event)
1872 {
1873         struct hist_elt_data *elt_data;
1874         u64 var_val = 0;
1875
1876         if (WARN_ON_ONCE(!elt))
1877                 return var_val;
1878
1879         elt_data = elt->private_data;
1880         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1881
1882         return var_val;
1883 }
1884
1885 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1886                              u64 *var_ref_vals, bool self)
1887 {
1888         struct hist_trigger_data *var_data;
1889         struct tracing_map_elt *var_elt;
1890         struct hist_field *hist_field;
1891         unsigned int i, var_idx;
1892         bool resolved = true;
1893         u64 var_val = 0;
1894
1895         for (i = 0; i < hist_data->n_var_refs; i++) {
1896                 hist_field = hist_data->var_refs[i];
1897                 var_idx = hist_field->var.idx;
1898                 var_data = hist_field->var.hist_data;
1899
1900                 if (var_data == NULL) {
1901                         resolved = false;
1902                         break;
1903                 }
1904
1905                 if ((self && var_data != hist_data) ||
1906                     (!self && var_data == hist_data))
1907                         continue;
1908
1909                 var_elt = tracing_map_lookup(var_data->map, key);
1910                 if (!var_elt) {
1911                         resolved = false;
1912                         break;
1913                 }
1914
1915                 if (!tracing_map_var_set(var_elt, var_idx)) {
1916                         resolved = false;
1917                         break;
1918                 }
1919
1920                 if (self || !hist_field->read_once)
1921                         var_val = tracing_map_read_var(var_elt, var_idx);
1922                 else
1923                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1924
1925                 var_ref_vals[i] = var_val;
1926         }
1927
1928         return resolved;
1929 }
1930
1931 static const char *hist_field_name(struct hist_field *field,
1932                                    unsigned int level)
1933 {
1934         const char *field_name = "";
1935
1936         if (level > 1)
1937                 return field_name;
1938
1939         if (field->field)
1940                 field_name = field->field->name;
1941         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1942                  field->flags & HIST_FIELD_FL_ALIAS)
1943                 field_name = hist_field_name(field->operands[0], ++level);
1944         else if (field->flags & HIST_FIELD_FL_CPU)
1945                 field_name = "cpu";
1946         else if (field->flags & HIST_FIELD_FL_EXPR ||
1947                  field->flags & HIST_FIELD_FL_VAR_REF) {
1948                 if (field->system) {
1949                         static char full_name[MAX_FILTER_STR_VAL];
1950
1951                         strcat(full_name, field->system);
1952                         strcat(full_name, ".");
1953                         strcat(full_name, field->event_name);
1954                         strcat(full_name, ".");
1955                         strcat(full_name, field->name);
1956                         field_name = full_name;
1957                 } else
1958                         field_name = field->name;
1959         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1960                 field_name = "common_timestamp";
1961
1962         if (field_name == NULL)
1963                 field_name = "";
1964
1965         return field_name;
1966 }
1967
1968 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1969 {
1970         hist_field_fn_t fn = NULL;
1971
1972         switch (field_size) {
1973         case 8:
1974                 if (field_is_signed)
1975                         fn = hist_field_s64;
1976                 else
1977                         fn = hist_field_u64;
1978                 break;
1979         case 4:
1980                 if (field_is_signed)
1981                         fn = hist_field_s32;
1982                 else
1983                         fn = hist_field_u32;
1984                 break;
1985         case 2:
1986                 if (field_is_signed)
1987                         fn = hist_field_s16;
1988                 else
1989                         fn = hist_field_u16;
1990                 break;
1991         case 1:
1992                 if (field_is_signed)
1993                         fn = hist_field_s8;
1994                 else
1995                         fn = hist_field_u8;
1996                 break;
1997         }
1998
1999         return fn;
2000 }
2001
2002 static int parse_map_size(char *str)
2003 {
2004         unsigned long size, map_bits;
2005         int ret;
2006
2007         strsep(&str, "=");
2008         if (!str) {
2009                 ret = -EINVAL;
2010                 goto out;
2011         }
2012
2013         ret = kstrtoul(str, 0, &size);
2014         if (ret)
2015                 goto out;
2016
2017         map_bits = ilog2(roundup_pow_of_two(size));
2018         if (map_bits < TRACING_MAP_BITS_MIN ||
2019             map_bits > TRACING_MAP_BITS_MAX)
2020                 ret = -EINVAL;
2021         else
2022                 ret = map_bits;
2023  out:
2024         return ret;
2025 }
2026
2027 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2028 {
2029         unsigned int i;
2030
2031         if (!attrs)
2032                 return;
2033
2034         for (i = 0; i < attrs->n_assignments; i++)
2035                 kfree(attrs->assignment_str[i]);
2036
2037         for (i = 0; i < attrs->n_actions; i++)
2038                 kfree(attrs->action_str[i]);
2039
2040         kfree(attrs->name);
2041         kfree(attrs->sort_key_str);
2042         kfree(attrs->keys_str);
2043         kfree(attrs->vals_str);
2044         kfree(attrs->clock);
2045         kfree(attrs);
2046 }
2047
2048 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2049 {
2050         int ret = -EINVAL;
2051
2052         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2053                 return ret;
2054
2055         if ((str_has_prefix(str, "onmatch(")) ||
2056             (str_has_prefix(str, "onmax(")) ||
2057             (str_has_prefix(str, "onchange("))) {
2058                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2059                 if (!attrs->action_str[attrs->n_actions]) {
2060                         ret = -ENOMEM;
2061                         return ret;
2062                 }
2063                 attrs->n_actions++;
2064                 ret = 0;
2065         }
2066         return ret;
2067 }
2068
2069 static int parse_assignment(struct trace_array *tr,
2070                             char *str, struct hist_trigger_attrs *attrs)
2071 {
2072         int ret = 0;
2073
2074         if ((str_has_prefix(str, "key=")) ||
2075             (str_has_prefix(str, "keys="))) {
2076                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
2077                 if (!attrs->keys_str) {
2078                         ret = -ENOMEM;
2079                         goto out;
2080                 }
2081         } else if ((str_has_prefix(str, "val=")) ||
2082                    (str_has_prefix(str, "vals=")) ||
2083                    (str_has_prefix(str, "values="))) {
2084                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
2085                 if (!attrs->vals_str) {
2086                         ret = -ENOMEM;
2087                         goto out;
2088                 }
2089         } else if (str_has_prefix(str, "sort=")) {
2090                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2091                 if (!attrs->sort_key_str) {
2092                         ret = -ENOMEM;
2093                         goto out;
2094                 }
2095         } else if (str_has_prefix(str, "name=")) {
2096                 attrs->name = kstrdup(str, GFP_KERNEL);
2097                 if (!attrs->name) {
2098                         ret = -ENOMEM;
2099                         goto out;
2100                 }
2101         } else if (str_has_prefix(str, "clock=")) {
2102                 strsep(&str, "=");
2103                 if (!str) {
2104                         ret = -EINVAL;
2105                         goto out;
2106                 }
2107
2108                 str = strstrip(str);
2109                 attrs->clock = kstrdup(str, GFP_KERNEL);
2110                 if (!attrs->clock) {
2111                         ret = -ENOMEM;
2112                         goto out;
2113                 }
2114         } else if (str_has_prefix(str, "size=")) {
2115                 int map_bits = parse_map_size(str);
2116
2117                 if (map_bits < 0) {
2118                         ret = map_bits;
2119                         goto out;
2120                 }
2121                 attrs->map_bits = map_bits;
2122         } else {
2123                 char *assignment;
2124
2125                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2126                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2127                         ret = -EINVAL;
2128                         goto out;
2129                 }
2130
2131                 assignment = kstrdup(str, GFP_KERNEL);
2132                 if (!assignment) {
2133                         ret = -ENOMEM;
2134                         goto out;
2135                 }
2136
2137                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2138         }
2139  out:
2140         return ret;
2141 }
2142
2143 static struct hist_trigger_attrs *
2144 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
2145 {
2146         struct hist_trigger_attrs *attrs;
2147         int ret = 0;
2148
2149         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2150         if (!attrs)
2151                 return ERR_PTR(-ENOMEM);
2152
2153         while (trigger_str) {
2154                 char *str = strsep(&trigger_str, ":");
2155
2156                 if (strchr(str, '=')) {
2157                         ret = parse_assignment(tr, str, attrs);
2158                         if (ret)
2159                                 goto free;
2160                 } else if (strcmp(str, "pause") == 0)
2161                         attrs->pause = true;
2162                 else if ((strcmp(str, "cont") == 0) ||
2163                          (strcmp(str, "continue") == 0))
2164                         attrs->cont = true;
2165                 else if (strcmp(str, "clear") == 0)
2166                         attrs->clear = true;
2167                 else {
2168                         ret = parse_action(str, attrs);
2169                         if (ret)
2170                                 goto free;
2171                 }
2172         }
2173
2174         if (!attrs->keys_str) {
2175                 ret = -EINVAL;
2176                 goto free;
2177         }
2178
2179         if (!attrs->clock) {
2180                 attrs->clock = kstrdup("global", GFP_KERNEL);
2181                 if (!attrs->clock) {
2182                         ret = -ENOMEM;
2183                         goto free;
2184                 }
2185         }
2186
2187         return attrs;
2188  free:
2189         destroy_hist_trigger_attrs(attrs);
2190
2191         return ERR_PTR(ret);
2192 }
2193
2194 static inline void save_comm(char *comm, struct task_struct *task)
2195 {
2196         if (!task->pid) {
2197                 strcpy(comm, "<idle>");
2198                 return;
2199         }
2200
2201         if (WARN_ON_ONCE(task->pid < 0)) {
2202                 strcpy(comm, "<XXX>");
2203                 return;
2204         }
2205
2206         strncpy(comm, task->comm, TASK_COMM_LEN);
2207 }
2208
2209 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2210 {
2211         unsigned int i;
2212
2213         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2214                 kfree(elt_data->field_var_str[i]);
2215
2216         kfree(elt_data->comm);
2217         kfree(elt_data);
2218 }
2219
2220 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2221 {
2222         struct hist_elt_data *elt_data = elt->private_data;
2223
2224         hist_elt_data_free(elt_data);
2225 }
2226
2227 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2228 {
2229         struct hist_trigger_data *hist_data = elt->map->private_data;
2230         unsigned int size = TASK_COMM_LEN;
2231         struct hist_elt_data *elt_data;
2232         struct hist_field *key_field;
2233         unsigned int i, n_str;
2234
2235         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2236         if (!elt_data)
2237                 return -ENOMEM;
2238
2239         for_each_hist_key_field(i, hist_data) {
2240                 key_field = hist_data->fields[i];
2241
2242                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2243                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2244                         if (!elt_data->comm) {
2245                                 kfree(elt_data);
2246                                 return -ENOMEM;
2247                         }
2248                         break;
2249                 }
2250         }
2251
2252         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2253
2254         size = STR_VAR_LEN_MAX;
2255
2256         for (i = 0; i < n_str; i++) {
2257                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2258                 if (!elt_data->field_var_str[i]) {
2259                         hist_elt_data_free(elt_data);
2260                         return -ENOMEM;
2261                 }
2262         }
2263
2264         elt->private_data = elt_data;
2265
2266         return 0;
2267 }
2268
2269 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2270 {
2271         struct hist_elt_data *elt_data = elt->private_data;
2272
2273         if (elt_data->comm)
2274                 save_comm(elt_data->comm, current);
2275 }
2276
2277 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2278         .elt_alloc      = hist_trigger_elt_data_alloc,
2279         .elt_free       = hist_trigger_elt_data_free,
2280         .elt_init       = hist_trigger_elt_data_init,
2281 };
2282
2283 static const char *get_hist_field_flags(struct hist_field *hist_field)
2284 {
2285         const char *flags_str = NULL;
2286
2287         if (hist_field->flags & HIST_FIELD_FL_HEX)
2288                 flags_str = "hex";
2289         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2290                 flags_str = "sym";
2291         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2292                 flags_str = "sym-offset";
2293         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2294                 flags_str = "execname";
2295         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2296                 flags_str = "syscall";
2297         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2298                 flags_str = "log2";
2299         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2300                 flags_str = "usecs";
2301
2302         return flags_str;
2303 }
2304
2305 static void expr_field_str(struct hist_field *field, char *expr)
2306 {
2307         if (field->flags & HIST_FIELD_FL_VAR_REF)
2308                 strcat(expr, "$");
2309
2310         strcat(expr, hist_field_name(field, 0));
2311
2312         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2313                 const char *flags_str = get_hist_field_flags(field);
2314
2315                 if (flags_str) {
2316                         strcat(expr, ".");
2317                         strcat(expr, flags_str);
2318                 }
2319         }
2320 }
2321
2322 static char *expr_str(struct hist_field *field, unsigned int level)
2323 {
2324         char *expr;
2325
2326         if (level > 1)
2327                 return NULL;
2328
2329         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2330         if (!expr)
2331                 return NULL;
2332
2333         if (!field->operands[0]) {
2334                 expr_field_str(field, expr);
2335                 return expr;
2336         }
2337
2338         if (field->operator == FIELD_OP_UNARY_MINUS) {
2339                 char *subexpr;
2340
2341                 strcat(expr, "-(");
2342                 subexpr = expr_str(field->operands[0], ++level);
2343                 if (!subexpr) {
2344                         kfree(expr);
2345                         return NULL;
2346                 }
2347                 strcat(expr, subexpr);
2348                 strcat(expr, ")");
2349
2350                 kfree(subexpr);
2351
2352                 return expr;
2353         }
2354
2355         expr_field_str(field->operands[0], expr);
2356
2357         switch (field->operator) {
2358         case FIELD_OP_MINUS:
2359                 strcat(expr, "-");
2360                 break;
2361         case FIELD_OP_PLUS:
2362                 strcat(expr, "+");
2363                 break;
2364         default:
2365                 kfree(expr);
2366                 return NULL;
2367         }
2368
2369         expr_field_str(field->operands[1], expr);
2370
2371         return expr;
2372 }
2373
2374 static int contains_operator(char *str)
2375 {
2376         enum field_op_id field_op = FIELD_OP_NONE;
2377         char *op;
2378
2379         op = strpbrk(str, "+-");
2380         if (!op)
2381                 return FIELD_OP_NONE;
2382
2383         switch (*op) {
2384         case '-':
2385                 if (*str == '-')
2386                         field_op = FIELD_OP_UNARY_MINUS;
2387                 else
2388                         field_op = FIELD_OP_MINUS;
2389                 break;
2390         case '+':
2391                 field_op = FIELD_OP_PLUS;
2392                 break;
2393         default:
2394                 break;
2395         }
2396
2397         return field_op;
2398 }
2399
2400 static void __destroy_hist_field(struct hist_field *hist_field)
2401 {
2402         kfree(hist_field->var.name);
2403         kfree(hist_field->name);
2404         kfree(hist_field->type);
2405
2406         kfree(hist_field);
2407 }
2408
2409 static void destroy_hist_field(struct hist_field *hist_field,
2410                                unsigned int level)
2411 {
2412         unsigned int i;
2413
2414         if (level > 3)
2415                 return;
2416
2417         if (!hist_field)
2418                 return;
2419
2420         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2421                 return; /* var refs will be destroyed separately */
2422
2423         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2424                 destroy_hist_field(hist_field->operands[i], level + 1);
2425
2426         __destroy_hist_field(hist_field);
2427 }
2428
2429 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2430                                             struct ftrace_event_field *field,
2431                                             unsigned long flags,
2432                                             char *var_name)
2433 {
2434         struct hist_field *hist_field;
2435
2436         if (field && is_function_field(field))
2437                 return NULL;
2438
2439         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2440         if (!hist_field)
2441                 return NULL;
2442
2443         hist_field->hist_data = hist_data;
2444
2445         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2446                 goto out; /* caller will populate */
2447
2448         if (flags & HIST_FIELD_FL_VAR_REF) {
2449                 hist_field->fn = hist_field_var_ref;
2450                 goto out;
2451         }
2452
2453         if (flags & HIST_FIELD_FL_HITCOUNT) {
2454                 hist_field->fn = hist_field_counter;
2455                 hist_field->size = sizeof(u64);
2456                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2457                 if (!hist_field->type)
2458                         goto free;
2459                 goto out;
2460         }
2461
2462         if (flags & HIST_FIELD_FL_STACKTRACE) {
2463                 hist_field->fn = hist_field_none;
2464                 goto out;
2465         }
2466
2467         if (flags & HIST_FIELD_FL_LOG2) {
2468                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2469                 hist_field->fn = hist_field_log2;
2470                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2471                 hist_field->size = hist_field->operands[0]->size;
2472                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2473                 if (!hist_field->type)
2474                         goto free;
2475                 goto out;
2476         }
2477
2478         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2479                 hist_field->fn = hist_field_timestamp;
2480                 hist_field->size = sizeof(u64);
2481                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2482                 if (!hist_field->type)
2483                         goto free;
2484                 goto out;
2485         }
2486
2487         if (flags & HIST_FIELD_FL_CPU) {
2488                 hist_field->fn = hist_field_cpu;
2489                 hist_field->size = sizeof(int);
2490                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2491                 if (!hist_field->type)
2492                         goto free;
2493                 goto out;
2494         }
2495
2496         if (WARN_ON_ONCE(!field))
2497                 goto out;
2498
2499         if (is_string_field(field)) {
2500                 flags |= HIST_FIELD_FL_STRING;
2501
2502                 hist_field->size = MAX_FILTER_STR_VAL;
2503                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2504                 if (!hist_field->type)
2505                         goto free;
2506
2507                 if (field->filter_type == FILTER_STATIC_STRING)
2508                         hist_field->fn = hist_field_string;
2509                 else if (field->filter_type == FILTER_DYN_STRING)
2510                         hist_field->fn = hist_field_dynstring;
2511                 else
2512                         hist_field->fn = hist_field_pstring;
2513         } else {
2514                 hist_field->size = field->size;
2515                 hist_field->is_signed = field->is_signed;
2516                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2517                 if (!hist_field->type)
2518                         goto free;
2519
2520                 hist_field->fn = select_value_fn(field->size,
2521                                                  field->is_signed);
2522                 if (!hist_field->fn) {
2523                         destroy_hist_field(hist_field, 0);
2524                         return NULL;
2525                 }
2526         }
2527  out:
2528         hist_field->field = field;
2529         hist_field->flags = flags;
2530
2531         if (var_name) {
2532                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2533                 if (!hist_field->var.name)
2534                         goto free;
2535         }
2536
2537         return hist_field;
2538  free:
2539         destroy_hist_field(hist_field, 0);
2540         return NULL;
2541 }
2542
2543 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2544 {
2545         unsigned int i;
2546
2547         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2548                 if (hist_data->fields[i]) {
2549                         destroy_hist_field(hist_data->fields[i], 0);
2550                         hist_data->fields[i] = NULL;
2551                 }
2552         }
2553
2554         for (i = 0; i < hist_data->n_var_refs; i++) {
2555                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2556                 __destroy_hist_field(hist_data->var_refs[i]);
2557                 hist_data->var_refs[i] = NULL;
2558         }
2559 }
2560
2561 static int init_var_ref(struct hist_field *ref_field,
2562                         struct hist_field *var_field,
2563                         char *system, char *event_name)
2564 {
2565         int err = 0;
2566
2567         ref_field->var.idx = var_field->var.idx;
2568         ref_field->var.hist_data = var_field->hist_data;
2569         ref_field->size = var_field->size;
2570         ref_field->is_signed = var_field->is_signed;
2571         ref_field->flags |= var_field->flags &
2572                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2573
2574         if (system) {
2575                 ref_field->system = kstrdup(system, GFP_KERNEL);
2576                 if (!ref_field->system)
2577                         return -ENOMEM;
2578         }
2579
2580         if (event_name) {
2581                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2582                 if (!ref_field->event_name) {
2583                         err = -ENOMEM;
2584                         goto free;
2585                 }
2586         }
2587
2588         if (var_field->var.name) {
2589                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2590                 if (!ref_field->name) {
2591                         err = -ENOMEM;
2592                         goto free;
2593                 }
2594         } else if (var_field->name) {
2595                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2596                 if (!ref_field->name) {
2597                         err = -ENOMEM;
2598                         goto free;
2599                 }
2600         }
2601
2602         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2603         if (!ref_field->type) {
2604                 err = -ENOMEM;
2605                 goto free;
2606         }
2607  out:
2608         return err;
2609  free:
2610         kfree(ref_field->system);
2611         kfree(ref_field->event_name);
2612         kfree(ref_field->name);
2613
2614         goto out;
2615 }
2616
2617 /**
2618  * create_var_ref - Create a variable reference and attach it to trigger
2619  * @hist_data: The trigger that will be referencing the variable
2620  * @var_field: The VAR field to create a reference to
2621  * @system: The optional system string
2622  * @event_name: The optional event_name string
2623  *
2624  * Given a variable hist_field, create a VAR_REF hist_field that
2625  * represents a reference to it.
2626  *
2627  * This function also adds the reference to the trigger that
2628  * now references the variable.
2629  *
2630  * Return: The VAR_REF field if successful, NULL if not
2631  */
2632 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2633                                          struct hist_field *var_field,
2634                                          char *system, char *event_name)
2635 {
2636         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2637         struct hist_field *ref_field;
2638
2639         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2640         if (ref_field) {
2641                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2642                         destroy_hist_field(ref_field, 0);
2643                         return NULL;
2644                 }
2645
2646                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2647                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2648         }
2649
2650         return ref_field;
2651 }
2652
2653 static bool is_var_ref(char *var_name)
2654 {
2655         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2656                 return false;
2657
2658         return true;
2659 }
2660
2661 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2662                                  char *var_name)
2663 {
2664         char *name, *field;
2665         unsigned int i;
2666
2667         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2668                 name = hist_data->attrs->var_defs.name[i];
2669
2670                 if (strcmp(var_name, name) == 0) {
2671                         field = hist_data->attrs->var_defs.expr[i];
2672                         if (contains_operator(field) || is_var_ref(field))
2673                                 continue;
2674                         return field;
2675                 }
2676         }
2677
2678         return NULL;
2679 }
2680
2681 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2682                                  char *system, char *event_name,
2683                                  char *var_name)
2684 {
2685         struct trace_event_call *call;
2686
2687         if (system && event_name) {
2688                 call = hist_data->event_file->event_call;
2689
2690                 if (strcmp(system, call->class->system) != 0)
2691                         return NULL;
2692
2693                 if (strcmp(event_name, trace_event_name(call)) != 0)
2694                         return NULL;
2695         }
2696
2697         if (!!system != !!event_name)
2698                 return NULL;
2699
2700         if (!is_var_ref(var_name))
2701                 return NULL;
2702
2703         var_name++;
2704
2705         return field_name_from_var(hist_data, var_name);
2706 }
2707
2708 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2709                                         char *system, char *event_name,
2710                                         char *var_name)
2711 {
2712         struct hist_field *var_field = NULL, *ref_field = NULL;
2713         struct trace_array *tr = hist_data->event_file->tr;
2714
2715         if (!is_var_ref(var_name))
2716                 return NULL;
2717
2718         var_name++;
2719
2720         var_field = find_event_var(hist_data, system, event_name, var_name);
2721         if (var_field)
2722                 ref_field = create_var_ref(hist_data, var_field,
2723                                            system, event_name);
2724
2725         if (!ref_field)
2726                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2727
2728         return ref_field;
2729 }
2730
2731 static struct ftrace_event_field *
2732 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2733             char *field_str, unsigned long *flags)
2734 {
2735         struct ftrace_event_field *field = NULL;
2736         char *field_name, *modifier, *str;
2737         struct trace_array *tr = file->tr;
2738
2739         modifier = str = kstrdup(field_str, GFP_KERNEL);
2740         if (!modifier)
2741                 return ERR_PTR(-ENOMEM);
2742
2743         field_name = strsep(&modifier, ".");
2744         if (modifier) {
2745                 if (strcmp(modifier, "hex") == 0)
2746                         *flags |= HIST_FIELD_FL_HEX;
2747                 else if (strcmp(modifier, "sym") == 0)
2748                         *flags |= HIST_FIELD_FL_SYM;
2749                 else if (strcmp(modifier, "sym-offset") == 0)
2750                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2751                 else if ((strcmp(modifier, "execname") == 0) &&
2752                          (strcmp(field_name, "common_pid") == 0))
2753                         *flags |= HIST_FIELD_FL_EXECNAME;
2754                 else if (strcmp(modifier, "syscall") == 0)
2755                         *flags |= HIST_FIELD_FL_SYSCALL;
2756                 else if (strcmp(modifier, "log2") == 0)
2757                         *flags |= HIST_FIELD_FL_LOG2;
2758                 else if (strcmp(modifier, "usecs") == 0)
2759                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2760                 else {
2761                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2762                         field = ERR_PTR(-EINVAL);
2763                         goto out;
2764                 }
2765         }
2766
2767         if (strcmp(field_name, "common_timestamp") == 0) {
2768                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2769                 hist_data->enable_timestamps = true;
2770                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2771                         hist_data->attrs->ts_in_usecs = true;
2772         } else if (strcmp(field_name, "cpu") == 0)
2773                 *flags |= HIST_FIELD_FL_CPU;
2774         else {
2775                 field = trace_find_event_field(file->event_call, field_name);
2776                 if (!field || !field->size) {
2777                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2778                         field = ERR_PTR(-EINVAL);
2779                         goto out;
2780                 }
2781         }
2782  out:
2783         kfree(str);
2784
2785         return field;
2786 }
2787
2788 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2789                                        struct hist_field *var_ref,
2790                                        char *var_name)
2791 {
2792         struct hist_field *alias = NULL;
2793         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2794
2795         alias = create_hist_field(hist_data, NULL, flags, var_name);
2796         if (!alias)
2797                 return NULL;
2798
2799         alias->fn = var_ref->fn;
2800         alias->operands[0] = var_ref;
2801
2802         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2803                 destroy_hist_field(alias, 0);
2804                 return NULL;
2805         }
2806
2807         alias->var_ref_idx = var_ref->var_ref_idx;
2808
2809         return alias;
2810 }
2811
2812 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2813                                      struct trace_event_file *file, char *str,
2814                                      unsigned long *flags, char *var_name)
2815 {
2816         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2817         struct ftrace_event_field *field = NULL;
2818         struct hist_field *hist_field = NULL;
2819         int ret = 0;
2820
2821         s = strchr(str, '.');
2822         if (s) {
2823                 s = strchr(++s, '.');
2824                 if (s) {
2825                         ref_system = strsep(&str, ".");
2826                         if (!str) {
2827                                 ret = -EINVAL;
2828                                 goto out;
2829                         }
2830                         ref_event = strsep(&str, ".");
2831                         if (!str) {
2832                                 ret = -EINVAL;
2833                                 goto out;
2834                         }
2835                         ref_var = str;
2836                 }
2837         }
2838
2839         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2840         if (!s) {
2841                 hist_field = parse_var_ref(hist_data, ref_system,
2842                                            ref_event, ref_var);
2843                 if (hist_field) {
2844                         if (var_name) {
2845                                 hist_field = create_alias(hist_data, hist_field, var_name);
2846                                 if (!hist_field) {
2847                                         ret = -ENOMEM;
2848                                         goto out;
2849                                 }
2850                         }
2851                         return hist_field;
2852                 }
2853         } else
2854                 str = s;
2855
2856         field = parse_field(hist_data, file, str, flags);
2857         if (IS_ERR(field)) {
2858                 ret = PTR_ERR(field);
2859                 goto out;
2860         }
2861
2862         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2863         if (!hist_field) {
2864                 ret = -ENOMEM;
2865                 goto out;
2866         }
2867
2868         return hist_field;
2869  out:
2870         return ERR_PTR(ret);
2871 }
2872
2873 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2874                                      struct trace_event_file *file,
2875                                      char *str, unsigned long flags,
2876                                      char *var_name, unsigned int level);
2877
2878 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2879                                       struct trace_event_file *file,
2880                                       char *str, unsigned long flags,
2881                                       char *var_name, unsigned int level)
2882 {
2883         struct hist_field *operand1, *expr = NULL;
2884         unsigned long operand_flags;
2885         int ret = 0;
2886         char *s;
2887
2888         /* we support only -(xxx) i.e. explicit parens required */
2889
2890         if (level > 3) {
2891                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2892                 ret = -EINVAL;
2893                 goto free;
2894         }
2895
2896         str++; /* skip leading '-' */
2897
2898         s = strchr(str, '(');
2899         if (s)
2900                 str++;
2901         else {
2902                 ret = -EINVAL;
2903                 goto free;
2904         }
2905
2906         s = strrchr(str, ')');
2907         if (s)
2908                 *s = '\0';
2909         else {
2910                 ret = -EINVAL; /* no closing ')' */
2911                 goto free;
2912         }
2913
2914         flags |= HIST_FIELD_FL_EXPR;
2915         expr = create_hist_field(hist_data, NULL, flags, var_name);
2916         if (!expr) {
2917                 ret = -ENOMEM;
2918                 goto free;
2919         }
2920
2921         operand_flags = 0;
2922         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2923         if (IS_ERR(operand1)) {
2924                 ret = PTR_ERR(operand1);
2925                 goto free;
2926         }
2927
2928         expr->flags |= operand1->flags &
2929                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2930         expr->fn = hist_field_unary_minus;
2931         expr->operands[0] = operand1;
2932         expr->operator = FIELD_OP_UNARY_MINUS;
2933         expr->name = expr_str(expr, 0);
2934         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2935         if (!expr->type) {
2936                 ret = -ENOMEM;
2937                 goto free;
2938         }
2939
2940         return expr;
2941  free:
2942         destroy_hist_field(expr, 0);
2943         return ERR_PTR(ret);
2944 }
2945
2946 static int check_expr_operands(struct trace_array *tr,
2947                                struct hist_field *operand1,
2948                                struct hist_field *operand2)
2949 {
2950         unsigned long operand1_flags = operand1->flags;
2951         unsigned long operand2_flags = operand2->flags;
2952
2953         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2954             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2955                 struct hist_field *var;
2956
2957                 var = find_var_field(operand1->var.hist_data, operand1->name);
2958                 if (!var)
2959                         return -EINVAL;
2960                 operand1_flags = var->flags;
2961         }
2962
2963         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2964             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2965                 struct hist_field *var;
2966
2967                 var = find_var_field(operand2->var.hist_data, operand2->name);
2968                 if (!var)
2969                         return -EINVAL;
2970                 operand2_flags = var->flags;
2971         }
2972
2973         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2974             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2975                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2976                 return -EINVAL;
2977         }
2978
2979         return 0;
2980 }
2981
2982 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2983                                      struct trace_event_file *file,
2984                                      char *str, unsigned long flags,
2985                                      char *var_name, unsigned int level)
2986 {
2987         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2988         unsigned long operand_flags;
2989         int field_op, ret = -EINVAL;
2990         char *sep, *operand1_str;
2991
2992         if (level > 3) {
2993                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2994                 return ERR_PTR(-EINVAL);
2995         }
2996
2997         field_op = contains_operator(str);
2998
2999         if (field_op == FIELD_OP_NONE)
3000                 return parse_atom(hist_data, file, str, &flags, var_name);
3001
3002         if (field_op == FIELD_OP_UNARY_MINUS)
3003                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
3004
3005         switch (field_op) {
3006         case FIELD_OP_MINUS:
3007                 sep = "-";
3008                 break;
3009         case FIELD_OP_PLUS:
3010                 sep = "+";
3011                 break;
3012         default:
3013                 goto free;
3014         }
3015
3016         operand1_str = strsep(&str, sep);
3017         if (!operand1_str || !str)
3018                 goto free;
3019
3020         operand_flags = 0;
3021         operand1 = parse_atom(hist_data, file, operand1_str,
3022                               &operand_flags, NULL);
3023         if (IS_ERR(operand1)) {
3024                 ret = PTR_ERR(operand1);
3025                 operand1 = NULL;
3026                 goto free;
3027         }
3028
3029         /* rest of string could be another expression e.g. b+c in a+b+c */
3030         operand_flags = 0;
3031         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3032         if (IS_ERR(operand2)) {
3033                 ret = PTR_ERR(operand2);
3034                 operand2 = NULL;
3035                 goto free;
3036         }
3037
3038         ret = check_expr_operands(file->tr, operand1, operand2);
3039         if (ret)
3040                 goto free;
3041
3042         flags |= HIST_FIELD_FL_EXPR;
3043
3044         flags |= operand1->flags &
3045                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3046
3047         expr = create_hist_field(hist_data, NULL, flags, var_name);
3048         if (!expr) {
3049                 ret = -ENOMEM;
3050                 goto free;
3051         }
3052
3053         operand1->read_once = true;
3054         operand2->read_once = true;
3055
3056         expr->operands[0] = operand1;
3057         expr->operands[1] = operand2;
3058         expr->operator = field_op;
3059         expr->name = expr_str(expr, 0);
3060         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3061         if (!expr->type) {
3062                 ret = -ENOMEM;
3063                 goto free;
3064         }
3065
3066         switch (field_op) {
3067         case FIELD_OP_MINUS:
3068                 expr->fn = hist_field_minus;
3069                 break;
3070         case FIELD_OP_PLUS:
3071                 expr->fn = hist_field_plus;
3072                 break;
3073         default:
3074                 ret = -EINVAL;
3075                 goto free;
3076         }
3077
3078         return expr;
3079  free:
3080         destroy_hist_field(operand1, 0);
3081         destroy_hist_field(operand2, 0);
3082         destroy_hist_field(expr, 0);
3083
3084         return ERR_PTR(ret);
3085 }
3086
3087 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3088                                  struct trace_event_file *file)
3089 {
3090         struct event_trigger_data *test;
3091
3092         list_for_each_entry_rcu(test, &file->triggers, list) {
3093                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3094                         if (test->private_data == hist_data)
3095                                 return test->filter_str;
3096                 }
3097         }
3098
3099         return NULL;
3100 }
3101
3102 static struct event_command trigger_hist_cmd;
3103 static int event_hist_trigger_func(struct event_command *cmd_ops,
3104                                    struct trace_event_file *file,
3105                                    char *glob, char *cmd, char *param);
3106
3107 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3108                             struct hist_trigger_data *hist_data,
3109                             unsigned int n_keys)
3110 {
3111         struct hist_field *target_hist_field, *hist_field;
3112         unsigned int n, i, j;
3113
3114         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3115                 return false;
3116
3117         i = hist_data->n_vals;
3118         j = target_hist_data->n_vals;
3119
3120         for (n = 0; n < n_keys; n++) {
3121                 hist_field = hist_data->fields[i + n];
3122                 target_hist_field = target_hist_data->fields[j + n];
3123
3124                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3125                         return false;
3126                 if (hist_field->size != target_hist_field->size)
3127                         return false;
3128                 if (hist_field->is_signed != target_hist_field->is_signed)
3129                         return false;
3130         }
3131
3132         return true;
3133 }
3134
3135 static struct hist_trigger_data *
3136 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3137                      struct trace_event_file *file)
3138 {
3139         struct hist_trigger_data *hist_data;
3140         struct event_trigger_data *test;
3141         unsigned int n_keys;
3142
3143         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3144
3145         list_for_each_entry_rcu(test, &file->triggers, list) {
3146                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3147                         hist_data = test->private_data;
3148
3149                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3150                                 return hist_data;
3151                 }
3152         }
3153
3154         return NULL;
3155 }
3156
3157 static struct trace_event_file *event_file(struct trace_array *tr,
3158                                            char *system, char *event_name)
3159 {
3160         struct trace_event_file *file;
3161
3162         file = __find_event_file(tr, system, event_name);
3163         if (!file)
3164                 return ERR_PTR(-EINVAL);
3165
3166         return file;
3167 }
3168
3169 static struct hist_field *
3170 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3171                          char *system, char *event_name, char *field_name)
3172 {
3173         struct hist_field *event_var;
3174         char *synthetic_name;
3175
3176         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3177         if (!synthetic_name)
3178                 return ERR_PTR(-ENOMEM);
3179
3180         strcpy(synthetic_name, "synthetic_");
3181         strcat(synthetic_name, field_name);
3182
3183         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3184
3185         kfree(synthetic_name);
3186
3187         return event_var;
3188 }
3189
3190 /**
3191  * create_field_var_hist - Automatically create a histogram and var for a field
3192  * @target_hist_data: The target hist trigger
3193  * @subsys_name: Optional subsystem name
3194  * @event_name: Optional event name
3195  * @field_name: The name of the field (and the resulting variable)
3196  *
3197  * Hist trigger actions fetch data from variables, not directly from
3198  * events.  However, for convenience, users are allowed to directly
3199  * specify an event field in an action, which will be automatically
3200  * converted into a variable on their behalf.
3201
3202  * If a user specifies a field on an event that isn't the event the
3203  * histogram currently being defined (the target event histogram), the
3204  * only way that can be accomplished is if a new hist trigger is
3205  * created and the field variable defined on that.
3206  *
3207  * This function creates a new histogram compatible with the target
3208  * event (meaning a histogram with the same key as the target
3209  * histogram), and creates a variable for the specified field, but
3210  * with 'synthetic_' prepended to the variable name in order to avoid
3211  * collision with normal field variables.
3212  *
3213  * Return: The variable created for the field.
3214  */
3215 static struct hist_field *
3216 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3217                       char *subsys_name, char *event_name, char *field_name)
3218 {
3219         struct trace_array *tr = target_hist_data->event_file->tr;
3220         struct hist_field *event_var = ERR_PTR(-EINVAL);
3221         struct hist_trigger_data *hist_data;
3222         unsigned int i, n, first = true;
3223         struct field_var_hist *var_hist;
3224         struct trace_event_file *file;
3225         struct hist_field *key_field;
3226         char *saved_filter;
3227         char *cmd;
3228         int ret;
3229
3230         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3231                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3232                 return ERR_PTR(-EINVAL);
3233         }
3234
3235         file = event_file(tr, subsys_name, event_name);
3236
3237         if (IS_ERR(file)) {
3238                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3239                 ret = PTR_ERR(file);
3240                 return ERR_PTR(ret);
3241         }
3242
3243         /*
3244          * Look for a histogram compatible with target.  We'll use the
3245          * found histogram specification to create a new matching
3246          * histogram with our variable on it.  target_hist_data is not
3247          * yet a registered histogram so we can't use that.
3248          */
3249         hist_data = find_compatible_hist(target_hist_data, file);
3250         if (!hist_data) {
3251                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3252                 return ERR_PTR(-EINVAL);
3253         }
3254
3255         /* See if a synthetic field variable has already been created */
3256         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3257                                              event_name, field_name);
3258         if (!IS_ERR_OR_NULL(event_var))
3259                 return event_var;
3260
3261         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3262         if (!var_hist)
3263                 return ERR_PTR(-ENOMEM);
3264
3265         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3266         if (!cmd) {
3267                 kfree(var_hist);
3268                 return ERR_PTR(-ENOMEM);
3269         }
3270
3271         /* Use the same keys as the compatible histogram */
3272         strcat(cmd, "keys=");
3273
3274         for_each_hist_key_field(i, hist_data) {
3275                 key_field = hist_data->fields[i];
3276                 if (!first)
3277                         strcat(cmd, ",");
3278                 strcat(cmd, key_field->field->name);
3279                 first = false;
3280         }
3281
3282         /* Create the synthetic field variable specification */
3283         strcat(cmd, ":synthetic_");
3284         strcat(cmd, field_name);
3285         strcat(cmd, "=");
3286         strcat(cmd, field_name);
3287
3288         /* Use the same filter as the compatible histogram */
3289         saved_filter = find_trigger_filter(hist_data, file);
3290         if (saved_filter) {
3291                 strcat(cmd, " if ");
3292                 strcat(cmd, saved_filter);
3293         }
3294
3295         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3296         if (!var_hist->cmd) {
3297                 kfree(cmd);
3298                 kfree(var_hist);
3299                 return ERR_PTR(-ENOMEM);
3300         }
3301
3302         /* Save the compatible histogram information */
3303         var_hist->hist_data = hist_data;
3304
3305         /* Create the new histogram with our variable */
3306         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3307                                       "", "hist", cmd);
3308         if (ret) {
3309                 kfree(cmd);
3310                 kfree(var_hist->cmd);
3311                 kfree(var_hist);
3312                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3313                 return ERR_PTR(ret);
3314         }
3315
3316         kfree(cmd);
3317
3318         /* If we can't find the variable, something went wrong */
3319         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3320                                              event_name, field_name);
3321         if (IS_ERR_OR_NULL(event_var)) {
3322                 kfree(var_hist->cmd);
3323                 kfree(var_hist);
3324                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3325                 return ERR_PTR(-EINVAL);
3326         }
3327
3328         n = target_hist_data->n_field_var_hists;
3329         target_hist_data->field_var_hists[n] = var_hist;
3330         target_hist_data->n_field_var_hists++;
3331
3332         return event_var;
3333 }
3334
3335 static struct hist_field *
3336 find_target_event_var(struct hist_trigger_data *hist_data,
3337                       char *subsys_name, char *event_name, char *var_name)
3338 {
3339         struct trace_event_file *file = hist_data->event_file;
3340         struct hist_field *hist_field = NULL;
3341
3342         if (subsys_name) {
3343                 struct trace_event_call *call;
3344
3345                 if (!event_name)
3346                         return NULL;
3347
3348                 call = file->event_call;
3349
3350                 if (strcmp(subsys_name, call->class->system) != 0)
3351                         return NULL;
3352
3353                 if (strcmp(event_name, trace_event_name(call)) != 0)
3354                         return NULL;
3355         }
3356
3357         hist_field = find_var_field(hist_data, var_name);
3358
3359         return hist_field;
3360 }
3361
3362 static inline void __update_field_vars(struct tracing_map_elt *elt,
3363                                        struct ring_buffer_event *rbe,
3364                                        void *rec,
3365                                        struct field_var **field_vars,
3366                                        unsigned int n_field_vars,
3367                                        unsigned int field_var_str_start)
3368 {
3369         struct hist_elt_data *elt_data = elt->private_data;
3370         unsigned int i, j, var_idx;
3371         u64 var_val;
3372
3373         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3374                 struct field_var *field_var = field_vars[i];
3375                 struct hist_field *var = field_var->var;
3376                 struct hist_field *val = field_var->val;
3377
3378                 var_val = val->fn(val, elt, rbe, rec);
3379                 var_idx = var->var.idx;
3380
3381                 if (val->flags & HIST_FIELD_FL_STRING) {
3382                         char *str = elt_data->field_var_str[j++];
3383                         char *val_str = (char *)(uintptr_t)var_val;
3384
3385                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3386                         var_val = (u64)(uintptr_t)str;
3387                 }
3388                 tracing_map_set_var(elt, var_idx, var_val);
3389         }
3390 }
3391
3392 static void update_field_vars(struct hist_trigger_data *hist_data,
3393                               struct tracing_map_elt *elt,
3394                               struct ring_buffer_event *rbe,
3395                               void *rec)
3396 {
3397         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3398                             hist_data->n_field_vars, 0);
3399 }
3400
3401 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3402                                  struct tracing_map_elt *elt, void *rec,
3403                                  struct ring_buffer_event *rbe, void *key,
3404                                  struct action_data *data, u64 *var_ref_vals)
3405 {
3406         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3407                             hist_data->n_save_vars, hist_data->n_field_var_str);
3408 }
3409
3410 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3411                                      struct trace_event_file *file,
3412                                      char *name, int size, const char *type)
3413 {
3414         struct hist_field *var;
3415         int idx;
3416
3417         if (find_var(hist_data, file, name) && !hist_data->remove) {
3418                 var = ERR_PTR(-EINVAL);
3419                 goto out;
3420         }
3421
3422         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3423         if (!var) {
3424                 var = ERR_PTR(-ENOMEM);
3425                 goto out;
3426         }
3427
3428         idx = tracing_map_add_var(hist_data->map);
3429         if (idx < 0) {
3430                 kfree(var);
3431                 var = ERR_PTR(-EINVAL);
3432                 goto out;
3433         }
3434
3435         var->flags = HIST_FIELD_FL_VAR;
3436         var->var.idx = idx;
3437         var->var.hist_data = var->hist_data = hist_data;
3438         var->size = size;
3439         var->var.name = kstrdup(name, GFP_KERNEL);
3440         var->type = kstrdup(type, GFP_KERNEL);
3441         if (!var->var.name || !var->type) {
3442                 kfree(var->var.name);
3443                 kfree(var->type);
3444                 kfree(var);
3445                 var = ERR_PTR(-ENOMEM);
3446         }
3447  out:
3448         return var;
3449 }
3450
3451 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3452                                           struct trace_event_file *file,
3453                                           char *field_name)
3454 {
3455         struct hist_field *val = NULL, *var = NULL;
3456         unsigned long flags = HIST_FIELD_FL_VAR;
3457         struct trace_array *tr = file->tr;
3458         struct field_var *field_var;
3459         int ret = 0;
3460
3461         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3462                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3463                 ret = -EINVAL;
3464                 goto err;
3465         }
3466
3467         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3468         if (IS_ERR(val)) {
3469                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3470                 ret = PTR_ERR(val);
3471                 goto err;
3472         }
3473
3474         var = create_var(hist_data, file, field_name, val->size, val->type);
3475         if (IS_ERR(var)) {
3476                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3477                 kfree(val);
3478                 ret = PTR_ERR(var);
3479                 goto err;
3480         }
3481
3482         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3483         if (!field_var) {
3484                 kfree(val);
3485                 kfree(var);
3486                 ret =  -ENOMEM;
3487                 goto err;
3488         }
3489
3490         field_var->var = var;
3491         field_var->val = val;
3492  out:
3493         return field_var;
3494  err:
3495         field_var = ERR_PTR(ret);
3496         goto out;
3497 }
3498
3499 /**
3500  * create_target_field_var - Automatically create a variable for a field
3501  * @target_hist_data: The target hist trigger
3502  * @subsys_name: Optional subsystem name
3503  * @event_name: Optional event name
3504  * @var_name: The name of the field (and the resulting variable)
3505  *
3506  * Hist trigger actions fetch data from variables, not directly from
3507  * events.  However, for convenience, users are allowed to directly
3508  * specify an event field in an action, which will be automatically
3509  * converted into a variable on their behalf.
3510
3511  * This function creates a field variable with the name var_name on
3512  * the hist trigger currently being defined on the target event.  If
3513  * subsys_name and event_name are specified, this function simply
3514  * verifies that they do in fact match the target event subsystem and
3515  * event name.
3516  *
3517  * Return: The variable created for the field.
3518  */
3519 static struct field_var *
3520 create_target_field_var(struct hist_trigger_data *target_hist_data,
3521                         char *subsys_name, char *event_name, char *var_name)
3522 {
3523         struct trace_event_file *file = target_hist_data->event_file;
3524
3525         if (subsys_name) {
3526                 struct trace_event_call *call;
3527
3528                 if (!event_name)
3529                         return NULL;
3530
3531                 call = file->event_call;
3532
3533                 if (strcmp(subsys_name, call->class->system) != 0)
3534                         return NULL;
3535
3536                 if (strcmp(event_name, trace_event_name(call)) != 0)
3537                         return NULL;
3538         }
3539
3540         return create_field_var(target_hist_data, file, var_name);
3541 }
3542
3543 static bool check_track_val_max(u64 track_val, u64 var_val)
3544 {
3545         if (var_val <= track_val)
3546                 return false;
3547
3548         return true;
3549 }
3550
3551 static bool check_track_val_changed(u64 track_val, u64 var_val)
3552 {
3553         if (var_val == track_val)
3554                 return false;
3555
3556         return true;
3557 }
3558
3559 static u64 get_track_val(struct hist_trigger_data *hist_data,
3560                          struct tracing_map_elt *elt,
3561                          struct action_data *data)
3562 {
3563         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3564         u64 track_val;
3565
3566         track_val = tracing_map_read_var(elt, track_var_idx);
3567
3568         return track_val;
3569 }
3570
3571 static void save_track_val(struct hist_trigger_data *hist_data,
3572                            struct tracing_map_elt *elt,
3573                            struct action_data *data, u64 var_val)
3574 {
3575         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3576
3577         tracing_map_set_var(elt, track_var_idx, var_val);
3578 }
3579
3580 static void save_track_data(struct hist_trigger_data *hist_data,
3581                             struct tracing_map_elt *elt, void *rec,
3582                             struct ring_buffer_event *rbe, void *key,
3583                             struct action_data *data, u64 *var_ref_vals)
3584 {
3585         if (data->track_data.save_data)
3586                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3587 }
3588
3589 static bool check_track_val(struct tracing_map_elt *elt,
3590                             struct action_data *data,
3591                             u64 var_val)
3592 {
3593         struct hist_trigger_data *hist_data;
3594         u64 track_val;
3595
3596         hist_data = data->track_data.track_var->hist_data;
3597         track_val = get_track_val(hist_data, elt, data);
3598
3599         return data->track_data.check_val(track_val, var_val);
3600 }
3601
3602 #ifdef CONFIG_TRACER_SNAPSHOT
3603 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3604 {
3605         /* called with tr->max_lock held */
3606         struct track_data *track_data = tr->cond_snapshot->cond_data;
3607         struct hist_elt_data *elt_data, *track_elt_data;
3608         struct snapshot_context *context = cond_data;
3609         struct action_data *action;
3610         u64 track_val;
3611
3612         if (!track_data)
3613                 return false;
3614
3615         action = track_data->action_data;
3616
3617         track_val = get_track_val(track_data->hist_data, context->elt,
3618                                   track_data->action_data);
3619
3620         if (!action->track_data.check_val(track_data->track_val, track_val))
3621                 return false;
3622
3623         track_data->track_val = track_val;
3624         memcpy(track_data->key, context->key, track_data->key_len);
3625
3626         elt_data = context->elt->private_data;
3627         track_elt_data = track_data->elt.private_data;
3628         if (elt_data->comm)
3629                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3630
3631         track_data->updated = true;
3632
3633         return true;
3634 }
3635
3636 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3637                                      struct tracing_map_elt *elt, void *rec,
3638                                      struct ring_buffer_event *rbe, void *key,
3639                                      struct action_data *data,
3640                                      u64 *var_ref_vals)
3641 {
3642         struct trace_event_file *file = hist_data->event_file;
3643         struct snapshot_context context;
3644
3645         context.elt = elt;
3646         context.key = key;
3647
3648         tracing_snapshot_cond(file->tr, &context);
3649 }
3650
3651 static void hist_trigger_print_key(struct seq_file *m,
3652                                    struct hist_trigger_data *hist_data,
3653                                    void *key,
3654                                    struct tracing_map_elt *elt);
3655
3656 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3657 {
3658         unsigned int i;
3659
3660         if (!hist_data->n_actions)
3661                 return NULL;
3662
3663         for (i = 0; i < hist_data->n_actions; i++) {
3664                 struct action_data *data = hist_data->actions[i];
3665
3666                 if (data->action == ACTION_SNAPSHOT)
3667                         return data;
3668         }
3669
3670         return NULL;
3671 }
3672
3673 static void track_data_snapshot_print(struct seq_file *m,
3674                                       struct hist_trigger_data *hist_data)
3675 {
3676         struct trace_event_file *file = hist_data->event_file;
3677         struct track_data *track_data;
3678         struct action_data *action;
3679
3680         track_data = tracing_cond_snapshot_data(file->tr);
3681         if (!track_data)
3682                 return;
3683
3684         if (!track_data->updated)
3685                 return;
3686
3687         action = snapshot_action(hist_data);
3688         if (!action)
3689                 return;
3690
3691         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3692         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3693                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3694                    action->track_data.var_str, track_data->track_val);
3695
3696         seq_puts(m, "\ttriggered by event with key: ");
3697         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3698         seq_putc(m, '\n');
3699 }
3700 #else
3701 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3702 {
3703         return false;
3704 }
3705 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3706                                      struct tracing_map_elt *elt, void *rec,
3707                                      struct ring_buffer_event *rbe, void *key,
3708                                      struct action_data *data,
3709                                      u64 *var_ref_vals) {}
3710 static void track_data_snapshot_print(struct seq_file *m,
3711                                       struct hist_trigger_data *hist_data) {}
3712 #endif /* CONFIG_TRACER_SNAPSHOT */
3713
3714 static void track_data_print(struct seq_file *m,
3715                              struct hist_trigger_data *hist_data,
3716                              struct tracing_map_elt *elt,
3717                              struct action_data *data)
3718 {
3719         u64 track_val = get_track_val(hist_data, elt, data);
3720         unsigned int i, save_var_idx;
3721
3722         if (data->handler == HANDLER_ONMAX)
3723                 seq_printf(m, "\n\tmax: %10llu", track_val);
3724         else if (data->handler == HANDLER_ONCHANGE)
3725                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3726
3727         if (data->action == ACTION_SNAPSHOT)
3728                 return;
3729
3730         for (i = 0; i < hist_data->n_save_vars; i++) {
3731                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3732                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3733                 u64 val;
3734
3735                 save_var_idx = save_var->var.idx;
3736
3737                 val = tracing_map_read_var(elt, save_var_idx);
3738
3739                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3740                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3741                                    (char *)(uintptr_t)(val));
3742                 } else
3743                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3744         }
3745 }
3746
3747 static void ontrack_action(struct hist_trigger_data *hist_data,
3748                            struct tracing_map_elt *elt, void *rec,
3749                            struct ring_buffer_event *rbe, void *key,
3750                            struct action_data *data, u64 *var_ref_vals)
3751 {
3752         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3753
3754         if (check_track_val(elt, data, var_val)) {
3755                 save_track_val(hist_data, elt, data, var_val);
3756                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3757         }
3758 }
3759
3760 static void action_data_destroy(struct action_data *data)
3761 {
3762         unsigned int i;
3763
3764         lockdep_assert_held(&event_mutex);
3765
3766         kfree(data->action_name);
3767
3768         for (i = 0; i < data->n_params; i++)
3769                 kfree(data->params[i]);
3770
3771         if (data->synth_event)
3772                 data->synth_event->ref--;
3773
3774         kfree(data->synth_event_name);
3775
3776         kfree(data);
3777 }
3778
3779 static void track_data_destroy(struct hist_trigger_data *hist_data,
3780                                struct action_data *data)
3781 {
3782         struct trace_event_file *file = hist_data->event_file;
3783
3784         destroy_hist_field(data->track_data.track_var, 0);
3785
3786         if (data->action == ACTION_SNAPSHOT) {
3787                 struct track_data *track_data;
3788
3789                 track_data = tracing_cond_snapshot_data(file->tr);
3790                 if (track_data && track_data->hist_data == hist_data) {
3791                         tracing_snapshot_cond_disable(file->tr);
3792                         track_data_free(track_data);
3793                 }
3794         }
3795
3796         kfree(data->track_data.var_str);
3797
3798         action_data_destroy(data);
3799 }
3800
3801 static int action_create(struct hist_trigger_data *hist_data,
3802                          struct action_data *data);
3803
3804 static int track_data_create(struct hist_trigger_data *hist_data,
3805                              struct action_data *data)
3806 {
3807         struct hist_field *var_field, *ref_field, *track_var = NULL;
3808         struct trace_event_file *file = hist_data->event_file;
3809         struct trace_array *tr = file->tr;
3810         char *track_data_var_str;
3811         int ret = 0;
3812
3813         track_data_var_str = data->track_data.var_str;
3814         if (track_data_var_str[0] != '$') {
3815                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3816                 return -EINVAL;
3817         }
3818         track_data_var_str++;
3819
3820         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3821         if (!var_field) {
3822                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3823                 return -EINVAL;
3824         }
3825
3826         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3827         if (!ref_field)
3828                 return -ENOMEM;
3829
3830         data->track_data.var_ref = ref_field;
3831
3832         if (data->handler == HANDLER_ONMAX)
3833                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3834         if (IS_ERR(track_var)) {
3835                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3836                 ret = PTR_ERR(track_var);
3837                 goto out;
3838         }
3839
3840         if (data->handler == HANDLER_ONCHANGE)
3841                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3842         if (IS_ERR(track_var)) {
3843                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3844                 ret = PTR_ERR(track_var);
3845                 goto out;
3846         }
3847         data->track_data.track_var = track_var;
3848
3849         ret = action_create(hist_data, data);
3850  out:
3851         return ret;
3852 }
3853
3854 static int parse_action_params(struct trace_array *tr, char *params,
3855                                struct action_data *data)
3856 {
3857         char *param, *saved_param;
3858         bool first_param = true;
3859         int ret = 0;
3860
3861         while (params) {
3862                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3863                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3864                         goto out;
3865                 }
3866
3867                 param = strsep(&params, ",");
3868                 if (!param) {
3869                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3870                         ret = -EINVAL;
3871                         goto out;
3872                 }
3873
3874                 param = strstrip(param);
3875                 if (strlen(param) < 2) {
3876                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3877                         ret = -EINVAL;
3878                         goto out;
3879                 }
3880
3881                 saved_param = kstrdup(param, GFP_KERNEL);
3882                 if (!saved_param) {
3883                         ret = -ENOMEM;
3884                         goto out;
3885                 }
3886
3887                 if (first_param && data->use_trace_keyword) {
3888                         data->synth_event_name = saved_param;
3889                         first_param = false;
3890                         continue;
3891                 }
3892                 first_param = false;
3893
3894                 data->params[data->n_params++] = saved_param;
3895         }
3896  out:
3897         return ret;
3898 }
3899
3900 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3901                         enum handler_id handler)
3902 {
3903         char *action_name;
3904         int ret = 0;
3905
3906         strsep(&str, ".");
3907         if (!str) {
3908                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3909                 ret = -EINVAL;
3910                 goto out;
3911         }
3912
3913         action_name = strsep(&str, "(");
3914         if (!action_name || !str) {
3915                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3916                 ret = -EINVAL;
3917                 goto out;
3918         }
3919
3920         if (str_has_prefix(action_name, "save")) {
3921                 char *params = strsep(&str, ")");
3922
3923                 if (!params) {
3924                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3925                         ret = -EINVAL;
3926                         goto out;
3927                 }
3928
3929                 ret = parse_action_params(tr, params, data);
3930                 if (ret)
3931                         goto out;
3932
3933                 if (handler == HANDLER_ONMAX)
3934                         data->track_data.check_val = check_track_val_max;
3935                 else if (handler == HANDLER_ONCHANGE)
3936                         data->track_data.check_val = check_track_val_changed;
3937                 else {
3938                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3939                         ret = -EINVAL;
3940                         goto out;
3941                 }
3942
3943                 data->track_data.save_data = save_track_data_vars;
3944                 data->fn = ontrack_action;
3945                 data->action = ACTION_SAVE;
3946         } else if (str_has_prefix(action_name, "snapshot")) {
3947                 char *params = strsep(&str, ")");
3948
3949                 if (!str) {
3950                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3951                         ret = -EINVAL;
3952                         goto out;
3953                 }
3954
3955                 if (handler == HANDLER_ONMAX)
3956                         data->track_data.check_val = check_track_val_max;
3957                 else if (handler == HANDLER_ONCHANGE)
3958                         data->track_data.check_val = check_track_val_changed;
3959                 else {
3960                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3961                         ret = -EINVAL;
3962                         goto out;
3963                 }
3964
3965                 data->track_data.save_data = save_track_data_snapshot;
3966                 data->fn = ontrack_action;
3967                 data->action = ACTION_SNAPSHOT;
3968         } else {
3969                 char *params = strsep(&str, ")");
3970
3971                 if (str_has_prefix(action_name, "trace"))
3972                         data->use_trace_keyword = true;
3973
3974                 if (params) {
3975                         ret = parse_action_params(tr, params, data);
3976                         if (ret)
3977                                 goto out;
3978                 }
3979
3980                 if (handler == HANDLER_ONMAX)
3981                         data->track_data.check_val = check_track_val_max;
3982                 else if (handler == HANDLER_ONCHANGE)
3983                         data->track_data.check_val = check_track_val_changed;
3984
3985                 if (handler != HANDLER_ONMATCH) {
3986                         data->track_data.save_data = action_trace;
3987                         data->fn = ontrack_action;
3988                 } else
3989                         data->fn = action_trace;
3990
3991                 data->action = ACTION_TRACE;
3992         }
3993
3994         data->action_name = kstrdup(action_name, GFP_KERNEL);
3995         if (!data->action_name) {
3996                 ret = -ENOMEM;
3997                 goto out;
3998         }
3999
4000         data->handler = handler;
4001  out:
4002         return ret;
4003 }
4004
4005 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4006                                             char *str, enum handler_id handler)
4007 {
4008         struct action_data *data;
4009         int ret = -EINVAL;
4010         char *var_str;
4011
4012         data = kzalloc(sizeof(*data), GFP_KERNEL);
4013         if (!data)
4014                 return ERR_PTR(-ENOMEM);
4015
4016         var_str = strsep(&str, ")");
4017         if (!var_str || !str) {
4018                 ret = -EINVAL;
4019                 goto free;
4020         }
4021
4022         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4023         if (!data->track_data.var_str) {
4024                 ret = -ENOMEM;
4025                 goto free;
4026         }
4027
4028         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4029         if (ret)
4030                 goto free;
4031  out:
4032         return data;
4033  free:
4034         track_data_destroy(hist_data, data);
4035         data = ERR_PTR(ret);
4036         goto out;
4037 }
4038
4039 static void onmatch_destroy(struct action_data *data)
4040 {
4041         kfree(data->match_data.event);
4042         kfree(data->match_data.event_system);
4043
4044         action_data_destroy(data);
4045 }
4046
4047 static void destroy_field_var(struct field_var *field_var)
4048 {
4049         if (!field_var)
4050                 return;
4051
4052         destroy_hist_field(field_var->var, 0);
4053         destroy_hist_field(field_var->val, 0);
4054
4055         kfree(field_var);
4056 }
4057
4058 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4059 {
4060         unsigned int i;
4061
4062         for (i = 0; i < hist_data->n_field_vars; i++)
4063                 destroy_field_var(hist_data->field_vars[i]);
4064 }
4065
4066 static void save_field_var(struct hist_trigger_data *hist_data,
4067                            struct field_var *field_var)
4068 {
4069         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4070
4071         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4072                 hist_data->n_field_var_str++;
4073 }
4074
4075
4076 static int check_synth_field(struct synth_event *event,
4077                              struct hist_field *hist_field,
4078                              unsigned int field_pos)
4079 {
4080         struct synth_field *field;
4081
4082         if (field_pos >= event->n_fields)
4083                 return -EINVAL;
4084
4085         field = event->fields[field_pos];
4086
4087         if (strcmp(field->type, hist_field->type) != 0)
4088                 return -EINVAL;
4089
4090         return 0;
4091 }
4092
4093 static struct hist_field *
4094 trace_action_find_var(struct hist_trigger_data *hist_data,
4095                       struct action_data *data,
4096                       char *system, char *event, char *var)
4097 {
4098         struct trace_array *tr = hist_data->event_file->tr;
4099         struct hist_field *hist_field;
4100
4101         var++; /* skip '$' */
4102
4103         hist_field = find_target_event_var(hist_data, system, event, var);
4104         if (!hist_field) {
4105                 if (!system && data->handler == HANDLER_ONMATCH) {
4106                         system = data->match_data.event_system;
4107                         event = data->match_data.event;
4108                 }
4109
4110                 hist_field = find_event_var(hist_data, system, event, var);
4111         }
4112
4113         if (!hist_field)
4114                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4115
4116         return hist_field;
4117 }
4118
4119 static struct hist_field *
4120 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4121                               struct action_data *data, char *system,
4122                               char *event, char *var)
4123 {
4124         struct hist_field *hist_field = NULL;
4125         struct field_var *field_var;
4126
4127         /*
4128          * First try to create a field var on the target event (the
4129          * currently being defined).  This will create a variable for
4130          * unqualified fields on the target event, or if qualified,
4131          * target fields that have qualified names matching the target.
4132          */
4133         field_var = create_target_field_var(hist_data, system, event, var);
4134
4135         if (field_var && !IS_ERR(field_var)) {
4136                 save_field_var(hist_data, field_var);
4137                 hist_field = field_var->var;
4138         } else {
4139                 field_var = NULL;
4140                 /*
4141                  * If no explicit system.event is specfied, default to
4142                  * looking for fields on the onmatch(system.event.xxx)
4143                  * event.
4144                  */
4145                 if (!system && data->handler == HANDLER_ONMATCH) {
4146                         system = data->match_data.event_system;
4147                         event = data->match_data.event;
4148                 }
4149
4150                 /*
4151                  * At this point, we're looking at a field on another
4152                  * event.  Because we can't modify a hist trigger on
4153                  * another event to add a variable for a field, we need
4154                  * to create a new trigger on that event and create the
4155                  * variable at the same time.
4156                  */
4157                 hist_field = create_field_var_hist(hist_data, system, event, var);
4158                 if (IS_ERR(hist_field))
4159                         goto free;
4160         }
4161  out:
4162         return hist_field;
4163  free:
4164         destroy_field_var(field_var);
4165         hist_field = NULL;
4166         goto out;
4167 }
4168
4169 static int trace_action_create(struct hist_trigger_data *hist_data,
4170                                struct action_data *data)
4171 {
4172         struct trace_array *tr = hist_data->event_file->tr;
4173         char *event_name, *param, *system = NULL;
4174         struct hist_field *hist_field, *var_ref;
4175         unsigned int i, var_ref_idx;
4176         unsigned int field_pos = 0;
4177         struct synth_event *event;
4178         char *synth_event_name;
4179         int ret = 0;
4180
4181         lockdep_assert_held(&event_mutex);
4182
4183         if (data->use_trace_keyword)
4184                 synth_event_name = data->synth_event_name;
4185         else
4186                 synth_event_name = data->action_name;
4187
4188         event = find_synth_event(synth_event_name);
4189         if (!event) {
4190                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4191                 return -EINVAL;
4192         }
4193
4194         event->ref++;
4195
4196         var_ref_idx = hist_data->n_var_refs;
4197
4198         for (i = 0; i < data->n_params; i++) {
4199                 char *p;
4200
4201                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4202                 if (!param) {
4203                         ret = -ENOMEM;
4204                         goto err;
4205                 }
4206
4207                 system = strsep(&param, ".");
4208                 if (!param) {
4209                         param = (char *)system;
4210                         system = event_name = NULL;
4211                 } else {
4212                         event_name = strsep(&param, ".");
4213                         if (!param) {
4214                                 kfree(p);
4215                                 ret = -EINVAL;
4216                                 goto err;
4217                         }
4218                 }
4219
4220                 if (param[0] == '$')
4221                         hist_field = trace_action_find_var(hist_data, data,
4222                                                            system, event_name,
4223                                                            param);
4224                 else
4225                         hist_field = trace_action_create_field_var(hist_data,
4226                                                                    data,
4227                                                                    system,
4228                                                                    event_name,
4229                                                                    param);
4230
4231                 if (!hist_field) {
4232                         kfree(p);
4233                         ret = -EINVAL;
4234                         goto err;
4235                 }
4236
4237                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4238                         var_ref = create_var_ref(hist_data, hist_field,
4239                                                  system, event_name);
4240                         if (!var_ref) {
4241                                 kfree(p);
4242                                 ret = -ENOMEM;
4243                                 goto err;
4244                         }
4245
4246                         field_pos++;
4247                         kfree(p);
4248                         continue;
4249                 }
4250
4251                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4252                 kfree(p);
4253                 ret = -EINVAL;
4254                 goto err;
4255         }
4256
4257         if (field_pos != event->n_fields) {
4258                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4259                 ret = -EINVAL;
4260                 goto err;
4261         }
4262
4263         data->synth_event = event;
4264         data->var_ref_idx = var_ref_idx;
4265  out:
4266         return ret;
4267  err:
4268         event->ref--;
4269
4270         goto out;
4271 }
4272
4273 static int action_create(struct hist_trigger_data *hist_data,
4274                          struct action_data *data)
4275 {
4276         struct trace_event_file *file = hist_data->event_file;
4277         struct trace_array *tr = file->tr;
4278         struct track_data *track_data;
4279         struct field_var *field_var;
4280         unsigned int i;
4281         char *param;
4282         int ret = 0;
4283
4284         if (data->action == ACTION_TRACE)
4285                 return trace_action_create(hist_data, data);
4286
4287         if (data->action == ACTION_SNAPSHOT) {
4288                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4289                 if (IS_ERR(track_data)) {
4290                         ret = PTR_ERR(track_data);
4291                         goto out;
4292                 }
4293
4294                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4295                                                    cond_snapshot_update);
4296                 if (ret)
4297                         track_data_free(track_data);
4298
4299                 goto out;
4300         }
4301
4302         if (data->action == ACTION_SAVE) {
4303                 if (hist_data->n_save_vars) {
4304                         ret = -EEXIST;
4305                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4306                         goto out;
4307                 }
4308
4309                 for (i = 0; i < data->n_params; i++) {
4310                         param = kstrdup(data->params[i], GFP_KERNEL);
4311                         if (!param) {
4312                                 ret = -ENOMEM;
4313                                 goto out;
4314                         }
4315
4316                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4317                         if (IS_ERR(field_var)) {
4318                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4319                                          errpos(param));
4320                                 ret = PTR_ERR(field_var);
4321                                 kfree(param);
4322                                 goto out;
4323                         }
4324
4325                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4326                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4327                                 hist_data->n_save_var_str++;
4328                         kfree(param);
4329                 }
4330         }
4331  out:
4332         return ret;
4333 }
4334
4335 static int onmatch_create(struct hist_trigger_data *hist_data,
4336                           struct action_data *data)
4337 {
4338         return action_create(hist_data, data);
4339 }
4340
4341 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4342 {
4343         char *match_event, *match_event_system;
4344         struct action_data *data;
4345         int ret = -EINVAL;
4346
4347         data = kzalloc(sizeof(*data), GFP_KERNEL);
4348         if (!data)
4349                 return ERR_PTR(-ENOMEM);
4350
4351         match_event = strsep(&str, ")");
4352         if (!match_event || !str) {
4353                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4354                 goto free;
4355         }
4356
4357         match_event_system = strsep(&match_event, ".");
4358         if (!match_event) {
4359                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4360                 goto free;
4361         }
4362
4363         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4364                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4365                 goto free;
4366         }
4367
4368         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4369         if (!data->match_data.event) {
4370                 ret = -ENOMEM;
4371                 goto free;
4372         }
4373
4374         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4375         if (!data->match_data.event_system) {
4376                 ret = -ENOMEM;
4377                 goto free;
4378         }
4379
4380         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4381         if (ret)
4382                 goto free;
4383  out:
4384         return data;
4385  free:
4386         onmatch_destroy(data);
4387         data = ERR_PTR(ret);
4388         goto out;
4389 }
4390
4391 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4392 {
4393         hist_data->fields[HITCOUNT_IDX] =
4394                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4395         if (!hist_data->fields[HITCOUNT_IDX])
4396                 return -ENOMEM;
4397
4398         hist_data->n_vals++;
4399         hist_data->n_fields++;
4400
4401         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4402                 return -EINVAL;
4403
4404         return 0;
4405 }
4406
4407 static int __create_val_field(struct hist_trigger_data *hist_data,
4408                               unsigned int val_idx,
4409                               struct trace_event_file *file,
4410                               char *var_name, char *field_str,
4411                               unsigned long flags)
4412 {
4413         struct hist_field *hist_field;
4414         int ret = 0;
4415
4416         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4417         if (IS_ERR(hist_field)) {
4418                 ret = PTR_ERR(hist_field);
4419                 goto out;
4420         }
4421
4422         hist_data->fields[val_idx] = hist_field;
4423
4424         ++hist_data->n_vals;
4425         ++hist_data->n_fields;
4426
4427         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4428                 ret = -EINVAL;
4429  out:
4430         return ret;
4431 }
4432
4433 static int create_val_field(struct hist_trigger_data *hist_data,
4434                             unsigned int val_idx,
4435                             struct trace_event_file *file,
4436                             char *field_str)
4437 {
4438         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4439                 return -EINVAL;
4440
4441         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4442 }
4443
4444 static int create_var_field(struct hist_trigger_data *hist_data,
4445                             unsigned int val_idx,
4446                             struct trace_event_file *file,
4447                             char *var_name, char *expr_str)
4448 {
4449         struct trace_array *tr = hist_data->event_file->tr;
4450         unsigned long flags = 0;
4451
4452         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4453                 return -EINVAL;
4454
4455         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4456                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4457                 return -EINVAL;
4458         }
4459
4460         flags |= HIST_FIELD_FL_VAR;
4461         hist_data->n_vars++;
4462         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4463                 return -EINVAL;
4464
4465         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4466 }
4467
4468 static int create_val_fields(struct hist_trigger_data *hist_data,
4469                              struct trace_event_file *file)
4470 {
4471         char *fields_str, *field_str;
4472         unsigned int i, j = 1;
4473         int ret;
4474
4475         ret = create_hitcount_val(hist_data);
4476         if (ret)
4477                 goto out;
4478
4479         fields_str = hist_data->attrs->vals_str;
4480         if (!fields_str)
4481                 goto out;
4482
4483         strsep(&fields_str, "=");
4484         if (!fields_str)
4485                 goto out;
4486
4487         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4488                      j < TRACING_MAP_VALS_MAX; i++) {
4489                 field_str = strsep(&fields_str, ",");
4490                 if (!field_str)
4491                         break;
4492
4493                 if (strcmp(field_str, "hitcount") == 0)
4494                         continue;
4495
4496                 ret = create_val_field(hist_data, j++, file, field_str);
4497                 if (ret)
4498                         goto out;
4499         }
4500
4501         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4502                 ret = -EINVAL;
4503  out:
4504         return ret;
4505 }
4506
4507 static int create_key_field(struct hist_trigger_data *hist_data,
4508                             unsigned int key_idx,
4509                             unsigned int key_offset,
4510                             struct trace_event_file *file,
4511                             char *field_str)
4512 {
4513         struct trace_array *tr = hist_data->event_file->tr;
4514         struct hist_field *hist_field = NULL;
4515         unsigned long flags = 0;
4516         unsigned int key_size;
4517         int ret = 0;
4518
4519         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4520                 return -EINVAL;
4521
4522         flags |= HIST_FIELD_FL_KEY;
4523
4524         if (strcmp(field_str, "stacktrace") == 0) {
4525                 flags |= HIST_FIELD_FL_STACKTRACE;
4526                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4527                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4528         } else {
4529                 hist_field = parse_expr(hist_data, file, field_str, flags,
4530                                         NULL, 0);
4531                 if (IS_ERR(hist_field)) {
4532                         ret = PTR_ERR(hist_field);
4533                         goto out;
4534                 }
4535
4536                 if (field_has_hist_vars(hist_field, 0)) {
4537                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4538                         destroy_hist_field(hist_field, 0);
4539                         ret = -EINVAL;
4540                         goto out;
4541                 }
4542
4543                 key_size = hist_field->size;
4544         }
4545
4546         hist_data->fields[key_idx] = hist_field;
4547
4548         key_size = ALIGN(key_size, sizeof(u64));
4549         hist_data->fields[key_idx]->size = key_size;
4550         hist_data->fields[key_idx]->offset = key_offset;
4551
4552         hist_data->key_size += key_size;
4553
4554         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4555                 ret = -EINVAL;
4556                 goto out;
4557         }
4558
4559         hist_data->n_keys++;
4560         hist_data->n_fields++;
4561
4562         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4563                 return -EINVAL;
4564
4565         ret = key_size;
4566  out:
4567         return ret;
4568 }
4569
4570 static int create_key_fields(struct hist_trigger_data *hist_data,
4571                              struct trace_event_file *file)
4572 {
4573         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4574         char *fields_str, *field_str;
4575         int ret = -EINVAL;
4576
4577         fields_str = hist_data->attrs->keys_str;
4578         if (!fields_str)
4579                 goto out;
4580
4581         strsep(&fields_str, "=");
4582         if (!fields_str)
4583                 goto out;
4584
4585         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4586                 field_str = strsep(&fields_str, ",");
4587                 if (!field_str)
4588                         break;
4589                 ret = create_key_field(hist_data, i, key_offset,
4590                                        file, field_str);
4591                 if (ret < 0)
4592                         goto out;
4593                 key_offset += ret;
4594         }
4595         if (fields_str) {
4596                 ret = -EINVAL;
4597                 goto out;
4598         }
4599         ret = 0;
4600  out:
4601         return ret;
4602 }
4603
4604 static int create_var_fields(struct hist_trigger_data *hist_data,
4605                              struct trace_event_file *file)
4606 {
4607         unsigned int i, j = hist_data->n_vals;
4608         int ret = 0;
4609
4610         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4611
4612         for (i = 0; i < n_vars; i++) {
4613                 char *var_name = hist_data->attrs->var_defs.name[i];
4614                 char *expr = hist_data->attrs->var_defs.expr[i];
4615
4616                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4617                 if (ret)
4618                         goto out;
4619         }
4620  out:
4621         return ret;
4622 }
4623
4624 static void free_var_defs(struct hist_trigger_data *hist_data)
4625 {
4626         unsigned int i;
4627
4628         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4629                 kfree(hist_data->attrs->var_defs.name[i]);
4630                 kfree(hist_data->attrs->var_defs.expr[i]);
4631         }
4632
4633         hist_data->attrs->var_defs.n_vars = 0;
4634 }
4635
4636 static int parse_var_defs(struct hist_trigger_data *hist_data)
4637 {
4638         struct trace_array *tr = hist_data->event_file->tr;
4639         char *s, *str, *var_name, *field_str;
4640         unsigned int i, j, n_vars = 0;
4641         int ret = 0;
4642
4643         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4644                 str = hist_data->attrs->assignment_str[i];
4645                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4646                         field_str = strsep(&str, ",");
4647                         if (!field_str)
4648                                 break;
4649
4650                         var_name = strsep(&field_str, "=");
4651                         if (!var_name || !field_str) {
4652                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4653                                          errpos(var_name));
4654                                 ret = -EINVAL;
4655                                 goto free;
4656                         }
4657
4658                         if (n_vars == TRACING_MAP_VARS_MAX) {
4659                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4660                                 ret = -EINVAL;
4661                                 goto free;
4662                         }
4663
4664                         s = kstrdup(var_name, GFP_KERNEL);
4665                         if (!s) {
4666                                 ret = -ENOMEM;
4667                                 goto free;
4668                         }
4669                         hist_data->attrs->var_defs.name[n_vars] = s;
4670
4671                         s = kstrdup(field_str, GFP_KERNEL);
4672                         if (!s) {
4673                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4674                                 ret = -ENOMEM;
4675                                 goto free;
4676                         }
4677                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4678
4679                         hist_data->attrs->var_defs.n_vars = n_vars;
4680                 }
4681         }
4682
4683         return ret;
4684  free:
4685         free_var_defs(hist_data);
4686
4687         return ret;
4688 }
4689
4690 static int create_hist_fields(struct hist_trigger_data *hist_data,
4691                               struct trace_event_file *file)
4692 {
4693         int ret;
4694
4695         ret = parse_var_defs(hist_data);
4696         if (ret)
4697                 goto out;
4698
4699         ret = create_val_fields(hist_data, file);
4700         if (ret)
4701                 goto out;
4702
4703         ret = create_var_fields(hist_data, file);
4704         if (ret)
4705                 goto out;
4706
4707         ret = create_key_fields(hist_data, file);
4708         if (ret)
4709                 goto out;
4710  out:
4711         free_var_defs(hist_data);
4712
4713         return ret;
4714 }
4715
4716 static int is_descending(const char *str)
4717 {
4718         if (!str)
4719                 return 0;
4720
4721         if (strcmp(str, "descending") == 0)
4722                 return 1;
4723
4724         if (strcmp(str, "ascending") == 0)
4725                 return 0;
4726
4727         return -EINVAL;
4728 }
4729
4730 static int create_sort_keys(struct hist_trigger_data *hist_data)
4731 {
4732         char *fields_str = hist_data->attrs->sort_key_str;
4733         struct tracing_map_sort_key *sort_key;
4734         int descending, ret = 0;
4735         unsigned int i, j, k;
4736
4737         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4738
4739         if (!fields_str)
4740                 goto out;
4741
4742         strsep(&fields_str, "=");
4743         if (!fields_str) {
4744                 ret = -EINVAL;
4745                 goto out;
4746         }
4747
4748         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4749                 struct hist_field *hist_field;
4750                 char *field_str, *field_name;
4751                 const char *test_name;
4752
4753                 sort_key = &hist_data->sort_keys[i];
4754
4755                 field_str = strsep(&fields_str, ",");
4756                 if (!field_str) {
4757                         if (i == 0)
4758                                 ret = -EINVAL;
4759                         break;
4760                 }
4761
4762                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4763                         ret = -EINVAL;
4764                         break;
4765                 }
4766
4767                 field_name = strsep(&field_str, ".");
4768                 if (!field_name) {
4769                         ret = -EINVAL;
4770                         break;
4771                 }
4772
4773                 if (strcmp(field_name, "hitcount") == 0) {
4774                         descending = is_descending(field_str);
4775                         if (descending < 0) {
4776                                 ret = descending;
4777                                 break;
4778                         }
4779                         sort_key->descending = descending;
4780                         continue;
4781                 }
4782
4783                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4784                         unsigned int idx;
4785
4786                         hist_field = hist_data->fields[j];
4787                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4788                                 continue;
4789
4790                         idx = k++;
4791
4792                         test_name = hist_field_name(hist_field, 0);
4793
4794                         if (strcmp(field_name, test_name) == 0) {
4795                                 sort_key->field_idx = idx;
4796                                 descending = is_descending(field_str);
4797                                 if (descending < 0) {
4798                                         ret = descending;
4799                                         goto out;
4800                                 }
4801                                 sort_key->descending = descending;
4802                                 break;
4803                         }
4804                 }
4805                 if (j == hist_data->n_fields) {
4806                         ret = -EINVAL;
4807                         break;
4808                 }
4809         }
4810
4811         hist_data->n_sort_keys = i;
4812  out:
4813         return ret;
4814 }
4815
4816 static void destroy_actions(struct hist_trigger_data *hist_data)
4817 {
4818         unsigned int i;
4819
4820         for (i = 0; i < hist_data->n_actions; i++) {
4821                 struct action_data *data = hist_data->actions[i];
4822
4823                 if (data->handler == HANDLER_ONMATCH)
4824                         onmatch_destroy(data);
4825                 else if (data->handler == HANDLER_ONMAX ||
4826                          data->handler == HANDLER_ONCHANGE)
4827                         track_data_destroy(hist_data, data);
4828                 else
4829                         kfree(data);
4830         }
4831 }
4832
4833 static int parse_actions(struct hist_trigger_data *hist_data)
4834 {
4835         struct trace_array *tr = hist_data->event_file->tr;
4836         struct action_data *data;
4837         unsigned int i;
4838         int ret = 0;
4839         char *str;
4840         int len;
4841
4842         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4843                 str = hist_data->attrs->action_str[i];
4844
4845                 if ((len = str_has_prefix(str, "onmatch("))) {
4846                         char *action_str = str + len;
4847
4848                         data = onmatch_parse(tr, action_str);
4849                         if (IS_ERR(data)) {
4850                                 ret = PTR_ERR(data);
4851                                 break;
4852                         }
4853                 } else if ((len = str_has_prefix(str, "onmax("))) {
4854                         char *action_str = str + len;
4855
4856                         data = track_data_parse(hist_data, action_str,
4857                                                 HANDLER_ONMAX);
4858                         if (IS_ERR(data)) {
4859                                 ret = PTR_ERR(data);
4860                                 break;
4861                         }
4862                 } else if ((len = str_has_prefix(str, "onchange("))) {
4863                         char *action_str = str + len;
4864
4865                         data = track_data_parse(hist_data, action_str,
4866                                                 HANDLER_ONCHANGE);
4867                         if (IS_ERR(data)) {
4868                                 ret = PTR_ERR(data);
4869                                 break;
4870                         }
4871                 } else {
4872                         ret = -EINVAL;
4873                         break;
4874                 }
4875
4876                 hist_data->actions[hist_data->n_actions++] = data;
4877         }
4878
4879         return ret;
4880 }
4881
4882 static int create_actions(struct hist_trigger_data *hist_data)
4883 {
4884         struct action_data *data;
4885         unsigned int i;
4886         int ret = 0;
4887
4888         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4889                 data = hist_data->actions[i];
4890
4891                 if (data->handler == HANDLER_ONMATCH) {
4892                         ret = onmatch_create(hist_data, data);
4893                         if (ret)
4894                                 break;
4895                 } else if (data->handler == HANDLER_ONMAX ||
4896                            data->handler == HANDLER_ONCHANGE) {
4897                         ret = track_data_create(hist_data, data);
4898                         if (ret)
4899                                 break;
4900                 } else {
4901                         ret = -EINVAL;
4902                         break;
4903                 }
4904         }
4905
4906         return ret;
4907 }
4908
4909 static void print_actions(struct seq_file *m,
4910                           struct hist_trigger_data *hist_data,
4911                           struct tracing_map_elt *elt)
4912 {
4913         unsigned int i;
4914
4915         for (i = 0; i < hist_data->n_actions; i++) {
4916                 struct action_data *data = hist_data->actions[i];
4917
4918                 if (data->action == ACTION_SNAPSHOT)
4919                         continue;
4920
4921                 if (data->handler == HANDLER_ONMAX ||
4922                     data->handler == HANDLER_ONCHANGE)
4923                         track_data_print(m, hist_data, elt, data);
4924         }
4925 }
4926
4927 static void print_action_spec(struct seq_file *m,
4928                               struct hist_trigger_data *hist_data,
4929                               struct action_data *data)
4930 {
4931         unsigned int i;
4932
4933         if (data->action == ACTION_SAVE) {
4934                 for (i = 0; i < hist_data->n_save_vars; i++) {
4935                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4936                         if (i < hist_data->n_save_vars - 1)
4937                                 seq_puts(m, ",");
4938                 }
4939         } else if (data->action == ACTION_TRACE) {
4940                 if (data->use_trace_keyword)
4941                         seq_printf(m, "%s", data->synth_event_name);
4942                 for (i = 0; i < data->n_params; i++) {
4943                         if (i || data->use_trace_keyword)
4944                                 seq_puts(m, ",");
4945                         seq_printf(m, "%s", data->params[i]);
4946                 }
4947         }
4948 }
4949
4950 static void print_track_data_spec(struct seq_file *m,
4951                                   struct hist_trigger_data *hist_data,
4952                                   struct action_data *data)
4953 {
4954         if (data->handler == HANDLER_ONMAX)
4955                 seq_puts(m, ":onmax(");
4956         else if (data->handler == HANDLER_ONCHANGE)
4957                 seq_puts(m, ":onchange(");
4958         seq_printf(m, "%s", data->track_data.var_str);
4959         seq_printf(m, ").%s(", data->action_name);
4960
4961         print_action_spec(m, hist_data, data);
4962
4963         seq_puts(m, ")");
4964 }
4965
4966 static void print_onmatch_spec(struct seq_file *m,
4967                                struct hist_trigger_data *hist_data,
4968                                struct action_data *data)
4969 {
4970         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4971                    data->match_data.event);
4972
4973         seq_printf(m, "%s(", data->action_name);
4974
4975         print_action_spec(m, hist_data, data);
4976
4977         seq_puts(m, ")");
4978 }
4979
4980 static bool actions_match(struct hist_trigger_data *hist_data,
4981                           struct hist_trigger_data *hist_data_test)
4982 {
4983         unsigned int i, j;
4984
4985         if (hist_data->n_actions != hist_data_test->n_actions)
4986                 return false;
4987
4988         for (i = 0; i < hist_data->n_actions; i++) {
4989                 struct action_data *data = hist_data->actions[i];
4990                 struct action_data *data_test = hist_data_test->actions[i];
4991                 char *action_name, *action_name_test;
4992
4993                 if (data->handler != data_test->handler)
4994                         return false;
4995                 if (data->action != data_test->action)
4996                         return false;
4997
4998                 if (data->n_params != data_test->n_params)
4999                         return false;
5000
5001                 for (j = 0; j < data->n_params; j++) {
5002                         if (strcmp(data->params[j], data_test->params[j]) != 0)
5003                                 return false;
5004                 }
5005
5006                 if (data->use_trace_keyword)
5007                         action_name = data->synth_event_name;
5008                 else
5009                         action_name = data->action_name;
5010
5011                 if (data_test->use_trace_keyword)
5012                         action_name_test = data_test->synth_event_name;
5013                 else
5014                         action_name_test = data_test->action_name;
5015
5016                 if (strcmp(action_name, action_name_test) != 0)
5017                         return false;
5018
5019                 if (data->handler == HANDLER_ONMATCH) {
5020                         if (strcmp(data->match_data.event_system,
5021                                    data_test->match_data.event_system) != 0)
5022                                 return false;
5023                         if (strcmp(data->match_data.event,
5024                                    data_test->match_data.event) != 0)
5025                                 return false;
5026                 } else if (data->handler == HANDLER_ONMAX ||
5027                            data->handler == HANDLER_ONCHANGE) {
5028                         if (strcmp(data->track_data.var_str,
5029                                    data_test->track_data.var_str) != 0)
5030                                 return false;
5031                 }
5032         }
5033
5034         return true;
5035 }
5036
5037
5038 static void print_actions_spec(struct seq_file *m,
5039                                struct hist_trigger_data *hist_data)
5040 {
5041         unsigned int i;
5042
5043         for (i = 0; i < hist_data->n_actions; i++) {
5044                 struct action_data *data = hist_data->actions[i];
5045
5046                 if (data->handler == HANDLER_ONMATCH)
5047                         print_onmatch_spec(m, hist_data, data);
5048                 else if (data->handler == HANDLER_ONMAX ||
5049                          data->handler == HANDLER_ONCHANGE)
5050                         print_track_data_spec(m, hist_data, data);
5051         }
5052 }
5053
5054 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5055 {
5056         unsigned int i;
5057
5058         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5059                 kfree(hist_data->field_var_hists[i]->cmd);
5060                 kfree(hist_data->field_var_hists[i]);
5061         }
5062 }
5063
5064 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5065 {
5066         if (!hist_data)
5067                 return;
5068
5069         destroy_hist_trigger_attrs(hist_data->attrs);
5070         destroy_hist_fields(hist_data);
5071         tracing_map_destroy(hist_data->map);
5072
5073         destroy_actions(hist_data);
5074         destroy_field_vars(hist_data);
5075         destroy_field_var_hists(hist_data);
5076
5077         kfree(hist_data);
5078 }
5079
5080 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5081 {
5082         struct tracing_map *map = hist_data->map;
5083         struct ftrace_event_field *field;
5084         struct hist_field *hist_field;
5085         int i, idx = 0;
5086
5087         for_each_hist_field(i, hist_data) {
5088                 hist_field = hist_data->fields[i];
5089                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5090                         tracing_map_cmp_fn_t cmp_fn;
5091
5092                         field = hist_field->field;
5093
5094                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5095                                 cmp_fn = tracing_map_cmp_none;
5096                         else if (!field)
5097                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5098                                                              hist_field->is_signed);
5099                         else if (is_string_field(field))
5100                                 cmp_fn = tracing_map_cmp_string;
5101                         else
5102                                 cmp_fn = tracing_map_cmp_num(field->size,
5103                                                              field->is_signed);
5104                         idx = tracing_map_add_key_field(map,
5105                                                         hist_field->offset,
5106                                                         cmp_fn);
5107                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5108                         idx = tracing_map_add_sum_field(map);
5109
5110                 if (idx < 0)
5111                         return idx;
5112
5113                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5114                         idx = tracing_map_add_var(map);
5115                         if (idx < 0)
5116                                 return idx;
5117                         hist_field->var.idx = idx;
5118                         hist_field->var.hist_data = hist_data;
5119                 }
5120         }
5121
5122         return 0;
5123 }
5124
5125 static struct hist_trigger_data *
5126 create_hist_data(unsigned int map_bits,
5127                  struct hist_trigger_attrs *attrs,
5128                  struct trace_event_file *file,
5129                  bool remove)
5130 {
5131         const struct tracing_map_ops *map_ops = NULL;
5132         struct hist_trigger_data *hist_data;
5133         int ret = 0;
5134
5135         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5136         if (!hist_data)
5137                 return ERR_PTR(-ENOMEM);
5138
5139         hist_data->attrs = attrs;
5140         hist_data->remove = remove;
5141         hist_data->event_file = file;
5142
5143         ret = parse_actions(hist_data);
5144         if (ret)
5145                 goto free;
5146
5147         ret = create_hist_fields(hist_data, file);
5148         if (ret)
5149                 goto free;
5150
5151         ret = create_sort_keys(hist_data);
5152         if (ret)
5153                 goto free;
5154
5155         map_ops = &hist_trigger_elt_data_ops;
5156
5157         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5158                                             map_ops, hist_data);
5159         if (IS_ERR(hist_data->map)) {
5160                 ret = PTR_ERR(hist_data->map);
5161                 hist_data->map = NULL;
5162                 goto free;
5163         }
5164
5165         ret = create_tracing_map_fields(hist_data);
5166         if (ret)
5167                 goto free;
5168  out:
5169         return hist_data;
5170  free:
5171         hist_data->attrs = NULL;
5172
5173         destroy_hist_data(hist_data);
5174
5175         hist_data = ERR_PTR(ret);
5176
5177         goto out;
5178 }
5179
5180 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5181                                     struct tracing_map_elt *elt, void *rec,
5182                                     struct ring_buffer_event *rbe,
5183                                     u64 *var_ref_vals)
5184 {
5185         struct hist_elt_data *elt_data;
5186         struct hist_field *hist_field;
5187         unsigned int i, var_idx;
5188         u64 hist_val;
5189
5190         elt_data = elt->private_data;
5191         elt_data->var_ref_vals = var_ref_vals;
5192
5193         for_each_hist_val_field(i, hist_data) {
5194                 hist_field = hist_data->fields[i];
5195                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5196                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5197                         var_idx = hist_field->var.idx;
5198                         tracing_map_set_var(elt, var_idx, hist_val);
5199                         continue;
5200                 }
5201                 tracing_map_update_sum(elt, i, hist_val);
5202         }
5203
5204         for_each_hist_key_field(i, hist_data) {
5205                 hist_field = hist_data->fields[i];
5206                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5207                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5208                         var_idx = hist_field->var.idx;
5209                         tracing_map_set_var(elt, var_idx, hist_val);
5210                 }
5211         }
5212
5213         update_field_vars(hist_data, elt, rbe, rec);
5214 }
5215
5216 static inline void add_to_key(char *compound_key, void *key,
5217                               struct hist_field *key_field, void *rec)
5218 {
5219         size_t size = key_field->size;
5220
5221         if (key_field->flags & HIST_FIELD_FL_STRING) {
5222                 struct ftrace_event_field *field;
5223
5224                 field = key_field->field;
5225                 if (field->filter_type == FILTER_DYN_STRING)
5226                         size = *(u32 *)(rec + field->offset) >> 16;
5227                 else if (field->filter_type == FILTER_PTR_STRING)
5228                         size = strlen(key);
5229                 else if (field->filter_type == FILTER_STATIC_STRING)
5230                         size = field->size;
5231
5232                 /* ensure NULL-termination */
5233                 if (size > key_field->size - 1)
5234                         size = key_field->size - 1;
5235
5236                 strncpy(compound_key + key_field->offset, (char *)key, size);
5237         } else
5238                 memcpy(compound_key + key_field->offset, key, size);
5239 }
5240
5241 static void
5242 hist_trigger_actions(struct hist_trigger_data *hist_data,
5243                      struct tracing_map_elt *elt, void *rec,
5244                      struct ring_buffer_event *rbe, void *key,
5245                      u64 *var_ref_vals)
5246 {
5247         struct action_data *data;
5248         unsigned int i;
5249
5250         for (i = 0; i < hist_data->n_actions; i++) {
5251                 data = hist_data->actions[i];
5252                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5253         }
5254 }
5255
5256 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5257                                struct ring_buffer_event *rbe)
5258 {
5259         struct hist_trigger_data *hist_data = data->private_data;
5260         bool use_compound_key = (hist_data->n_keys > 1);
5261         unsigned long entries[HIST_STACKTRACE_DEPTH];
5262         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5263         char compound_key[HIST_KEY_SIZE_MAX];
5264         struct tracing_map_elt *elt = NULL;
5265         struct hist_field *key_field;
5266         u64 field_contents;
5267         void *key = NULL;
5268         unsigned int i;
5269
5270         memset(compound_key, 0, hist_data->key_size);
5271
5272         for_each_hist_key_field(i, hist_data) {
5273                 key_field = hist_data->fields[i];
5274
5275                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5276                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5277                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5278                                          HIST_STACKTRACE_SKIP);
5279                         key = entries;
5280                 } else {
5281                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5282                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5283                                 key = (void *)(unsigned long)field_contents;
5284                                 use_compound_key = true;
5285                         } else
5286                                 key = (void *)&field_contents;
5287                 }
5288
5289                 if (use_compound_key)
5290                         add_to_key(compound_key, key, key_field, rec);
5291         }
5292
5293         if (use_compound_key)
5294                 key = compound_key;
5295
5296         if (hist_data->n_var_refs &&
5297             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5298                 return;
5299
5300         elt = tracing_map_insert(hist_data->map, key);
5301         if (!elt)
5302                 return;
5303
5304         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5305
5306         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5307                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5308 }
5309
5310 static void hist_trigger_stacktrace_print(struct seq_file *m,
5311                                           unsigned long *stacktrace_entries,
5312                                           unsigned int max_entries)
5313 {
5314         char str[KSYM_SYMBOL_LEN];
5315         unsigned int spaces = 8;
5316         unsigned int i;
5317
5318         for (i = 0; i < max_entries; i++) {
5319                 if (!stacktrace_entries[i])
5320                         return;
5321
5322                 seq_printf(m, "%*c", 1 + spaces, ' ');
5323                 sprint_symbol(str, stacktrace_entries[i]);
5324                 seq_printf(m, "%s\n", str);
5325         }
5326 }
5327
5328 static void hist_trigger_print_key(struct seq_file *m,
5329                                    struct hist_trigger_data *hist_data,
5330                                    void *key,
5331                                    struct tracing_map_elt *elt)
5332 {
5333         struct hist_field *key_field;
5334         char str[KSYM_SYMBOL_LEN];
5335         bool multiline = false;
5336         const char *field_name;
5337         unsigned int i;
5338         u64 uval;
5339
5340         seq_puts(m, "{ ");
5341
5342         for_each_hist_key_field(i, hist_data) {
5343                 key_field = hist_data->fields[i];
5344
5345                 if (i > hist_data->n_vals)
5346                         seq_puts(m, ", ");
5347
5348                 field_name = hist_field_name(key_field, 0);
5349
5350                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5351                         uval = *(u64 *)(key + key_field->offset);
5352                         seq_printf(m, "%s: %llx", field_name, uval);
5353                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5354                         uval = *(u64 *)(key + key_field->offset);
5355                         sprint_symbol_no_offset(str, uval);
5356                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5357                                    uval, str);
5358                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5359                         uval = *(u64 *)(key + key_field->offset);
5360                         sprint_symbol(str, uval);
5361                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5362                                    uval, str);
5363                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5364                         struct hist_elt_data *elt_data = elt->private_data;
5365                         char *comm;
5366
5367                         if (WARN_ON_ONCE(!elt_data))
5368                                 return;
5369
5370                         comm = elt_data->comm;
5371
5372                         uval = *(u64 *)(key + key_field->offset);
5373                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5374                                    comm, uval);
5375                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5376                         const char *syscall_name;
5377
5378                         uval = *(u64 *)(key + key_field->offset);
5379                         syscall_name = get_syscall_name(uval);
5380                         if (!syscall_name)
5381                                 syscall_name = "unknown_syscall";
5382
5383                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5384                                    syscall_name, uval);
5385                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5386                         seq_puts(m, "stacktrace:\n");
5387                         hist_trigger_stacktrace_print(m,
5388                                                       key + key_field->offset,
5389                                                       HIST_STACKTRACE_DEPTH);
5390                         multiline = true;
5391                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5392                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5393                                    *(u64 *)(key + key_field->offset));
5394                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5395                         seq_printf(m, "%s: %-50s", field_name,
5396                                    (char *)(key + key_field->offset));
5397                 } else {
5398                         uval = *(u64 *)(key + key_field->offset);
5399                         seq_printf(m, "%s: %10llu", field_name, uval);
5400                 }
5401         }
5402
5403         if (!multiline)
5404                 seq_puts(m, " ");
5405
5406         seq_puts(m, "}");
5407 }
5408
5409 static void hist_trigger_entry_print(struct seq_file *m,
5410                                      struct hist_trigger_data *hist_data,
5411                                      void *key,
5412                                      struct tracing_map_elt *elt)
5413 {
5414         const char *field_name;
5415         unsigned int i;
5416
5417         hist_trigger_print_key(m, hist_data, key, elt);
5418
5419         seq_printf(m, " hitcount: %10llu",
5420                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5421
5422         for (i = 1; i < hist_data->n_vals; i++) {
5423                 field_name = hist_field_name(hist_data->fields[i], 0);
5424
5425                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5426                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5427                         continue;
5428
5429                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5430                         seq_printf(m, "  %s: %10llx", field_name,
5431                                    tracing_map_read_sum(elt, i));
5432                 } else {
5433                         seq_printf(m, "  %s: %10llu", field_name,
5434                                    tracing_map_read_sum(elt, i));
5435                 }
5436         }
5437
5438         print_actions(m, hist_data, elt);
5439
5440         seq_puts(m, "\n");
5441 }
5442
5443 static int print_entries(struct seq_file *m,
5444                          struct hist_trigger_data *hist_data)
5445 {
5446         struct tracing_map_sort_entry **sort_entries = NULL;
5447         struct tracing_map *map = hist_data->map;
5448         int i, n_entries;
5449
5450         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5451                                              hist_data->n_sort_keys,
5452                                              &sort_entries);
5453         if (n_entries < 0)
5454                 return n_entries;
5455
5456         for (i = 0; i < n_entries; i++)
5457                 hist_trigger_entry_print(m, hist_data,
5458                                          sort_entries[i]->key,
5459                                          sort_entries[i]->elt);
5460
5461         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5462
5463         return n_entries;
5464 }
5465
5466 static void hist_trigger_show(struct seq_file *m,
5467                               struct event_trigger_data *data, int n)
5468 {
5469         struct hist_trigger_data *hist_data;
5470         int n_entries;
5471
5472         if (n > 0)
5473                 seq_puts(m, "\n\n");
5474
5475         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5476         data->ops->print(m, data->ops, data);
5477         seq_puts(m, "#\n\n");
5478
5479         hist_data = data->private_data;
5480         n_entries = print_entries(m, hist_data);
5481         if (n_entries < 0)
5482                 n_entries = 0;
5483
5484         track_data_snapshot_print(m, hist_data);
5485
5486         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5487                    (u64)atomic64_read(&hist_data->map->hits),
5488                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5489 }
5490
5491 static int hist_show(struct seq_file *m, void *v)
5492 {
5493         struct event_trigger_data *data;
5494         struct trace_event_file *event_file;
5495         int n = 0, ret = 0;
5496
5497         mutex_lock(&event_mutex);
5498
5499         event_file = event_file_data(m->private);
5500         if (unlikely(!event_file)) {
5501                 ret = -ENODEV;
5502                 goto out_unlock;
5503         }
5504
5505         list_for_each_entry_rcu(data, &event_file->triggers, list) {
5506                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5507                         hist_trigger_show(m, data, n++);
5508         }
5509
5510  out_unlock:
5511         mutex_unlock(&event_mutex);
5512
5513         return ret;
5514 }
5515
5516 static int event_hist_open(struct inode *inode, struct file *file)
5517 {
5518         return single_open(file, hist_show, file);
5519 }
5520
5521 const struct file_operations event_hist_fops = {
5522         .open = event_hist_open,
5523         .read = seq_read,
5524         .llseek = seq_lseek,
5525         .release = single_release,
5526 };
5527
5528 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5529 {
5530         const char *field_name = hist_field_name(hist_field, 0);
5531
5532         if (hist_field->var.name)
5533                 seq_printf(m, "%s=", hist_field->var.name);
5534
5535         if (hist_field->flags & HIST_FIELD_FL_CPU)
5536                 seq_puts(m, "cpu");
5537         else if (field_name) {
5538                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5539                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5540                         seq_putc(m, '$');
5541                 seq_printf(m, "%s", field_name);
5542         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5543                 seq_puts(m, "common_timestamp");
5544
5545         if (hist_field->flags) {
5546                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5547                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5548                         const char *flags = get_hist_field_flags(hist_field);
5549
5550                         if (flags)
5551                                 seq_printf(m, ".%s", flags);
5552                 }
5553         }
5554 }
5555
5556 static int event_hist_trigger_print(struct seq_file *m,
5557                                     struct event_trigger_ops *ops,
5558                                     struct event_trigger_data *data)
5559 {
5560         struct hist_trigger_data *hist_data = data->private_data;
5561         struct hist_field *field;
5562         bool have_var = false;
5563         unsigned int i;
5564
5565         seq_puts(m, "hist:");
5566
5567         if (data->name)
5568                 seq_printf(m, "%s:", data->name);
5569
5570         seq_puts(m, "keys=");
5571
5572         for_each_hist_key_field(i, hist_data) {
5573                 field = hist_data->fields[i];
5574
5575                 if (i > hist_data->n_vals)
5576                         seq_puts(m, ",");
5577
5578                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5579                         seq_puts(m, "stacktrace");
5580                 else
5581                         hist_field_print(m, field);
5582         }
5583
5584         seq_puts(m, ":vals=");
5585
5586         for_each_hist_val_field(i, hist_data) {
5587                 field = hist_data->fields[i];
5588                 if (field->flags & HIST_FIELD_FL_VAR) {
5589                         have_var = true;
5590                         continue;
5591                 }
5592
5593                 if (i == HITCOUNT_IDX)
5594                         seq_puts(m, "hitcount");
5595                 else {
5596                         seq_puts(m, ",");
5597                         hist_field_print(m, field);
5598                 }
5599         }
5600
5601         if (have_var) {
5602                 unsigned int n = 0;
5603
5604                 seq_puts(m, ":");
5605
5606                 for_each_hist_val_field(i, hist_data) {
5607                         field = hist_data->fields[i];
5608
5609                         if (field->flags & HIST_FIELD_FL_VAR) {
5610                                 if (n++)
5611                                         seq_puts(m, ",");
5612                                 hist_field_print(m, field);
5613                         }
5614                 }
5615         }
5616
5617         seq_puts(m, ":sort=");
5618
5619         for (i = 0; i < hist_data->n_sort_keys; i++) {
5620                 struct tracing_map_sort_key *sort_key;
5621                 unsigned int idx, first_key_idx;
5622
5623                 /* skip VAR vals */
5624                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5625
5626                 sort_key = &hist_data->sort_keys[i];
5627                 idx = sort_key->field_idx;
5628
5629                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5630                         return -EINVAL;
5631
5632                 if (i > 0)
5633                         seq_puts(m, ",");
5634
5635                 if (idx == HITCOUNT_IDX)
5636                         seq_puts(m, "hitcount");
5637                 else {
5638                         if (idx >= first_key_idx)
5639                                 idx += hist_data->n_vars;
5640                         hist_field_print(m, hist_data->fields[idx]);
5641                 }
5642
5643                 if (sort_key->descending)
5644                         seq_puts(m, ".descending");
5645         }
5646         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5647         if (hist_data->enable_timestamps)
5648                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5649
5650         print_actions_spec(m, hist_data);
5651
5652         if (data->filter_str)
5653                 seq_printf(m, " if %s", data->filter_str);
5654
5655         if (data->paused)
5656                 seq_puts(m, " [paused]");
5657         else
5658                 seq_puts(m, " [active]");
5659
5660         seq_putc(m, '\n');
5661
5662         return 0;
5663 }
5664
5665 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5666                                    struct event_trigger_data *data)
5667 {
5668         struct hist_trigger_data *hist_data = data->private_data;
5669
5670         if (!data->ref && hist_data->attrs->name)
5671                 save_named_trigger(hist_data->attrs->name, data);
5672
5673         data->ref++;
5674
5675         return 0;
5676 }
5677
5678 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5679 {
5680         struct trace_event_file *file;
5681         unsigned int i;
5682         char *cmd;
5683         int ret;
5684
5685         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5686                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5687                 cmd = hist_data->field_var_hists[i]->cmd;
5688                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5689                                               "!hist", "hist", cmd);
5690         }
5691 }
5692
5693 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5694                                     struct event_trigger_data *data)
5695 {
5696         struct hist_trigger_data *hist_data = data->private_data;
5697
5698         if (WARN_ON_ONCE(data->ref <= 0))
5699                 return;
5700
5701         data->ref--;
5702         if (!data->ref) {
5703                 if (data->name)
5704                         del_named_trigger(data);
5705
5706                 trigger_data_free(data);
5707
5708                 remove_hist_vars(hist_data);
5709
5710                 unregister_field_var_hists(hist_data);
5711
5712                 destroy_hist_data(hist_data);
5713         }
5714 }
5715
5716 static struct event_trigger_ops event_hist_trigger_ops = {
5717         .func                   = event_hist_trigger,
5718         .print                  = event_hist_trigger_print,
5719         .init                   = event_hist_trigger_init,
5720         .free                   = event_hist_trigger_free,
5721 };
5722
5723 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5724                                          struct event_trigger_data *data)
5725 {
5726         data->ref++;
5727
5728         save_named_trigger(data->named_data->name, data);
5729
5730         event_hist_trigger_init(ops, data->named_data);
5731
5732         return 0;
5733 }
5734
5735 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5736                                           struct event_trigger_data *data)
5737 {
5738         if (WARN_ON_ONCE(data->ref <= 0))
5739                 return;
5740
5741         event_hist_trigger_free(ops, data->named_data);
5742
5743         data->ref--;
5744         if (!data->ref) {
5745                 del_named_trigger(data);
5746                 trigger_data_free(data);
5747         }
5748 }
5749
5750 static struct event_trigger_ops event_hist_trigger_named_ops = {
5751         .func                   = event_hist_trigger,
5752         .print                  = event_hist_trigger_print,
5753         .init                   = event_hist_trigger_named_init,
5754         .free                   = event_hist_trigger_named_free,
5755 };
5756
5757 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5758                                                             char *param)
5759 {
5760         return &event_hist_trigger_ops;
5761 }
5762
5763 static void hist_clear(struct event_trigger_data *data)
5764 {
5765         struct hist_trigger_data *hist_data = data->private_data;
5766
5767         if (data->name)
5768                 pause_named_trigger(data);
5769
5770         tracepoint_synchronize_unregister();
5771
5772         tracing_map_clear(hist_data->map);
5773
5774         if (data->name)
5775                 unpause_named_trigger(data);
5776 }
5777
5778 static bool compatible_field(struct ftrace_event_field *field,
5779                              struct ftrace_event_field *test_field)
5780 {
5781         if (field == test_field)
5782                 return true;
5783         if (field == NULL || test_field == NULL)
5784                 return false;
5785         if (strcmp(field->name, test_field->name) != 0)
5786                 return false;
5787         if (strcmp(field->type, test_field->type) != 0)
5788                 return false;
5789         if (field->size != test_field->size)
5790                 return false;
5791         if (field->is_signed != test_field->is_signed)
5792                 return false;
5793
5794         return true;
5795 }
5796
5797 static bool hist_trigger_match(struct event_trigger_data *data,
5798                                struct event_trigger_data *data_test,
5799                                struct event_trigger_data *named_data,
5800                                bool ignore_filter)
5801 {
5802         struct tracing_map_sort_key *sort_key, *sort_key_test;
5803         struct hist_trigger_data *hist_data, *hist_data_test;
5804         struct hist_field *key_field, *key_field_test;
5805         unsigned int i;
5806
5807         if (named_data && (named_data != data_test) &&
5808             (named_data != data_test->named_data))
5809                 return false;
5810
5811         if (!named_data && is_named_trigger(data_test))
5812                 return false;
5813
5814         hist_data = data->private_data;
5815         hist_data_test = data_test->private_data;
5816
5817         if (hist_data->n_vals != hist_data_test->n_vals ||
5818             hist_data->n_fields != hist_data_test->n_fields ||
5819             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5820                 return false;
5821
5822         if (!ignore_filter) {
5823                 if ((data->filter_str && !data_test->filter_str) ||
5824                    (!data->filter_str && data_test->filter_str))
5825                         return false;
5826         }
5827
5828         for_each_hist_field(i, hist_data) {
5829                 key_field = hist_data->fields[i];
5830                 key_field_test = hist_data_test->fields[i];
5831
5832                 if (key_field->flags != key_field_test->flags)
5833                         return false;
5834                 if (!compatible_field(key_field->field, key_field_test->field))
5835                         return false;
5836                 if (key_field->offset != key_field_test->offset)
5837                         return false;
5838                 if (key_field->size != key_field_test->size)
5839                         return false;
5840                 if (key_field->is_signed != key_field_test->is_signed)
5841                         return false;
5842                 if (!!key_field->var.name != !!key_field_test->var.name)
5843                         return false;
5844                 if (key_field->var.name &&
5845                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5846                         return false;
5847         }
5848
5849         for (i = 0; i < hist_data->n_sort_keys; i++) {
5850                 sort_key = &hist_data->sort_keys[i];
5851                 sort_key_test = &hist_data_test->sort_keys[i];
5852
5853                 if (sort_key->field_idx != sort_key_test->field_idx ||
5854                     sort_key->descending != sort_key_test->descending)
5855                         return false;
5856         }
5857
5858         if (!ignore_filter && data->filter_str &&
5859             (strcmp(data->filter_str, data_test->filter_str) != 0))
5860                 return false;
5861
5862         if (!actions_match(hist_data, hist_data_test))
5863                 return false;
5864
5865         return true;
5866 }
5867
5868 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5869                                  struct event_trigger_data *data,
5870                                  struct trace_event_file *file)
5871 {
5872         struct hist_trigger_data *hist_data = data->private_data;
5873         struct event_trigger_data *test, *named_data = NULL;
5874         struct trace_array *tr = file->tr;
5875         int ret = 0;
5876
5877         if (hist_data->attrs->name) {
5878                 named_data = find_named_trigger(hist_data->attrs->name);
5879                 if (named_data) {
5880                         if (!hist_trigger_match(data, named_data, named_data,
5881                                                 true)) {
5882                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5883                                 ret = -EINVAL;
5884                                 goto out;
5885                         }
5886                 }
5887         }
5888
5889         if (hist_data->attrs->name && !named_data)
5890                 goto new;
5891
5892         list_for_each_entry_rcu(test, &file->triggers, list) {
5893                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5894                         if (!hist_trigger_match(data, test, named_data, false))
5895                                 continue;
5896                         if (hist_data->attrs->pause)
5897                                 test->paused = true;
5898                         else if (hist_data->attrs->cont)
5899                                 test->paused = false;
5900                         else if (hist_data->attrs->clear)
5901                                 hist_clear(test);
5902                         else {
5903                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5904                                 ret = -EEXIST;
5905                         }
5906                         goto out;
5907                 }
5908         }
5909  new:
5910         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5911                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5912                 ret = -ENOENT;
5913                 goto out;
5914         }
5915
5916         if (hist_data->attrs->pause)
5917                 data->paused = true;
5918
5919         if (named_data) {
5920                 data->private_data = named_data->private_data;
5921                 set_named_trigger_data(data, named_data);
5922                 data->ops = &event_hist_trigger_named_ops;
5923         }
5924
5925         if (data->ops->init) {
5926                 ret = data->ops->init(data->ops, data);
5927                 if (ret < 0)
5928                         goto out;
5929         }
5930
5931         if (hist_data->enable_timestamps) {
5932                 char *clock = hist_data->attrs->clock;
5933
5934                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5935                 if (ret) {
5936                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5937                         goto out;
5938                 }
5939
5940                 tracing_set_time_stamp_abs(file->tr, true);
5941         }
5942
5943         if (named_data)
5944                 destroy_hist_data(hist_data);
5945
5946         ret++;
5947  out:
5948         return ret;
5949 }
5950
5951 static int hist_trigger_enable(struct event_trigger_data *data,
5952                                struct trace_event_file *file)
5953 {
5954         int ret = 0;
5955
5956         list_add_tail_rcu(&data->list, &file->triggers);
5957
5958         update_cond_flag(file);
5959
5960         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5961                 list_del_rcu(&data->list);
5962                 update_cond_flag(file);
5963                 ret--;
5964         }
5965
5966         return ret;
5967 }
5968
5969 static bool have_hist_trigger_match(struct event_trigger_data *data,
5970                                     struct trace_event_file *file)
5971 {
5972         struct hist_trigger_data *hist_data = data->private_data;
5973         struct event_trigger_data *test, *named_data = NULL;
5974         bool match = false;
5975
5976         if (hist_data->attrs->name)
5977                 named_data = find_named_trigger(hist_data->attrs->name);
5978
5979         list_for_each_entry_rcu(test, &file->triggers, list) {
5980                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5981                         if (hist_trigger_match(data, test, named_data, false)) {
5982                                 match = true;
5983                                 break;
5984                         }
5985                 }
5986         }
5987
5988         return match;
5989 }
5990
5991 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5992                                     struct trace_event_file *file)
5993 {
5994         struct hist_trigger_data *hist_data = data->private_data;
5995         struct event_trigger_data *test, *named_data = NULL;
5996
5997         if (hist_data->attrs->name)
5998                 named_data = find_named_trigger(hist_data->attrs->name);
5999
6000         list_for_each_entry_rcu(test, &file->triggers, list) {
6001                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6002                         if (!hist_trigger_match(data, test, named_data, false))
6003                                 continue;
6004                         hist_data = test->private_data;
6005                         if (check_var_refs(hist_data))
6006                                 return true;
6007                         break;
6008                 }
6009         }
6010
6011         return false;
6012 }
6013
6014 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6015                                     struct event_trigger_data *data,
6016                                     struct trace_event_file *file)
6017 {
6018         struct hist_trigger_data *hist_data = data->private_data;
6019         struct event_trigger_data *test, *named_data = NULL;
6020         bool unregistered = false;
6021
6022         if (hist_data->attrs->name)
6023                 named_data = find_named_trigger(hist_data->attrs->name);
6024
6025         list_for_each_entry_rcu(test, &file->triggers, list) {
6026                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6027                         if (!hist_trigger_match(data, test, named_data, false))
6028                                 continue;
6029                         unregistered = true;
6030                         list_del_rcu(&test->list);
6031                         trace_event_trigger_enable_disable(file, 0);
6032                         update_cond_flag(file);
6033                         break;
6034                 }
6035         }
6036
6037         if (unregistered && test->ops->free)
6038                 test->ops->free(test->ops, test);
6039
6040         if (hist_data->enable_timestamps) {
6041                 if (!hist_data->remove || unregistered)
6042                         tracing_set_time_stamp_abs(file->tr, false);
6043         }
6044 }
6045
6046 static bool hist_file_check_refs(struct trace_event_file *file)
6047 {
6048         struct hist_trigger_data *hist_data;
6049         struct event_trigger_data *test;
6050
6051         list_for_each_entry_rcu(test, &file->triggers, list) {
6052                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6053                         hist_data = test->private_data;
6054                         if (check_var_refs(hist_data))
6055                                 return true;
6056                 }
6057         }
6058
6059         return false;
6060 }
6061
6062 static void hist_unreg_all(struct trace_event_file *file)
6063 {
6064         struct event_trigger_data *test, *n;
6065         struct hist_trigger_data *hist_data;
6066         struct synth_event *se;
6067         const char *se_name;
6068
6069         lockdep_assert_held(&event_mutex);
6070
6071         if (hist_file_check_refs(file))
6072                 return;
6073
6074         list_for_each_entry_safe(test, n, &file->triggers, list) {
6075                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6076                         hist_data = test->private_data;
6077                         list_del_rcu(&test->list);
6078                         trace_event_trigger_enable_disable(file, 0);
6079
6080                         se_name = trace_event_name(file->event_call);
6081                         se = find_synth_event(se_name);
6082                         if (se)
6083                                 se->ref--;
6084
6085                         update_cond_flag(file);
6086                         if (hist_data->enable_timestamps)
6087                                 tracing_set_time_stamp_abs(file->tr, false);
6088                         if (test->ops->free)
6089                                 test->ops->free(test->ops, test);
6090                 }
6091         }
6092 }
6093
6094 static int event_hist_trigger_func(struct event_command *cmd_ops,
6095                                    struct trace_event_file *file,
6096                                    char *glob, char *cmd, char *param)
6097 {
6098         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6099         struct event_trigger_data *trigger_data;
6100         struct hist_trigger_attrs *attrs;
6101         struct event_trigger_ops *trigger_ops;
6102         struct hist_trigger_data *hist_data;
6103         struct synth_event *se;
6104         const char *se_name;
6105         bool remove = false;
6106         char *trigger, *p;
6107         int ret = 0;
6108
6109         lockdep_assert_held(&event_mutex);
6110
6111         if (glob && strlen(glob)) {
6112                 hist_err_clear();
6113                 last_cmd_set(file, param);
6114         }
6115
6116         if (!param)
6117                 return -EINVAL;
6118
6119         if (glob[0] == '!')
6120                 remove = true;
6121
6122         /*
6123          * separate the trigger from the filter (k:v [if filter])
6124          * allowing for whitespace in the trigger
6125          */
6126         p = trigger = param;
6127         do {
6128                 p = strstr(p, "if");
6129                 if (!p)
6130                         break;
6131                 if (p == param)
6132                         return -EINVAL;
6133                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6134                         p++;
6135                         continue;
6136                 }
6137                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6138                         return -EINVAL;
6139                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6140                         p++;
6141                         continue;
6142                 }
6143                 break;
6144         } while (p);
6145
6146         if (!p)
6147                 param = NULL;
6148         else {
6149                 *(p - 1) = '\0';
6150                 param = strstrip(p);
6151                 trigger = strstrip(trigger);
6152         }
6153
6154         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6155         if (IS_ERR(attrs))
6156                 return PTR_ERR(attrs);
6157
6158         if (attrs->map_bits)
6159                 hist_trigger_bits = attrs->map_bits;
6160
6161         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6162         if (IS_ERR(hist_data)) {
6163                 destroy_hist_trigger_attrs(attrs);
6164                 return PTR_ERR(hist_data);
6165         }
6166
6167         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6168
6169         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6170         if (!trigger_data) {
6171                 ret = -ENOMEM;
6172                 goto out_free;
6173         }
6174
6175         trigger_data->count = -1;
6176         trigger_data->ops = trigger_ops;
6177         trigger_data->cmd_ops = cmd_ops;
6178
6179         INIT_LIST_HEAD(&trigger_data->list);
6180         RCU_INIT_POINTER(trigger_data->filter, NULL);
6181
6182         trigger_data->private_data = hist_data;
6183
6184         /* if param is non-empty, it's supposed to be a filter */
6185         if (param && cmd_ops->set_filter) {
6186                 ret = cmd_ops->set_filter(param, trigger_data, file);
6187                 if (ret < 0)
6188                         goto out_free;
6189         }
6190
6191         if (remove) {
6192                 if (!have_hist_trigger_match(trigger_data, file))
6193                         goto out_free;
6194
6195                 if (hist_trigger_check_refs(trigger_data, file)) {
6196                         ret = -EBUSY;
6197                         goto out_free;
6198                 }
6199
6200                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6201                 se_name = trace_event_name(file->event_call);
6202                 se = find_synth_event(se_name);
6203                 if (se)
6204                         se->ref--;
6205                 ret = 0;
6206                 goto out_free;
6207         }
6208
6209         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6210         /*
6211          * The above returns on success the # of triggers registered,
6212          * but if it didn't register any it returns zero.  Consider no
6213          * triggers registered a failure too.
6214          */
6215         if (!ret) {
6216                 if (!(attrs->pause || attrs->cont || attrs->clear))
6217                         ret = -ENOENT;
6218                 goto out_free;
6219         } else if (ret < 0)
6220                 goto out_free;
6221
6222         if (get_named_trigger_data(trigger_data))
6223                 goto enable;
6224
6225         if (has_hist_vars(hist_data))
6226                 save_hist_vars(hist_data);
6227
6228         ret = create_actions(hist_data);
6229         if (ret)
6230                 goto out_unreg;
6231
6232         ret = tracing_map_init(hist_data->map);
6233         if (ret)
6234                 goto out_unreg;
6235 enable:
6236         ret = hist_trigger_enable(trigger_data, file);
6237         if (ret)
6238                 goto out_unreg;
6239
6240         se_name = trace_event_name(file->event_call);
6241         se = find_synth_event(se_name);
6242         if (se)
6243                 se->ref++;
6244         /* Just return zero, not the number of registered triggers */
6245         ret = 0;
6246  out:
6247         if (ret == 0)
6248                 hist_err_clear();
6249
6250         return ret;
6251  out_unreg:
6252         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6253  out_free:
6254         if (cmd_ops->set_filter)
6255                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6256
6257         remove_hist_vars(hist_data);
6258
6259         kfree(trigger_data);
6260
6261         destroy_hist_data(hist_data);
6262         goto out;
6263 }
6264
6265 static struct event_command trigger_hist_cmd = {
6266         .name                   = "hist",
6267         .trigger_type           = ETT_EVENT_HIST,
6268         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6269         .func                   = event_hist_trigger_func,
6270         .reg                    = hist_register_trigger,
6271         .unreg                  = hist_unregister_trigger,
6272         .unreg_all              = hist_unreg_all,
6273         .get_trigger_ops        = event_hist_get_trigger_ops,
6274         .set_filter             = set_trigger_filter,
6275 };
6276
6277 __init int register_trigger_hist_cmd(void)
6278 {
6279         int ret;
6280
6281         ret = register_event_command(&trigger_hist_cmd);
6282         WARN_ON(ret < 0);
6283
6284         return ret;
6285 }
6286
6287 static void
6288 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6289                     struct ring_buffer_event *event)
6290 {
6291         struct enable_trigger_data *enable_data = data->private_data;
6292         struct event_trigger_data *test;
6293
6294         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
6295                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6296                         if (enable_data->enable)
6297                                 test->paused = false;
6298                         else
6299                                 test->paused = true;
6300                 }
6301         }
6302 }
6303
6304 static void
6305 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6306                           struct ring_buffer_event *event)
6307 {
6308         if (!data->count)
6309                 return;
6310
6311         if (data->count != -1)
6312                 (data->count)--;
6313
6314         hist_enable_trigger(data, rec, event);
6315 }
6316
6317 static struct event_trigger_ops hist_enable_trigger_ops = {
6318         .func                   = hist_enable_trigger,
6319         .print                  = event_enable_trigger_print,
6320         .init                   = event_trigger_init,
6321         .free                   = event_enable_trigger_free,
6322 };
6323
6324 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6325         .func                   = hist_enable_count_trigger,
6326         .print                  = event_enable_trigger_print,
6327         .init                   = event_trigger_init,
6328         .free                   = event_enable_trigger_free,
6329 };
6330
6331 static struct event_trigger_ops hist_disable_trigger_ops = {
6332         .func                   = hist_enable_trigger,
6333         .print                  = event_enable_trigger_print,
6334         .init                   = event_trigger_init,
6335         .free                   = event_enable_trigger_free,
6336 };
6337
6338 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6339         .func                   = hist_enable_count_trigger,
6340         .print                  = event_enable_trigger_print,
6341         .init                   = event_trigger_init,
6342         .free                   = event_enable_trigger_free,
6343 };
6344
6345 static struct event_trigger_ops *
6346 hist_enable_get_trigger_ops(char *cmd, char *param)
6347 {
6348         struct event_trigger_ops *ops;
6349         bool enable;
6350
6351         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6352
6353         if (enable)
6354                 ops = param ? &hist_enable_count_trigger_ops :
6355                         &hist_enable_trigger_ops;
6356         else
6357                 ops = param ? &hist_disable_count_trigger_ops :
6358                         &hist_disable_trigger_ops;
6359
6360         return ops;
6361 }
6362
6363 static void hist_enable_unreg_all(struct trace_event_file *file)
6364 {
6365         struct event_trigger_data *test, *n;
6366
6367         list_for_each_entry_safe(test, n, &file->triggers, list) {
6368                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6369                         list_del_rcu(&test->list);
6370                         update_cond_flag(file);
6371                         trace_event_trigger_enable_disable(file, 0);
6372                         if (test->ops->free)
6373                                 test->ops->free(test->ops, test);
6374                 }
6375         }
6376 }
6377
6378 static struct event_command trigger_hist_enable_cmd = {
6379         .name                   = ENABLE_HIST_STR,
6380         .trigger_type           = ETT_HIST_ENABLE,
6381         .func                   = event_enable_trigger_func,
6382         .reg                    = event_enable_register_trigger,
6383         .unreg                  = event_enable_unregister_trigger,
6384         .unreg_all              = hist_enable_unreg_all,
6385         .get_trigger_ops        = hist_enable_get_trigger_ops,
6386         .set_filter             = set_trigger_filter,
6387 };
6388
6389 static struct event_command trigger_hist_disable_cmd = {
6390         .name                   = DISABLE_HIST_STR,
6391         .trigger_type           = ETT_HIST_ENABLE,
6392         .func                   = event_enable_trigger_func,
6393         .reg                    = event_enable_register_trigger,
6394         .unreg                  = event_enable_unregister_trigger,
6395         .unreg_all              = hist_enable_unreg_all,
6396         .get_trigger_ops        = hist_enable_get_trigger_ops,
6397         .set_filter             = set_trigger_filter,
6398 };
6399
6400 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6401 {
6402         unregister_event_command(&trigger_hist_enable_cmd);
6403         unregister_event_command(&trigger_hist_disable_cmd);
6404 }
6405
6406 __init int register_trigger_hist_enable_disable_cmds(void)
6407 {
6408         int ret;
6409
6410         ret = register_event_command(&trigger_hist_enable_cmd);
6411         if (WARN_ON(ret < 0))
6412                 return ret;
6413         ret = register_event_command(&trigger_hist_disable_cmd);
6414         if (WARN_ON(ret < 0))
6415                 unregister_trigger_hist_enable_disable_cmds();
6416
6417         return ret;
6418 }
6419
6420 static __init int trace_events_hist_init(void)
6421 {
6422         struct dentry *entry = NULL;
6423         struct dentry *d_tracer;
6424         int err = 0;
6425
6426         err = dyn_event_register(&synth_event_ops);
6427         if (err) {
6428                 pr_warn("Could not register synth_event_ops\n");
6429                 return err;
6430         }
6431
6432         d_tracer = tracing_init_dentry();
6433         if (IS_ERR(d_tracer)) {
6434                 err = PTR_ERR(d_tracer);
6435                 goto err;
6436         }
6437
6438         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6439                                     NULL, &synth_events_fops);
6440         if (!entry) {
6441                 err = -ENODEV;
6442                 goto err;
6443         }
6444
6445         return err;
6446  err:
6447         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6448
6449         return err;
6450 }
6451
6452 fs_initcall(trace_events_hist_init);