kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  69
  70 #undef C
  71 #define C(a, b)         HIST_ERR_##a
  72
  73 enum { ERRORS };
  74
  75 #undef C
  76 #define C(a, b)         b
  77
  78 static const char *err_text[] = { ERRORS };
  79
  80 struct hist_field;
  81
  82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  83                                 struct tracing_map_elt *elt,
  84                                 struct trace_buffer *buffer,
  85                                 struct ring_buffer_event *rbe,
  86                                 void *event);
  87
  88 #define HIST_FIELD_OPERANDS_MAX 2
  89 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  90 #define HIST_ACTIONS_MAX        8
  91
  92 enum field_op_id {
  93         FIELD_OP_NONE,
  94         FIELD_OP_PLUS,
  95         FIELD_OP_MINUS,
  96         FIELD_OP_UNARY_MINUS,
  97 };
  98
  99 /*
 100  * A hist_var (histogram variable) contains variable information for
 101  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 102  * flag set.  A hist_var has a variable name e.g. ts0, and is
 103  * associated with a given histogram trigger, as specified by
 104  * hist_data.  The hist_var idx is the unique index assigned to the
 105  * variable by the hist trigger's tracing_map.  The idx is what is
 106  * used to set a variable's value and, by a variable reference, to
 107  * retrieve it.
 108  */
 109 struct hist_var {
 110         char                            *name;
 111         struct hist_trigger_data        *hist_data;
 112         unsigned int                    idx;
 113 };
 114
 115 struct hist_field {
 116         struct ftrace_event_field       *field;
 117         unsigned long                   flags;
 118         hist_field_fn_t                 fn;
 119         unsigned int                    ref;
 120         unsigned int                    size;
 121         unsigned int                    offset;
 122         unsigned int                    is_signed;
 123         const char                      *type;
 124         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 125         struct hist_trigger_data        *hist_data;
 126
 127         /*
 128          * Variable fields contain variable-specific info in var.
 129          */
 130         struct hist_var                 var;
 131         enum field_op_id                operator;
 132         char                            *system;
 133         char                            *event_name;
 134
 135         /*
 136          * The name field is used for EXPR and VAR_REF fields.  VAR
 137          * fields contain the variable name in var.name.
 138          */
 139         char                            *name;
 140
 141         /*
 142          * When a histogram trigger is hit, if it has any references
 143          * to variables, the values of those variables are collected
 144          * into a var_ref_vals array by resolve_var_refs().  The
 145          * current value of each variable is read from the tracing_map
 146          * using the hist field's hist_var.idx and entered into the
 147          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 148          */
 149         unsigned int                    var_ref_idx;
 150         bool                            read_once;
 151
 152         unsigned int                    var_str_idx;
 153 };
 154
 155 static u64 hist_field_none(struct hist_field *field,
 156                            struct tracing_map_elt *elt,
 157                            struct trace_buffer *buffer,
 158                            struct ring_buffer_event *rbe,
 159                            void *event)
 160 {
 161         return 0;
 162 }
 163
 164 static u64 hist_field_counter(struct hist_field *field,
 165                               struct tracing_map_elt *elt,
 166                               struct trace_buffer *buffer,
 167                               struct ring_buffer_event *rbe,
 168                               void *event)
 169 {
 170         return 1;
 171 }
 172
 173 static u64 hist_field_string(struct hist_field *hist_field,
 174                              struct tracing_map_elt *elt,
 175                              struct trace_buffer *buffer,
 176                              struct ring_buffer_event *rbe,
 177                              void *event)
 178 {
 179         char *addr = (char *)(event + hist_field->field->offset);
 180
 181         return (u64)(unsigned long)addr;
 182 }
 183
 184 static u64 hist_field_dynstring(struct hist_field *hist_field,
 185                                 struct tracing_map_elt *elt,
 186                                 struct trace_buffer *buffer,
 187                                 struct ring_buffer_event *rbe,
 188                                 void *event)
 189 {
 190         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 191         int str_loc = str_item & 0xffff;
 192         char *addr = (char *)(event + str_loc);
 193
 194         return (u64)(unsigned long)addr;
 195 }
 196
 197 static u64 hist_field_pstring(struct hist_field *hist_field,
 198                               struct tracing_map_elt *elt,
 199                               struct trace_buffer *buffer,
 200                               struct ring_buffer_event *rbe,
 201                               void *event)
 202 {
 203         char **addr = (char **)(event + hist_field->field->offset);
 204
 205         return (u64)(unsigned long)*addr;
 206 }
 207
 208 static u64 hist_field_log2(struct hist_field *hist_field,
 209                            struct tracing_map_elt *elt,
 210                            struct trace_buffer *buffer,
 211                            struct ring_buffer_event *rbe,
 212                            void *event)
 213 {
 214         struct hist_field *operand = hist_field->operands[0];
 215
 216         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 217
 218         return (u64) ilog2(roundup_pow_of_two(val));
 219 }
 220
 221 static u64 hist_field_plus(struct hist_field *hist_field,
 222                            struct tracing_map_elt *elt,
 223                            struct trace_buffer *buffer,
 224                            struct ring_buffer_event *rbe,
 225                            void *event)
 226 {
 227         struct hist_field *operand1 = hist_field->operands[0];
 228         struct hist_field *operand2 = hist_field->operands[1];
 229
 230         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 231         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 232
 233         return val1 + val2;
 234 }
 235
 236 static u64 hist_field_minus(struct hist_field *hist_field,
 237                             struct tracing_map_elt *elt,
 238                             struct trace_buffer *buffer,
 239                             struct ring_buffer_event *rbe,
 240                             void *event)
 241 {
 242         struct hist_field *operand1 = hist_field->operands[0];
 243         struct hist_field *operand2 = hist_field->operands[1];
 244
 245         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 246         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 247
 248         return val1 - val2;
 249 }
 250
 251 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 252                                   struct tracing_map_elt *elt,
 253                                   struct trace_buffer *buffer,
 254                                   struct ring_buffer_event *rbe,
 255                                   void *event)
 256 {
 257         struct hist_field *operand = hist_field->operands[0];
 258
 259         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 260         u64 val = (u64)-sval;
 261
 262         return val;
 263 }
 264
 265 #define DEFINE_HIST_FIELD_FN(type)                                      \
 266         static u64 hist_field_##type(struct hist_field *hist_field,     \
 267                                      struct tracing_map_elt *elt,       \
 268                                      struct trace_buffer *buffer,       \
 269                                      struct ring_buffer_event *rbe,     \
 270                                      void *event)                       \
 271 {                                                                       \
 272         type *addr = (type *)(event + hist_field->field->offset);       \
 273                                                                         \
 274         return (u64)(unsigned long)*addr;                               \
 275 }
 276
 277 DEFINE_HIST_FIELD_FN(s64);
 278 DEFINE_HIST_FIELD_FN(u64);
 279 DEFINE_HIST_FIELD_FN(s32);
 280 DEFINE_HIST_FIELD_FN(u32);
 281 DEFINE_HIST_FIELD_FN(s16);
 282 DEFINE_HIST_FIELD_FN(u16);
 283 DEFINE_HIST_FIELD_FN(s8);
 284 DEFINE_HIST_FIELD_FN(u8);
 285
 286 #define for_each_hist_field(i, hist_data)       \
 287         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 288
 289 #define for_each_hist_val_field(i, hist_data)   \
 290         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 291
 292 #define for_each_hist_key_field(i, hist_data)   \
 293         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 294
 295 #define HIST_STACKTRACE_DEPTH   16
 296 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 297 #define HIST_STACKTRACE_SKIP    5
 298
 299 #define HITCOUNT_IDX            0
 300 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 301
 302 enum hist_field_flags {
 303         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 304         HIST_FIELD_FL_KEY               = 1 << 1,
 305         HIST_FIELD_FL_STRING            = 1 << 2,
 306         HIST_FIELD_FL_HEX               = 1 << 3,
 307         HIST_FIELD_FL_SYM               = 1 << 4,
 308         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 309         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 310         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 311         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 312         HIST_FIELD_FL_LOG2              = 1 << 9,
 313         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 314         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 315         HIST_FIELD_FL_VAR               = 1 << 12,
 316         HIST_FIELD_FL_EXPR              = 1 << 13,
 317         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 318         HIST_FIELD_FL_CPU               = 1 << 15,
 319         HIST_FIELD_FL_ALIAS             = 1 << 16,
 320 };
 321
 322 struct var_defs {
 323         unsigned int    n_vars;
 324         char            *name[TRACING_MAP_VARS_MAX];
 325         char            *expr[TRACING_MAP_VARS_MAX];
 326 };
 327
 328 struct hist_trigger_attrs {
 329         char            *keys_str;
 330         char            *vals_str;
 331         char            *sort_key_str;
 332         char            *name;
 333         char            *clock;
 334         bool            pause;
 335         bool            cont;
 336         bool            clear;
 337         bool            ts_in_usecs;
 338         unsigned int    map_bits;
 339
 340         char            *assignment_str[TRACING_MAP_VARS_MAX];
 341         unsigned int    n_assignments;
 342
 343         char            *action_str[HIST_ACTIONS_MAX];
 344         unsigned int    n_actions;
 345
 346         struct var_defs var_defs;
 347 };
 348
 349 struct field_var {
 350         struct hist_field       *var;
 351         struct hist_field       *val;
 352 };
 353
 354 struct field_var_hist {
 355         struct hist_trigger_data        *hist_data;
 356         char                            *cmd;
 357 };
 358
 359 struct hist_trigger_data {
 360         struct hist_field               *fields[HIST_FIELDS_MAX];
 361         unsigned int                    n_vals;
 362         unsigned int                    n_keys;
 363         unsigned int                    n_fields;
 364         unsigned int                    n_vars;
 365         unsigned int                    n_var_str;
 366         unsigned int                    key_size;
 367         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 368         unsigned int                    n_sort_keys;
 369         struct trace_event_file         *event_file;
 370         struct hist_trigger_attrs       *attrs;
 371         struct tracing_map              *map;
 372         bool                            enable_timestamps;
 373         bool                            remove;
 374         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 375         unsigned int                    n_var_refs;
 376
 377         struct action_data              *actions[HIST_ACTIONS_MAX];
 378         unsigned int                    n_actions;
 379
 380         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 381         unsigned int                    n_field_vars;
 382         unsigned int                    n_field_var_str;
 383         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 384         unsigned int                    n_field_var_hists;
 385
 386         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 387         unsigned int                    n_save_vars;
 388         unsigned int                    n_save_var_str;
 389 };
 390
 391 struct action_data;
 392
 393 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 394                              struct tracing_map_elt *elt,
 395                              struct trace_buffer *buffer, void *rec,
 396                              struct ring_buffer_event *rbe, void *key,
 397                              struct action_data *data, u64 *var_ref_vals);
 398
 399 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 400
 401 enum handler_id {
 402         HANDLER_ONMATCH = 1,
 403         HANDLER_ONMAX,
 404         HANDLER_ONCHANGE,
 405 };
 406
 407 enum action_id {
 408         ACTION_SAVE = 1,
 409         ACTION_TRACE,
 410         ACTION_SNAPSHOT,
 411 };
 412
 413 struct action_data {
 414         enum handler_id         handler;
 415         enum action_id          action;
 416         char                    *action_name;
 417         action_fn_t             fn;
 418
 419         unsigned int            n_params;
 420         char                    *params[SYNTH_FIELDS_MAX];
 421
 422         /*
 423          * When a histogram trigger is hit, the values of any
 424          * references to variables, including variables being passed
 425          * as parameters to synthetic events, are collected into a
 426          * var_ref_vals array.  This var_ref_idx array is an array of
 427          * indices into the var_ref_vals array, one for each synthetic
 428          * event param, and is passed to the synthetic event
 429          * invocation.
 430          */
 431         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 432         struct synth_event      *synth_event;
 433         bool                    use_trace_keyword;
 434         char                    *synth_event_name;
 435
 436         union {
 437                 struct {
 438                         char                    *event;
 439                         char                    *event_system;
 440                 } match_data;
 441
 442                 struct {
 443                         /*
 444                          * var_str contains the $-unstripped variable
 445                          * name referenced by var_ref, and used when
 446                          * printing the action.  Because var_ref
 447                          * creation is deferred to create_actions(),
 448                          * we need a per-action way to save it until
 449                          * then, thus var_str.
 450                          */
 451                         char                    *var_str;
 452
 453                         /*
 454                          * var_ref refers to the variable being
 455                          * tracked e.g onmax($var).
 456                          */
 457                         struct hist_field       *var_ref;
 458
 459                         /*
 460                          * track_var contains the 'invisible' tracking
 461                          * variable created to keep the current
 462                          * e.g. max value.
 463                          */
 464                         struct hist_field       *track_var;
 465
 466                         check_track_val_fn_t    check_val;
 467                         action_fn_t             save_data;
 468                 } track_data;
 469         };
 470 };
 471
 472 struct track_data {
 473         u64                             track_val;
 474         bool                            updated;
 475
 476         unsigned int                    key_len;
 477         void                            *key;
 478         struct tracing_map_elt          elt;
 479
 480         struct action_data              *action_data;
 481         struct hist_trigger_data        *hist_data;
 482 };
 483
 484 struct hist_elt_data {
 485         char *comm;
 486         u64 *var_ref_vals;
 487         char *field_var_str[SYNTH_FIELDS_MAX];
 488 };
 489
 490 struct snapshot_context {
 491         struct tracing_map_elt  *elt;
 492         void                    *key;
 493 };
 494
 495 static void track_data_free(struct track_data *track_data)
 496 {
 497         struct hist_elt_data *elt_data;
 498
 499         if (!track_data)
 500                 return;
 501
 502         kfree(track_data->key);
 503
 504         elt_data = track_data->elt.private_data;
 505         if (elt_data) {
 506                 kfree(elt_data->comm);
 507                 kfree(elt_data);
 508         }
 509
 510         kfree(track_data);
 511 }
 512
 513 static struct track_data *track_data_alloc(unsigned int key_len,
 514                                            struct action_data *action_data,
 515                                            struct hist_trigger_data *hist_data)
 516 {
 517         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 518         struct hist_elt_data *elt_data;
 519
 520         if (!data)
 521                 return ERR_PTR(-ENOMEM);
 522
 523         data->key = kzalloc(key_len, GFP_KERNEL);
 524         if (!data->key) {
 525                 track_data_free(data);
 526                 return ERR_PTR(-ENOMEM);
 527         }
 528
 529         data->key_len = key_len;
 530         data->action_data = action_data;
 531         data->hist_data = hist_data;
 532
 533         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 534         if (!elt_data) {
 535                 track_data_free(data);
 536                 return ERR_PTR(-ENOMEM);
 537         }
 538
 539         data->elt.private_data = elt_data;
 540
 541         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 542         if (!elt_data->comm) {
 543                 track_data_free(data);
 544                 return ERR_PTR(-ENOMEM);
 545         }
 546
 547         return data;
 548 }
 549
 550 static char last_cmd[MAX_FILTER_STR_VAL];
 551 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 552
 553 static int errpos(char *str)
 554 {
 555         return err_pos(last_cmd, str);
 556 }
 557
 558 static void last_cmd_set(struct trace_event_file *file, char *str)
 559 {
 560         const char *system = NULL, *name = NULL;
 561         struct trace_event_call *call;
 562
 563         if (!str)
 564                 return;
 565
 566         strcpy(last_cmd, "hist:");
 567         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 568
 569         if (file) {
 570                 call = file->event_call;
 571                 system = call->class->system;
 572                 if (system) {
 573                         name = trace_event_name(call);
 574                         if (!name)
 575                                 system = NULL;
 576                 }
 577         }
 578
 579         if (system)
 580                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 581 }
 582
 583 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 584 {
 585         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 586                         err_type, err_pos);
 587 }
 588
 589 static void hist_err_clear(void)
 590 {
 591         last_cmd[0] = '\0';
 592         last_cmd_loc[0] = '\0';
 593 }
 594
 595 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 596                                     unsigned int *var_ref_idx);
 597
 598 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 599                                unsigned int *var_ref_idx)
 600 {
 601         struct tracepoint *tp = event->tp;
 602
 603         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 604                 struct tracepoint_func *probe_func_ptr;
 605                 synth_probe_func_t probe_func;
 606                 void *__data;
 607
 608                 if (!(cpu_online(raw_smp_processor_id())))
 609                         return;
 610
 611                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 612                 if (probe_func_ptr) {
 613                         do {
 614                                 probe_func = probe_func_ptr->func;
 615                                 __data = probe_func_ptr->data;
 616                                 probe_func(__data, var_ref_vals, var_ref_idx);
 617                         } while ((++probe_func_ptr)->func);
 618                 }
 619         }
 620 }
 621
 622 static void action_trace(struct hist_trigger_data *hist_data,
 623                          struct tracing_map_elt *elt,
 624                          struct trace_buffer *buffer, void *rec,
 625                          struct ring_buffer_event *rbe, void *key,
 626                          struct action_data *data, u64 *var_ref_vals)
 627 {
 628         struct synth_event *event = data->synth_event;
 629
 630         trace_synth(event, var_ref_vals, data->var_ref_idx);
 631 }
 632
 633 struct hist_var_data {
 634         struct list_head list;
 635         struct hist_trigger_data *hist_data;
 636 };
 637
 638 static u64 hist_field_timestamp(struct hist_field *hist_field,
 639                                 struct tracing_map_elt *elt,
 640                                 struct trace_buffer *buffer,
 641                                 struct ring_buffer_event *rbe,
 642                                 void *event)
 643 {
 644         struct hist_trigger_data *hist_data = hist_field->hist_data;
 645         struct trace_array *tr = hist_data->event_file->tr;
 646
 647         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 648
 649         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 650                 ts = ns2usecs(ts);
 651
 652         return ts;
 653 }
 654
 655 static u64 hist_field_cpu(struct hist_field *hist_field,
 656                           struct tracing_map_elt *elt,
 657                           struct trace_buffer *buffer,
 658                           struct ring_buffer_event *rbe,
 659                           void *event)
 660 {
 661         int cpu = smp_processor_id();
 662
 663         return cpu;
 664 }
 665
 666 /**
 667  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 668  * @hist_field: The VAR_REF field to check
 669  * @var_data: The hist trigger that owns the variable
 670  * @var_idx: The trigger variable identifier
 671  *
 672  * Check the given VAR_REF field to see whether or not it references
 673  * the given variable associated with the given trigger.
 674  *
 675  * Return: The VAR_REF field if it does reference the variable, NULL if not
 676  */
 677 static struct hist_field *
 678 check_field_for_var_ref(struct hist_field *hist_field,
 679                         struct hist_trigger_data *var_data,
 680                         unsigned int var_idx)
 681 {
 682         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 683
 684         if (hist_field && hist_field->var.idx == var_idx &&
 685             hist_field->var.hist_data == var_data)
 686                 return hist_field;
 687
 688         return NULL;
 689 }
 690
 691 /**
 692  * find_var_ref - Check if a trigger has a reference to a trigger variable
 693  * @hist_data: The hist trigger that might have a reference to the variable
 694  * @var_data: The hist trigger that owns the variable
 695  * @var_idx: The trigger variable identifier
 696  *
 697  * Check the list of var_refs[] on the first hist trigger to see
 698  * whether any of them are references to the variable on the second
 699  * trigger.
 700  *
 701  * Return: The VAR_REF field referencing the variable if so, NULL if not
 702  */
 703 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 704                                        struct hist_trigger_data *var_data,
 705                                        unsigned int var_idx)
 706 {
 707         struct hist_field *hist_field;
 708         unsigned int i;
 709
 710         for (i = 0; i < hist_data->n_var_refs; i++) {
 711                 hist_field = hist_data->var_refs[i];
 712                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 713                         return hist_field;
 714         }
 715
 716         return NULL;
 717 }
 718
 719 /**
 720  * find_any_var_ref - Check if there is a reference to a given trigger variable
 721  * @hist_data: The hist trigger
 722  * @var_idx: The trigger variable identifier
 723  *
 724  * Check to see whether the given variable is currently referenced by
 725  * any other trigger.
 726  *
 727  * The trigger the variable is defined on is explicitly excluded - the
 728  * assumption being that a self-reference doesn't prevent a trigger
 729  * from being removed.
 730  *
 731  * Return: The VAR_REF field referencing the variable if so, NULL if not
 732  */
 733 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 734                                            unsigned int var_idx)
 735 {
 736         struct trace_array *tr = hist_data->event_file->tr;
 737         struct hist_field *found = NULL;
 738         struct hist_var_data *var_data;
 739
 740         list_for_each_entry(var_data, &tr->hist_vars, list) {
 741                 if (var_data->hist_data == hist_data)
 742                         continue;
 743                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 744                 if (found)
 745                         break;
 746         }
 747
 748         return found;
 749 }
 750
 751 /**
 752  * check_var_refs - Check if there is a reference to any of trigger's variables
 753  * @hist_data: The hist trigger
 754  *
 755  * A trigger can define one or more variables.  If any one of them is
 756  * currently referenced by any other trigger, this function will
 757  * determine that.
 758
 759  * Typically used to determine whether or not a trigger can be removed
 760  * - if there are any references to a trigger's variables, it cannot.
 761  *
 762  * Return: True if there is a reference to any of trigger's variables
 763  */
 764 static bool check_var_refs(struct hist_trigger_data *hist_data)
 765 {
 766         struct hist_field *field;
 767         bool found = false;
 768         int i;
 769
 770         for_each_hist_field(i, hist_data) {
 771                 field = hist_data->fields[i];
 772                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 773                         if (find_any_var_ref(hist_data, field->var.idx)) {
 774                                 found = true;
 775                                 break;
 776                         }
 777                 }
 778         }
 779
 780         return found;
 781 }
 782
 783 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 784 {
 785         struct trace_array *tr = hist_data->event_file->tr;
 786         struct hist_var_data *var_data, *found = NULL;
 787
 788         list_for_each_entry(var_data, &tr->hist_vars, list) {
 789                 if (var_data->hist_data == hist_data) {
 790                         found = var_data;
 791                         break;
 792                 }
 793         }
 794
 795         return found;
 796 }
 797
 798 static bool field_has_hist_vars(struct hist_field *hist_field,
 799                                 unsigned int level)
 800 {
 801         int i;
 802
 803         if (level > 3)
 804                 return false;
 805
 806         if (!hist_field)
 807                 return false;
 808
 809         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 810             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 811                 return true;
 812
 813         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 814                 struct hist_field *operand;
 815
 816                 operand = hist_field->operands[i];
 817                 if (field_has_hist_vars(operand, level + 1))
 818                         return true;
 819         }
 820
 821         return false;
 822 }
 823
 824 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 825 {
 826         struct hist_field *hist_field;
 827         int i;
 828
 829         for_each_hist_field(i, hist_data) {
 830                 hist_field = hist_data->fields[i];
 831                 if (field_has_hist_vars(hist_field, 0))
 832                         return true;
 833         }
 834
 835         return false;
 836 }
 837
 838 static int save_hist_vars(struct hist_trigger_data *hist_data)
 839 {
 840         struct trace_array *tr = hist_data->event_file->tr;
 841         struct hist_var_data *var_data;
 842
 843         var_data = find_hist_vars(hist_data);
 844         if (var_data)
 845                 return 0;
 846
 847         if (tracing_check_open_get_tr(tr))
 848                 return -ENODEV;
 849
 850         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 851         if (!var_data) {
 852                 trace_array_put(tr);
 853                 return -ENOMEM;
 854         }
 855
 856         var_data->hist_data = hist_data;
 857         list_add(&var_data->list, &tr->hist_vars);
 858
 859         return 0;
 860 }
 861
 862 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 863 {
 864         struct trace_array *tr = hist_data->event_file->tr;
 865         struct hist_var_data *var_data;
 866
 867         var_data = find_hist_vars(hist_data);
 868         if (!var_data)
 869                 return;
 870
 871         if (WARN_ON(check_var_refs(hist_data)))
 872                 return;
 873
 874         list_del(&var_data->list);
 875
 876         kfree(var_data);
 877
 878         trace_array_put(tr);
 879 }
 880
 881 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 882                                          const char *var_name)
 883 {
 884         struct hist_field *hist_field, *found = NULL;
 885         int i;
 886
 887         for_each_hist_field(i, hist_data) {
 888                 hist_field = hist_data->fields[i];
 889                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 890                     strcmp(hist_field->var.name, var_name) == 0) {
 891                         found = hist_field;
 892                         break;
 893                 }
 894         }
 895
 896         return found;
 897 }
 898
 899 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 900                                    struct trace_event_file *file,
 901                                    const char *var_name)
 902 {
 903         struct hist_trigger_data *test_data;
 904         struct event_trigger_data *test;
 905         struct hist_field *hist_field;
 906
 907         lockdep_assert_held(&event_mutex);
 908
 909         hist_field = find_var_field(hist_data, var_name);
 910         if (hist_field)
 911                 return hist_field;
 912
 913         list_for_each_entry(test, &file->triggers, list) {
 914                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 915                         test_data = test->private_data;
 916                         hist_field = find_var_field(test_data, var_name);
 917                         if (hist_field)
 918                                 return hist_field;
 919                 }
 920         }
 921
 922         return NULL;
 923 }
 924
 925 static struct trace_event_file *find_var_file(struct trace_array *tr,
 926                                               char *system,
 927                                               char *event_name,
 928                                               char *var_name)
 929 {
 930         struct hist_trigger_data *var_hist_data;
 931         struct hist_var_data *var_data;
 932         struct trace_event_file *file, *found = NULL;
 933
 934         if (system)
 935                 return find_event_file(tr, system, event_name);
 936
 937         list_for_each_entry(var_data, &tr->hist_vars, list) {
 938                 var_hist_data = var_data->hist_data;
 939                 file = var_hist_data->event_file;
 940                 if (file == found)
 941                         continue;
 942
 943                 if (find_var_field(var_hist_data, var_name)) {
 944                         if (found) {
 945                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 946                                 return NULL;
 947                         }
 948
 949                         found = file;
 950                 }
 951         }
 952
 953         return found;
 954 }
 955
 956 static struct hist_field *find_file_var(struct trace_event_file *file,
 957                                         const char *var_name)
 958 {
 959         struct hist_trigger_data *test_data;
 960         struct event_trigger_data *test;
 961         struct hist_field *hist_field;
 962
 963         lockdep_assert_held(&event_mutex);
 964
 965         list_for_each_entry(test, &file->triggers, list) {
 966                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 967                         test_data = test->private_data;
 968                         hist_field = find_var_field(test_data, var_name);
 969                         if (hist_field)
 970                                 return hist_field;
 971                 }
 972         }
 973
 974         return NULL;
 975 }
 976
 977 static struct hist_field *
 978 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 979 {
 980         struct trace_array *tr = hist_data->event_file->tr;
 981         struct hist_field *hist_field, *found = NULL;
 982         struct trace_event_file *file;
 983         unsigned int i;
 984
 985         for (i = 0; i < hist_data->n_actions; i++) {
 986                 struct action_data *data = hist_data->actions[i];
 987
 988                 if (data->handler == HANDLER_ONMATCH) {
 989                         char *system = data->match_data.event_system;
 990                         char *event_name = data->match_data.event;
 991
 992                         file = find_var_file(tr, system, event_name, var_name);
 993                         if (!file)
 994                                 continue;
 995                         hist_field = find_file_var(file, var_name);
 996                         if (hist_field) {
 997                                 if (found) {
 998                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
 999                                                  errpos(var_name));
1000                                         return ERR_PTR(-EINVAL);
1001                                 }
1002
1003                                 found = hist_field;
1004                         }
1005                 }
1006         }
1007         return found;
1008 }
1009
1010 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1011                                          char *system,
1012                                          char *event_name,
1013                                          char *var_name)
1014 {
1015         struct trace_array *tr = hist_data->event_file->tr;
1016         struct hist_field *hist_field = NULL;
1017         struct trace_event_file *file;
1018
1019         if (!system || !event_name) {
1020                 hist_field = find_match_var(hist_data, var_name);
1021                 if (IS_ERR(hist_field))
1022                         return NULL;
1023                 if (hist_field)
1024                         return hist_field;
1025         }
1026
1027         file = find_var_file(tr, system, event_name, var_name);
1028         if (!file)
1029                 return NULL;
1030
1031         hist_field = find_file_var(file, var_name);
1032
1033         return hist_field;
1034 }
1035
1036 static u64 hist_field_var_ref(struct hist_field *hist_field,
1037                               struct tracing_map_elt *elt,
1038                               struct trace_buffer *buffer,
1039                               struct ring_buffer_event *rbe,
1040                               void *event)
1041 {
1042         struct hist_elt_data *elt_data;
1043         u64 var_val = 0;
1044
1045         if (WARN_ON_ONCE(!elt))
1046                 return var_val;
1047
1048         elt_data = elt->private_data;
1049         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1050
1051         return var_val;
1052 }
1053
1054 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1055                              u64 *var_ref_vals, bool self)
1056 {
1057         struct hist_trigger_data *var_data;
1058         struct tracing_map_elt *var_elt;
1059         struct hist_field *hist_field;
1060         unsigned int i, var_idx;
1061         bool resolved = true;
1062         u64 var_val = 0;
1063
1064         for (i = 0; i < hist_data->n_var_refs; i++) {
1065                 hist_field = hist_data->var_refs[i];
1066                 var_idx = hist_field->var.idx;
1067                 var_data = hist_field->var.hist_data;
1068
1069                 if (var_data == NULL) {
1070                         resolved = false;
1071                         break;
1072                 }
1073
1074                 if ((self && var_data != hist_data) ||
1075                     (!self && var_data == hist_data))
1076                         continue;
1077
1078                 var_elt = tracing_map_lookup(var_data->map, key);
1079                 if (!var_elt) {
1080                         resolved = false;
1081                         break;
1082                 }
1083
1084                 if (!tracing_map_var_set(var_elt, var_idx)) {
1085                         resolved = false;
1086                         break;
1087                 }
1088
1089                 if (self || !hist_field->read_once)
1090                         var_val = tracing_map_read_var(var_elt, var_idx);
1091                 else
1092                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1093
1094                 var_ref_vals[i] = var_val;
1095         }
1096
1097         return resolved;
1098 }
1099
1100 static const char *hist_field_name(struct hist_field *field,
1101                                    unsigned int level)
1102 {
1103         const char *field_name = "";
1104
1105         if (level > 1)
1106                 return field_name;
1107
1108         if (field->field)
1109                 field_name = field->field->name;
1110         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1111                  field->flags & HIST_FIELD_FL_ALIAS)
1112                 field_name = hist_field_name(field->operands[0], ++level);
1113         else if (field->flags & HIST_FIELD_FL_CPU)
1114                 field_name = "common_cpu";
1115         else if (field->flags & HIST_FIELD_FL_EXPR ||
1116                  field->flags & HIST_FIELD_FL_VAR_REF) {
1117                 if (field->system) {
1118                         static char full_name[MAX_FILTER_STR_VAL];
1119
1120                         strcat(full_name, field->system);
1121                         strcat(full_name, ".");
1122                         strcat(full_name, field->event_name);
1123                         strcat(full_name, ".");
1124                         strcat(full_name, field->name);
1125                         field_name = full_name;
1126                 } else
1127                         field_name = field->name;
1128         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1129                 field_name = "common_timestamp";
1130
1131         if (field_name == NULL)
1132                 field_name = "";
1133
1134         return field_name;
1135 }
1136
1137 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1138 {
1139         hist_field_fn_t fn = NULL;
1140
1141         switch (field_size) {
1142         case 8:
1143                 if (field_is_signed)
1144                         fn = hist_field_s64;
1145                 else
1146                         fn = hist_field_u64;
1147                 break;
1148         case 4:
1149                 if (field_is_signed)
1150                         fn = hist_field_s32;
1151                 else
1152                         fn = hist_field_u32;
1153                 break;
1154         case 2:
1155                 if (field_is_signed)
1156                         fn = hist_field_s16;
1157                 else
1158                         fn = hist_field_u16;
1159                 break;
1160         case 1:
1161                 if (field_is_signed)
1162                         fn = hist_field_s8;
1163                 else
1164                         fn = hist_field_u8;
1165                 break;
1166         }
1167
1168         return fn;
1169 }
1170
1171 static int parse_map_size(char *str)
1172 {
1173         unsigned long size, map_bits;
1174         int ret;
1175
1176         ret = kstrtoul(str, 0, &size);
1177         if (ret)
1178                 goto out;
1179
1180         map_bits = ilog2(roundup_pow_of_two(size));
1181         if (map_bits < TRACING_MAP_BITS_MIN ||
1182             map_bits > TRACING_MAP_BITS_MAX)
1183                 ret = -EINVAL;
1184         else
1185                 ret = map_bits;
1186  out:
1187         return ret;
1188 }
1189
1190 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1191 {
1192         unsigned int i;
1193
1194         if (!attrs)
1195                 return;
1196
1197         for (i = 0; i < attrs->n_assignments; i++)
1198                 kfree(attrs->assignment_str[i]);
1199
1200         for (i = 0; i < attrs->n_actions; i++)
1201                 kfree(attrs->action_str[i]);
1202
1203         kfree(attrs->name);
1204         kfree(attrs->sort_key_str);
1205         kfree(attrs->keys_str);
1206         kfree(attrs->vals_str);
1207         kfree(attrs->clock);
1208         kfree(attrs);
1209 }
1210
1211 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1212 {
1213         int ret = -EINVAL;
1214
1215         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1216                 return ret;
1217
1218         if ((str_has_prefix(str, "onmatch(")) ||
1219             (str_has_prefix(str, "onmax(")) ||
1220             (str_has_prefix(str, "onchange("))) {
1221                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1222                 if (!attrs->action_str[attrs->n_actions]) {
1223                         ret = -ENOMEM;
1224                         return ret;
1225                 }
1226                 attrs->n_actions++;
1227                 ret = 0;
1228         }
1229         return ret;
1230 }
1231
1232 static int parse_assignment(struct trace_array *tr,
1233                             char *str, struct hist_trigger_attrs *attrs)
1234 {
1235         int len, ret = 0;
1236
1237         if ((len = str_has_prefix(str, "key=")) ||
1238             (len = str_has_prefix(str, "keys="))) {
1239                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1240                 if (!attrs->keys_str) {
1241                         ret = -ENOMEM;
1242                         goto out;
1243                 }
1244         } else if ((len = str_has_prefix(str, "val=")) ||
1245                    (len = str_has_prefix(str, "vals=")) ||
1246                    (len = str_has_prefix(str, "values="))) {
1247                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1248                 if (!attrs->vals_str) {
1249                         ret = -ENOMEM;
1250                         goto out;
1251                 }
1252         } else if ((len = str_has_prefix(str, "sort="))) {
1253                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1254                 if (!attrs->sort_key_str) {
1255                         ret = -ENOMEM;
1256                         goto out;
1257                 }
1258         } else if (str_has_prefix(str, "name=")) {
1259                 attrs->name = kstrdup(str, GFP_KERNEL);
1260                 if (!attrs->name) {
1261                         ret = -ENOMEM;
1262                         goto out;
1263                 }
1264         } else if ((len = str_has_prefix(str, "clock="))) {
1265                 str += len;
1266
1267                 str = strstrip(str);
1268                 attrs->clock = kstrdup(str, GFP_KERNEL);
1269                 if (!attrs->clock) {
1270                         ret = -ENOMEM;
1271                         goto out;
1272                 }
1273         } else if ((len = str_has_prefix(str, "size="))) {
1274                 int map_bits = parse_map_size(str + len);
1275
1276                 if (map_bits < 0) {
1277                         ret = map_bits;
1278                         goto out;
1279                 }
1280                 attrs->map_bits = map_bits;
1281         } else {
1282                 char *assignment;
1283
1284                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1285                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1286                         ret = -EINVAL;
1287                         goto out;
1288                 }
1289
1290                 assignment = kstrdup(str, GFP_KERNEL);
1291                 if (!assignment) {
1292                         ret = -ENOMEM;
1293                         goto out;
1294                 }
1295
1296                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1297         }
1298  out:
1299         return ret;
1300 }
1301
1302 static struct hist_trigger_attrs *
1303 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1304 {
1305         struct hist_trigger_attrs *attrs;
1306         int ret = 0;
1307
1308         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1309         if (!attrs)
1310                 return ERR_PTR(-ENOMEM);
1311
1312         while (trigger_str) {
1313                 char *str = strsep(&trigger_str, ":");
1314                 char *rhs;
1315
1316                 rhs = strchr(str, '=');
1317                 if (rhs) {
1318                         if (!strlen(++rhs)) {
1319                                 ret = -EINVAL;
1320                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1321                                 goto free;
1322                         }
1323                         ret = parse_assignment(tr, str, attrs);
1324                         if (ret)
1325                                 goto free;
1326                 } else if (strcmp(str, "pause") == 0)
1327                         attrs->pause = true;
1328                 else if ((strcmp(str, "cont") == 0) ||
1329                          (strcmp(str, "continue") == 0))
1330                         attrs->cont = true;
1331                 else if (strcmp(str, "clear") == 0)
1332                         attrs->clear = true;
1333                 else {
1334                         ret = parse_action(str, attrs);
1335                         if (ret)
1336                                 goto free;
1337                 }
1338         }
1339
1340         if (!attrs->keys_str) {
1341                 ret = -EINVAL;
1342                 goto free;
1343         }
1344
1345         if (!attrs->clock) {
1346                 attrs->clock = kstrdup("global", GFP_KERNEL);
1347                 if (!attrs->clock) {
1348                         ret = -ENOMEM;
1349                         goto free;
1350                 }
1351         }
1352
1353         return attrs;
1354  free:
1355         destroy_hist_trigger_attrs(attrs);
1356
1357         return ERR_PTR(ret);
1358 }
1359
1360 static inline void save_comm(char *comm, struct task_struct *task)
1361 {
1362         if (!task->pid) {
1363                 strcpy(comm, "<idle>");
1364                 return;
1365         }
1366
1367         if (WARN_ON_ONCE(task->pid < 0)) {
1368                 strcpy(comm, "<XXX>");
1369                 return;
1370         }
1371
1372         strncpy(comm, task->comm, TASK_COMM_LEN);
1373 }
1374
1375 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1376 {
1377         unsigned int i;
1378
1379         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1380                 kfree(elt_data->field_var_str[i]);
1381
1382         kfree(elt_data->comm);
1383         kfree(elt_data);
1384 }
1385
1386 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1387 {
1388         struct hist_elt_data *elt_data = elt->private_data;
1389
1390         hist_elt_data_free(elt_data);
1391 }
1392
1393 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1394 {
1395         struct hist_trigger_data *hist_data = elt->map->private_data;
1396         unsigned int size = TASK_COMM_LEN;
1397         struct hist_elt_data *elt_data;
1398         struct hist_field *key_field;
1399         unsigned int i, n_str;
1400
1401         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1402         if (!elt_data)
1403                 return -ENOMEM;
1404
1405         for_each_hist_key_field(i, hist_data) {
1406                 key_field = hist_data->fields[i];
1407
1408                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1409                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1410                         if (!elt_data->comm) {
1411                                 kfree(elt_data);
1412                                 return -ENOMEM;
1413                         }
1414                         break;
1415                 }
1416         }
1417
1418         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1419                 hist_data->n_var_str;
1420         if (n_str > SYNTH_FIELDS_MAX) {
1421                 hist_elt_data_free(elt_data);
1422                 return -EINVAL;
1423         }
1424
1425         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1426
1427         size = STR_VAR_LEN_MAX;
1428
1429         for (i = 0; i < n_str; i++) {
1430                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1431                 if (!elt_data->field_var_str[i]) {
1432                         hist_elt_data_free(elt_data);
1433                         return -ENOMEM;
1434                 }
1435         }
1436
1437         elt->private_data = elt_data;
1438
1439         return 0;
1440 }
1441
1442 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1443 {
1444         struct hist_elt_data *elt_data = elt->private_data;
1445
1446         if (elt_data->comm)
1447                 save_comm(elt_data->comm, current);
1448 }
1449
1450 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1451         .elt_alloc      = hist_trigger_elt_data_alloc,
1452         .elt_free       = hist_trigger_elt_data_free,
1453         .elt_init       = hist_trigger_elt_data_init,
1454 };
1455
1456 static const char *get_hist_field_flags(struct hist_field *hist_field)
1457 {
1458         const char *flags_str = NULL;
1459
1460         if (hist_field->flags & HIST_FIELD_FL_HEX)
1461                 flags_str = "hex";
1462         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1463                 flags_str = "sym";
1464         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1465                 flags_str = "sym-offset";
1466         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1467                 flags_str = "execname";
1468         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1469                 flags_str = "syscall";
1470         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1471                 flags_str = "log2";
1472         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1473                 flags_str = "usecs";
1474
1475         return flags_str;
1476 }
1477
1478 static void expr_field_str(struct hist_field *field, char *expr)
1479 {
1480         if (field->flags & HIST_FIELD_FL_VAR_REF)
1481                 strcat(expr, "$");
1482
1483         strcat(expr, hist_field_name(field, 0));
1484
1485         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1486                 const char *flags_str = get_hist_field_flags(field);
1487
1488                 if (flags_str) {
1489                         strcat(expr, ".");
1490                         strcat(expr, flags_str);
1491                 }
1492         }
1493 }
1494
1495 static char *expr_str(struct hist_field *field, unsigned int level)
1496 {
1497         char *expr;
1498
1499         if (level > 1)
1500                 return NULL;
1501
1502         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1503         if (!expr)
1504                 return NULL;
1505
1506         if (!field->operands[0]) {
1507                 expr_field_str(field, expr);
1508                 return expr;
1509         }
1510
1511         if (field->operator == FIELD_OP_UNARY_MINUS) {
1512                 char *subexpr;
1513
1514                 strcat(expr, "-(");
1515                 subexpr = expr_str(field->operands[0], ++level);
1516                 if (!subexpr) {
1517                         kfree(expr);
1518                         return NULL;
1519                 }
1520                 strcat(expr, subexpr);
1521                 strcat(expr, ")");
1522
1523                 kfree(subexpr);
1524
1525                 return expr;
1526         }
1527
1528         expr_field_str(field->operands[0], expr);
1529
1530         switch (field->operator) {
1531         case FIELD_OP_MINUS:
1532                 strcat(expr, "-");
1533                 break;
1534         case FIELD_OP_PLUS:
1535                 strcat(expr, "+");
1536                 break;
1537         default:
1538                 kfree(expr);
1539                 return NULL;
1540         }
1541
1542         expr_field_str(field->operands[1], expr);
1543
1544         return expr;
1545 }
1546
1547 static int contains_operator(char *str)
1548 {
1549         enum field_op_id field_op = FIELD_OP_NONE;
1550         char *op;
1551
1552         op = strpbrk(str, "+-");
1553         if (!op)
1554                 return FIELD_OP_NONE;
1555
1556         switch (*op) {
1557         case '-':
1558                 /*
1559                  * Unfortunately, the modifier ".sym-offset"
1560                  * can confuse things.
1561                  */
1562                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1563                         return FIELD_OP_NONE;
1564
1565                 if (*str == '-')
1566                         field_op = FIELD_OP_UNARY_MINUS;
1567                 else
1568                         field_op = FIELD_OP_MINUS;
1569                 break;
1570         case '+':
1571                 field_op = FIELD_OP_PLUS;
1572                 break;
1573         default:
1574                 break;
1575         }
1576
1577         return field_op;
1578 }
1579
1580 static void get_hist_field(struct hist_field *hist_field)
1581 {
1582         hist_field->ref++;
1583 }
1584
1585 static void __destroy_hist_field(struct hist_field *hist_field)
1586 {
1587         if (--hist_field->ref > 1)
1588                 return;
1589
1590         kfree(hist_field->var.name);
1591         kfree(hist_field->name);
1592         kfree(hist_field->type);
1593
1594         kfree(hist_field->system);
1595         kfree(hist_field->event_name);
1596
1597         kfree(hist_field);
1598 }
1599
1600 static void destroy_hist_field(struct hist_field *hist_field,
1601                                unsigned int level)
1602 {
1603         unsigned int i;
1604
1605         if (level > 3)
1606                 return;
1607
1608         if (!hist_field)
1609                 return;
1610
1611         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1612                 return; /* var refs will be destroyed separately */
1613
1614         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1615                 destroy_hist_field(hist_field->operands[i], level + 1);
1616
1617         __destroy_hist_field(hist_field);
1618 }
1619
1620 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1621                                             struct ftrace_event_field *field,
1622                                             unsigned long flags,
1623                                             char *var_name)
1624 {
1625         struct hist_field *hist_field;
1626
1627         if (field && is_function_field(field))
1628                 return NULL;
1629
1630         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1631         if (!hist_field)
1632                 return NULL;
1633
1634         hist_field->ref = 1;
1635
1636         hist_field->hist_data = hist_data;
1637
1638         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1639                 goto out; /* caller will populate */
1640
1641         if (flags & HIST_FIELD_FL_VAR_REF) {
1642                 hist_field->fn = hist_field_var_ref;
1643                 goto out;
1644         }
1645
1646         if (flags & HIST_FIELD_FL_HITCOUNT) {
1647                 hist_field->fn = hist_field_counter;
1648                 hist_field->size = sizeof(u64);
1649                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1650                 if (!hist_field->type)
1651                         goto free;
1652                 goto out;
1653         }
1654
1655         if (flags & HIST_FIELD_FL_STACKTRACE) {
1656                 hist_field->fn = hist_field_none;
1657                 goto out;
1658         }
1659
1660         if (flags & HIST_FIELD_FL_LOG2) {
1661                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1662                 hist_field->fn = hist_field_log2;
1663                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1664                 hist_field->size = hist_field->operands[0]->size;
1665                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1666                 if (!hist_field->type)
1667                         goto free;
1668                 goto out;
1669         }
1670
1671         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1672                 hist_field->fn = hist_field_timestamp;
1673                 hist_field->size = sizeof(u64);
1674                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1675                 if (!hist_field->type)
1676                         goto free;
1677                 goto out;
1678         }
1679
1680         if (flags & HIST_FIELD_FL_CPU) {
1681                 hist_field->fn = hist_field_cpu;
1682                 hist_field->size = sizeof(int);
1683                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1684                 if (!hist_field->type)
1685                         goto free;
1686                 goto out;
1687         }
1688
1689         if (WARN_ON_ONCE(!field))
1690                 goto out;
1691
1692         /* Pointers to strings are just pointers and dangerous to dereference */
1693         if (is_string_field(field) &&
1694             (field->filter_type != FILTER_PTR_STRING)) {
1695                 flags |= HIST_FIELD_FL_STRING;
1696
1697                 hist_field->size = MAX_FILTER_STR_VAL;
1698                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1699                 if (!hist_field->type)
1700                         goto free;
1701
1702                 if (field->filter_type == FILTER_STATIC_STRING)
1703                         hist_field->fn = hist_field_string;
1704                 else if (field->filter_type == FILTER_DYN_STRING)
1705                         hist_field->fn = hist_field_dynstring;
1706                 else
1707                         hist_field->fn = hist_field_pstring;
1708         } else {
1709                 hist_field->size = field->size;
1710                 hist_field->is_signed = field->is_signed;
1711                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1712                 if (!hist_field->type)
1713                         goto free;
1714
1715                 hist_field->fn = select_value_fn(field->size,
1716                                                  field->is_signed);
1717                 if (!hist_field->fn) {
1718                         destroy_hist_field(hist_field, 0);
1719                         return NULL;
1720                 }
1721         }
1722  out:
1723         hist_field->field = field;
1724         hist_field->flags = flags;
1725
1726         if (var_name) {
1727                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1728                 if (!hist_field->var.name)
1729                         goto free;
1730         }
1731
1732         return hist_field;
1733  free:
1734         destroy_hist_field(hist_field, 0);
1735         return NULL;
1736 }
1737
1738 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1739 {
1740         unsigned int i;
1741
1742         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1743                 if (hist_data->fields[i]) {
1744                         destroy_hist_field(hist_data->fields[i], 0);
1745                         hist_data->fields[i] = NULL;
1746                 }
1747         }
1748
1749         for (i = 0; i < hist_data->n_var_refs; i++) {
1750                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1751                 __destroy_hist_field(hist_data->var_refs[i]);
1752                 hist_data->var_refs[i] = NULL;
1753         }
1754 }
1755
1756 static int init_var_ref(struct hist_field *ref_field,
1757                         struct hist_field *var_field,
1758                         char *system, char *event_name)
1759 {
1760         int err = 0;
1761
1762         ref_field->var.idx = var_field->var.idx;
1763         ref_field->var.hist_data = var_field->hist_data;
1764         ref_field->size = var_field->size;
1765         ref_field->is_signed = var_field->is_signed;
1766         ref_field->flags |= var_field->flags &
1767                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1768
1769         if (system) {
1770                 ref_field->system = kstrdup(system, GFP_KERNEL);
1771                 if (!ref_field->system)
1772                         return -ENOMEM;
1773         }
1774
1775         if (event_name) {
1776                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1777                 if (!ref_field->event_name) {
1778                         err = -ENOMEM;
1779                         goto free;
1780                 }
1781         }
1782
1783         if (var_field->var.name) {
1784                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1785                 if (!ref_field->name) {
1786                         err = -ENOMEM;
1787                         goto free;
1788                 }
1789         } else if (var_field->name) {
1790                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1791                 if (!ref_field->name) {
1792                         err = -ENOMEM;
1793                         goto free;
1794                 }
1795         }
1796
1797         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1798         if (!ref_field->type) {
1799                 err = -ENOMEM;
1800                 goto free;
1801         }
1802  out:
1803         return err;
1804  free:
1805         kfree(ref_field->system);
1806         kfree(ref_field->event_name);
1807         kfree(ref_field->name);
1808
1809         goto out;
1810 }
1811
1812 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1813                             struct hist_field *var_field)
1814 {
1815         struct hist_field *ref_field;
1816         int i;
1817
1818         for (i = 0; i < hist_data->n_var_refs; i++) {
1819                 ref_field = hist_data->var_refs[i];
1820                 if (ref_field->var.idx == var_field->var.idx &&
1821                     ref_field->var.hist_data == var_field->hist_data)
1822                         return i;
1823         }
1824
1825         return -ENOENT;
1826 }
1827
1828 /**
1829  * create_var_ref - Create a variable reference and attach it to trigger
1830  * @hist_data: The trigger that will be referencing the variable
1831  * @var_field: The VAR field to create a reference to
1832  * @system: The optional system string
1833  * @event_name: The optional event_name string
1834  *
1835  * Given a variable hist_field, create a VAR_REF hist_field that
1836  * represents a reference to it.
1837  *
1838  * This function also adds the reference to the trigger that
1839  * now references the variable.
1840  *
1841  * Return: The VAR_REF field if successful, NULL if not
1842  */
1843 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1844                                          struct hist_field *var_field,
1845                                          char *system, char *event_name)
1846 {
1847         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1848         struct hist_field *ref_field;
1849         int i;
1850
1851         /* Check if the variable already exists */
1852         for (i = 0; i < hist_data->n_var_refs; i++) {
1853                 ref_field = hist_data->var_refs[i];
1854                 if (ref_field->var.idx == var_field->var.idx &&
1855                     ref_field->var.hist_data == var_field->hist_data) {
1856                         get_hist_field(ref_field);
1857                         return ref_field;
1858                 }
1859         }
1860
1861         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1862         if (ref_field) {
1863                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1864                         destroy_hist_field(ref_field, 0);
1865                         return NULL;
1866                 }
1867
1868                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1869                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1870         }
1871
1872         return ref_field;
1873 }
1874
1875 static bool is_var_ref(char *var_name)
1876 {
1877         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1878                 return false;
1879
1880         return true;
1881 }
1882
1883 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1884                                  char *var_name)
1885 {
1886         char *name, *field;
1887         unsigned int i;
1888
1889         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1890                 name = hist_data->attrs->var_defs.name[i];
1891
1892                 if (strcmp(var_name, name) == 0) {
1893                         field = hist_data->attrs->var_defs.expr[i];
1894                         if (contains_operator(field) || is_var_ref(field))
1895                                 continue;
1896                         return field;
1897                 }
1898         }
1899
1900         return NULL;
1901 }
1902
1903 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1904                                  char *system, char *event_name,
1905                                  char *var_name)
1906 {
1907         struct trace_event_call *call;
1908
1909         if (system && event_name) {
1910                 call = hist_data->event_file->event_call;
1911
1912                 if (strcmp(system, call->class->system) != 0)
1913                         return NULL;
1914
1915                 if (strcmp(event_name, trace_event_name(call)) != 0)
1916                         return NULL;
1917         }
1918
1919         if (!!system != !!event_name)
1920                 return NULL;
1921
1922         if (!is_var_ref(var_name))
1923                 return NULL;
1924
1925         var_name++;
1926
1927         return field_name_from_var(hist_data, var_name);
1928 }
1929
1930 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1931                                         char *system, char *event_name,
1932                                         char *var_name)
1933 {
1934         struct hist_field *var_field = NULL, *ref_field = NULL;
1935         struct trace_array *tr = hist_data->event_file->tr;
1936
1937         if (!is_var_ref(var_name))
1938                 return NULL;
1939
1940         var_name++;
1941
1942         var_field = find_event_var(hist_data, system, event_name, var_name);
1943         if (var_field)
1944                 ref_field = create_var_ref(hist_data, var_field,
1945                                            system, event_name);
1946
1947         if (!ref_field)
1948                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1949
1950         return ref_field;
1951 }
1952
1953 static struct ftrace_event_field *
1954 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1955             char *field_str, unsigned long *flags)
1956 {
1957         struct ftrace_event_field *field = NULL;
1958         char *field_name, *modifier, *str;
1959         struct trace_array *tr = file->tr;
1960
1961         modifier = str = kstrdup(field_str, GFP_KERNEL);
1962         if (!modifier)
1963                 return ERR_PTR(-ENOMEM);
1964
1965         field_name = strsep(&modifier, ".");
1966         if (modifier) {
1967                 if (strcmp(modifier, "hex") == 0)
1968                         *flags |= HIST_FIELD_FL_HEX;
1969                 else if (strcmp(modifier, "sym") == 0)
1970                         *flags |= HIST_FIELD_FL_SYM;
1971                 else if (strcmp(modifier, "sym-offset") == 0)
1972                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1973                 else if ((strcmp(modifier, "execname") == 0) &&
1974                          (strcmp(field_name, "common_pid") == 0))
1975                         *flags |= HIST_FIELD_FL_EXECNAME;
1976                 else if (strcmp(modifier, "syscall") == 0)
1977                         *flags |= HIST_FIELD_FL_SYSCALL;
1978                 else if (strcmp(modifier, "log2") == 0)
1979                         *flags |= HIST_FIELD_FL_LOG2;
1980                 else if (strcmp(modifier, "usecs") == 0)
1981                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1982                 else {
1983                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1984                         field = ERR_PTR(-EINVAL);
1985                         goto out;
1986                 }
1987         }
1988
1989         if (strcmp(field_name, "common_timestamp") == 0) {
1990                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1991                 hist_data->enable_timestamps = true;
1992                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1993                         hist_data->attrs->ts_in_usecs = true;
1994         } else if (strcmp(field_name, "common_cpu") == 0)
1995                 *flags |= HIST_FIELD_FL_CPU;
1996         else {
1997                 field = trace_find_event_field(file->event_call, field_name);
1998                 if (!field || !field->size) {
1999                         /*
2000                          * For backward compatibility, if field_name
2001                          * was "cpu", then we treat this the same as
2002                          * common_cpu.
2003                          */
2004                         if (strcmp(field_name, "cpu") == 0) {
2005                                 *flags |= HIST_FIELD_FL_CPU;
2006                         } else {
2007                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2008                                          errpos(field_name));
2009                                 field = ERR_PTR(-EINVAL);
2010                                 goto out;
2011                         }
2012                 }
2013         }
2014  out:
2015         kfree(str);
2016
2017         return field;
2018 }
2019
2020 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2021                                        struct hist_field *var_ref,
2022                                        char *var_name)
2023 {
2024         struct hist_field *alias = NULL;
2025         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2026
2027         alias = create_hist_field(hist_data, NULL, flags, var_name);
2028         if (!alias)
2029                 return NULL;
2030
2031         alias->fn = var_ref->fn;
2032         alias->operands[0] = var_ref;
2033
2034         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2035                 destroy_hist_field(alias, 0);
2036                 return NULL;
2037         }
2038
2039         alias->var_ref_idx = var_ref->var_ref_idx;
2040
2041         return alias;
2042 }
2043
2044 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2045                                      struct trace_event_file *file, char *str,
2046                                      unsigned long *flags, char *var_name)
2047 {
2048         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2049         struct ftrace_event_field *field = NULL;
2050         struct hist_field *hist_field = NULL;
2051         int ret = 0;
2052
2053         s = strchr(str, '.');
2054         if (s) {
2055                 s = strchr(++s, '.');
2056                 if (s) {
2057                         ref_system = strsep(&str, ".");
2058                         if (!str) {
2059                                 ret = -EINVAL;
2060                                 goto out;
2061                         }
2062                         ref_event = strsep(&str, ".");
2063                         if (!str) {
2064                                 ret = -EINVAL;
2065                                 goto out;
2066                         }
2067                         ref_var = str;
2068                 }
2069         }
2070
2071         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2072         if (!s) {
2073                 hist_field = parse_var_ref(hist_data, ref_system,
2074                                            ref_event, ref_var);
2075                 if (hist_field) {
2076                         if (var_name) {
2077                                 hist_field = create_alias(hist_data, hist_field, var_name);
2078                                 if (!hist_field) {
2079                                         ret = -ENOMEM;
2080                                         goto out;
2081                                 }
2082                         }
2083                         return hist_field;
2084                 }
2085         } else
2086                 str = s;
2087
2088         field = parse_field(hist_data, file, str, flags);
2089         if (IS_ERR(field)) {
2090                 ret = PTR_ERR(field);
2091                 goto out;
2092         }
2093
2094         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2095         if (!hist_field) {
2096                 ret = -ENOMEM;
2097                 goto out;
2098         }
2099
2100         return hist_field;
2101  out:
2102         return ERR_PTR(ret);
2103 }
2104
2105 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2106                                      struct trace_event_file *file,
2107                                      char *str, unsigned long flags,
2108                                      char *var_name, unsigned int level);
2109
2110 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2111                                       struct trace_event_file *file,
2112                                       char *str, unsigned long flags,
2113                                       char *var_name, unsigned int level)
2114 {
2115         struct hist_field *operand1, *expr = NULL;
2116         unsigned long operand_flags;
2117         int ret = 0;
2118         char *s;
2119
2120         /* we support only -(xxx) i.e. explicit parens required */
2121
2122         if (level > 3) {
2123                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2124                 ret = -EINVAL;
2125                 goto free;
2126         }
2127
2128         str++; /* skip leading '-' */
2129
2130         s = strchr(str, '(');
2131         if (s)
2132                 str++;
2133         else {
2134                 ret = -EINVAL;
2135                 goto free;
2136         }
2137
2138         s = strrchr(str, ')');
2139         if (s)
2140                 *s = '\0';
2141         else {
2142                 ret = -EINVAL; /* no closing ')' */
2143                 goto free;
2144         }
2145
2146         flags |= HIST_FIELD_FL_EXPR;
2147         expr = create_hist_field(hist_data, NULL, flags, var_name);
2148         if (!expr) {
2149                 ret = -ENOMEM;
2150                 goto free;
2151         }
2152
2153         operand_flags = 0;
2154         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2155         if (IS_ERR(operand1)) {
2156                 ret = PTR_ERR(operand1);
2157                 goto free;
2158         }
2159
2160         expr->flags |= operand1->flags &
2161                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2162         expr->fn = hist_field_unary_minus;
2163         expr->operands[0] = operand1;
2164         expr->operator = FIELD_OP_UNARY_MINUS;
2165         expr->name = expr_str(expr, 0);
2166         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2167         if (!expr->type) {
2168                 ret = -ENOMEM;
2169                 goto free;
2170         }
2171
2172         return expr;
2173  free:
2174         destroy_hist_field(expr, 0);
2175         return ERR_PTR(ret);
2176 }
2177
2178 static int check_expr_operands(struct trace_array *tr,
2179                                struct hist_field *operand1,
2180                                struct hist_field *operand2)
2181 {
2182         unsigned long operand1_flags = operand1->flags;
2183         unsigned long operand2_flags = operand2->flags;
2184
2185         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2186             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2187                 struct hist_field *var;
2188
2189                 var = find_var_field(operand1->var.hist_data, operand1->name);
2190                 if (!var)
2191                         return -EINVAL;
2192                 operand1_flags = var->flags;
2193         }
2194
2195         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2196             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2197                 struct hist_field *var;
2198
2199                 var = find_var_field(operand2->var.hist_data, operand2->name);
2200                 if (!var)
2201                         return -EINVAL;
2202                 operand2_flags = var->flags;
2203         }
2204
2205         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2206             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2207                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2208                 return -EINVAL;
2209         }
2210
2211         return 0;
2212 }
2213
2214 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2215                                      struct trace_event_file *file,
2216                                      char *str, unsigned long flags,
2217                                      char *var_name, unsigned int level)
2218 {
2219         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2220         unsigned long operand_flags;
2221         int field_op, ret = -EINVAL;
2222         char *sep, *operand1_str;
2223
2224         if (level > 3) {
2225                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2226                 return ERR_PTR(-EINVAL);
2227         }
2228
2229         field_op = contains_operator(str);
2230
2231         if (field_op == FIELD_OP_NONE)
2232                 return parse_atom(hist_data, file, str, &flags, var_name);
2233
2234         if (field_op == FIELD_OP_UNARY_MINUS)
2235                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2236
2237         switch (field_op) {
2238         case FIELD_OP_MINUS:
2239                 sep = "-";
2240                 break;
2241         case FIELD_OP_PLUS:
2242                 sep = "+";
2243                 break;
2244         default:
2245                 goto free;
2246         }
2247
2248         operand1_str = strsep(&str, sep);
2249         if (!operand1_str || !str)
2250                 goto free;
2251
2252         operand_flags = 0;
2253         operand1 = parse_atom(hist_data, file, operand1_str,
2254                               &operand_flags, NULL);
2255         if (IS_ERR(operand1)) {
2256                 ret = PTR_ERR(operand1);
2257                 operand1 = NULL;
2258                 goto free;
2259         }
2260
2261         /* rest of string could be another expression e.g. b+c in a+b+c */
2262         operand_flags = 0;
2263         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2264         if (IS_ERR(operand2)) {
2265                 ret = PTR_ERR(operand2);
2266                 operand2 = NULL;
2267                 goto free;
2268         }
2269
2270         ret = check_expr_operands(file->tr, operand1, operand2);
2271         if (ret)
2272                 goto free;
2273
2274         flags |= HIST_FIELD_FL_EXPR;
2275
2276         flags |= operand1->flags &
2277                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2278
2279         expr = create_hist_field(hist_data, NULL, flags, var_name);
2280         if (!expr) {
2281                 ret = -ENOMEM;
2282                 goto free;
2283         }
2284
2285         operand1->read_once = true;
2286         operand2->read_once = true;
2287
2288         expr->operands[0] = operand1;
2289         expr->operands[1] = operand2;
2290         expr->operator = field_op;
2291         expr->name = expr_str(expr, 0);
2292         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2293         if (!expr->type) {
2294                 ret = -ENOMEM;
2295                 goto free;
2296         }
2297
2298         switch (field_op) {
2299         case FIELD_OP_MINUS:
2300                 expr->fn = hist_field_minus;
2301                 break;
2302         case FIELD_OP_PLUS:
2303                 expr->fn = hist_field_plus;
2304                 break;
2305         default:
2306                 ret = -EINVAL;
2307                 goto free;
2308         }
2309
2310         return expr;
2311  free:
2312         destroy_hist_field(operand1, 0);
2313         destroy_hist_field(operand2, 0);
2314         destroy_hist_field(expr, 0);
2315
2316         return ERR_PTR(ret);
2317 }
2318
2319 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2320                                  struct trace_event_file *file)
2321 {
2322         struct event_trigger_data *test;
2323
2324         lockdep_assert_held(&event_mutex);
2325
2326         list_for_each_entry(test, &file->triggers, list) {
2327                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2328                         if (test->private_data == hist_data)
2329                                 return test->filter_str;
2330                 }
2331         }
2332
2333         return NULL;
2334 }
2335
2336 static struct event_command trigger_hist_cmd;
2337 static int event_hist_trigger_func(struct event_command *cmd_ops,
2338                                    struct trace_event_file *file,
2339                                    char *glob, char *cmd, char *param);
2340
2341 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2342                             struct hist_trigger_data *hist_data,
2343                             unsigned int n_keys)
2344 {
2345         struct hist_field *target_hist_field, *hist_field;
2346         unsigned int n, i, j;
2347
2348         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2349                 return false;
2350
2351         i = hist_data->n_vals;
2352         j = target_hist_data->n_vals;
2353
2354         for (n = 0; n < n_keys; n++) {
2355                 hist_field = hist_data->fields[i + n];
2356                 target_hist_field = target_hist_data->fields[j + n];
2357
2358                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2359                         return false;
2360                 if (hist_field->size != target_hist_field->size)
2361                         return false;
2362                 if (hist_field->is_signed != target_hist_field->is_signed)
2363                         return false;
2364         }
2365
2366         return true;
2367 }
2368
2369 static struct hist_trigger_data *
2370 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2371                      struct trace_event_file *file)
2372 {
2373         struct hist_trigger_data *hist_data;
2374         struct event_trigger_data *test;
2375         unsigned int n_keys;
2376
2377         lockdep_assert_held(&event_mutex);
2378
2379         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2380
2381         list_for_each_entry(test, &file->triggers, list) {
2382                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2383                         hist_data = test->private_data;
2384
2385                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2386                                 return hist_data;
2387                 }
2388         }
2389
2390         return NULL;
2391 }
2392
2393 static struct trace_event_file *event_file(struct trace_array *tr,
2394                                            char *system, char *event_name)
2395 {
2396         struct trace_event_file *file;
2397
2398         file = __find_event_file(tr, system, event_name);
2399         if (!file)
2400                 return ERR_PTR(-EINVAL);
2401
2402         return file;
2403 }
2404
2405 static struct hist_field *
2406 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2407                          char *system, char *event_name, char *field_name)
2408 {
2409         struct hist_field *event_var;
2410         char *synthetic_name;
2411
2412         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2413         if (!synthetic_name)
2414                 return ERR_PTR(-ENOMEM);
2415
2416         strcpy(synthetic_name, "synthetic_");
2417         strcat(synthetic_name, field_name);
2418
2419         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2420
2421         kfree(synthetic_name);
2422
2423         return event_var;
2424 }
2425
2426 /**
2427  * create_field_var_hist - Automatically create a histogram and var for a field
2428  * @target_hist_data: The target hist trigger
2429  * @subsys_name: Optional subsystem name
2430  * @event_name: Optional event name
2431  * @field_name: The name of the field (and the resulting variable)
2432  *
2433  * Hist trigger actions fetch data from variables, not directly from
2434  * events.  However, for convenience, users are allowed to directly
2435  * specify an event field in an action, which will be automatically
2436  * converted into a variable on their behalf.
2437
2438  * If a user specifies a field on an event that isn't the event the
2439  * histogram currently being defined (the target event histogram), the
2440  * only way that can be accomplished is if a new hist trigger is
2441  * created and the field variable defined on that.
2442  *
2443  * This function creates a new histogram compatible with the target
2444  * event (meaning a histogram with the same key as the target
2445  * histogram), and creates a variable for the specified field, but
2446  * with 'synthetic_' prepended to the variable name in order to avoid
2447  * collision with normal field variables.
2448  *
2449  * Return: The variable created for the field.
2450  */
2451 static struct hist_field *
2452 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2453                       char *subsys_name, char *event_name, char *field_name)
2454 {
2455         struct trace_array *tr = target_hist_data->event_file->tr;
2456         struct hist_trigger_data *hist_data;
2457         unsigned int i, n, first = true;
2458         struct field_var_hist *var_hist;
2459         struct trace_event_file *file;
2460         struct hist_field *key_field;
2461         struct hist_field *event_var;
2462         char *saved_filter;
2463         char *cmd;
2464         int ret;
2465
2466         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2467                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2468                 return ERR_PTR(-EINVAL);
2469         }
2470
2471         file = event_file(tr, subsys_name, event_name);
2472
2473         if (IS_ERR(file)) {
2474                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2475                 ret = PTR_ERR(file);
2476                 return ERR_PTR(ret);
2477         }
2478
2479         /*
2480          * Look for a histogram compatible with target.  We'll use the
2481          * found histogram specification to create a new matching
2482          * histogram with our variable on it.  target_hist_data is not
2483          * yet a registered histogram so we can't use that.
2484          */
2485         hist_data = find_compatible_hist(target_hist_data, file);
2486         if (!hist_data) {
2487                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2488                 return ERR_PTR(-EINVAL);
2489         }
2490
2491         /* See if a synthetic field variable has already been created */
2492         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2493                                              event_name, field_name);
2494         if (!IS_ERR_OR_NULL(event_var))
2495                 return event_var;
2496
2497         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2498         if (!var_hist)
2499                 return ERR_PTR(-ENOMEM);
2500
2501         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2502         if (!cmd) {
2503                 kfree(var_hist);
2504                 return ERR_PTR(-ENOMEM);
2505         }
2506
2507         /* Use the same keys as the compatible histogram */
2508         strcat(cmd, "keys=");
2509
2510         for_each_hist_key_field(i, hist_data) {
2511                 key_field = hist_data->fields[i];
2512                 if (!first)
2513                         strcat(cmd, ",");
2514                 strcat(cmd, key_field->field->name);
2515                 first = false;
2516         }
2517
2518         /* Create the synthetic field variable specification */
2519         strcat(cmd, ":synthetic_");
2520         strcat(cmd, field_name);
2521         strcat(cmd, "=");
2522         strcat(cmd, field_name);
2523
2524         /* Use the same filter as the compatible histogram */
2525         saved_filter = find_trigger_filter(hist_data, file);
2526         if (saved_filter) {
2527                 strcat(cmd, " if ");
2528                 strcat(cmd, saved_filter);
2529         }
2530
2531         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2532         if (!var_hist->cmd) {
2533                 kfree(cmd);
2534                 kfree(var_hist);
2535                 return ERR_PTR(-ENOMEM);
2536         }
2537
2538         /* Save the compatible histogram information */
2539         var_hist->hist_data = hist_data;
2540
2541         /* Create the new histogram with our variable */
2542         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2543                                       "", "hist", cmd);
2544         if (ret) {
2545                 kfree(cmd);
2546                 kfree(var_hist->cmd);
2547                 kfree(var_hist);
2548                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2549                 return ERR_PTR(ret);
2550         }
2551
2552         kfree(cmd);
2553
2554         /* If we can't find the variable, something went wrong */
2555         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2556                                              event_name, field_name);
2557         if (IS_ERR_OR_NULL(event_var)) {
2558                 kfree(var_hist->cmd);
2559                 kfree(var_hist);
2560                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2561                 return ERR_PTR(-EINVAL);
2562         }
2563
2564         n = target_hist_data->n_field_var_hists;
2565         target_hist_data->field_var_hists[n] = var_hist;
2566         target_hist_data->n_field_var_hists++;
2567
2568         return event_var;
2569 }
2570
2571 static struct hist_field *
2572 find_target_event_var(struct hist_trigger_data *hist_data,
2573                       char *subsys_name, char *event_name, char *var_name)
2574 {
2575         struct trace_event_file *file = hist_data->event_file;
2576         struct hist_field *hist_field = NULL;
2577
2578         if (subsys_name) {
2579                 struct trace_event_call *call;
2580
2581                 if (!event_name)
2582                         return NULL;
2583
2584                 call = file->event_call;
2585
2586                 if (strcmp(subsys_name, call->class->system) != 0)
2587                         return NULL;
2588
2589                 if (strcmp(event_name, trace_event_name(call)) != 0)
2590                         return NULL;
2591         }
2592
2593         hist_field = find_var_field(hist_data, var_name);
2594
2595         return hist_field;
2596 }
2597
2598 static inline void __update_field_vars(struct tracing_map_elt *elt,
2599                                        struct trace_buffer *buffer,
2600                                        struct ring_buffer_event *rbe,
2601                                        void *rec,
2602                                        struct field_var **field_vars,
2603                                        unsigned int n_field_vars,
2604                                        unsigned int field_var_str_start)
2605 {
2606         struct hist_elt_data *elt_data = elt->private_data;
2607         unsigned int i, j, var_idx;
2608         u64 var_val;
2609
2610         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2611                 struct field_var *field_var = field_vars[i];
2612                 struct hist_field *var = field_var->var;
2613                 struct hist_field *val = field_var->val;
2614
2615                 var_val = val->fn(val, elt, buffer, rbe, rec);
2616                 var_idx = var->var.idx;
2617
2618                 if (val->flags & HIST_FIELD_FL_STRING) {
2619                         char *str = elt_data->field_var_str[j++];
2620                         char *val_str = (char *)(uintptr_t)var_val;
2621
2622                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2623                         var_val = (u64)(uintptr_t)str;
2624                 }
2625                 tracing_map_set_var(elt, var_idx, var_val);
2626         }
2627 }
2628
2629 static void update_field_vars(struct hist_trigger_data *hist_data,
2630                               struct tracing_map_elt *elt,
2631                               struct trace_buffer *buffer,
2632                               struct ring_buffer_event *rbe,
2633                               void *rec)
2634 {
2635         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2636                             hist_data->n_field_vars, 0);
2637 }
2638
2639 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2640                                  struct tracing_map_elt *elt,
2641                                  struct trace_buffer *buffer,  void *rec,
2642                                  struct ring_buffer_event *rbe, void *key,
2643                                  struct action_data *data, u64 *var_ref_vals)
2644 {
2645         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2646                             hist_data->n_save_vars, hist_data->n_field_var_str);
2647 }
2648
2649 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2650                                      struct trace_event_file *file,
2651                                      char *name, int size, const char *type)
2652 {
2653         struct hist_field *var;
2654         int idx;
2655
2656         if (find_var(hist_data, file, name) && !hist_data->remove) {
2657                 var = ERR_PTR(-EINVAL);
2658                 goto out;
2659         }
2660
2661         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2662         if (!var) {
2663                 var = ERR_PTR(-ENOMEM);
2664                 goto out;
2665         }
2666
2667         idx = tracing_map_add_var(hist_data->map);
2668         if (idx < 0) {
2669                 kfree(var);
2670                 var = ERR_PTR(-EINVAL);
2671                 goto out;
2672         }
2673
2674         var->ref = 1;
2675         var->flags = HIST_FIELD_FL_VAR;
2676         var->var.idx = idx;
2677         var->var.hist_data = var->hist_data = hist_data;
2678         var->size = size;
2679         var->var.name = kstrdup(name, GFP_KERNEL);
2680         var->type = kstrdup(type, GFP_KERNEL);
2681         if (!var->var.name || !var->type) {
2682                 kfree(var->var.name);
2683                 kfree(var->type);
2684                 kfree(var);
2685                 var = ERR_PTR(-ENOMEM);
2686         }
2687  out:
2688         return var;
2689 }
2690
2691 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2692                                           struct trace_event_file *file,
2693                                           char *field_name)
2694 {
2695         struct hist_field *val = NULL, *var = NULL;
2696         unsigned long flags = HIST_FIELD_FL_VAR;
2697         struct trace_array *tr = file->tr;
2698         struct field_var *field_var;
2699         int ret = 0;
2700
2701         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2702                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2703                 ret = -EINVAL;
2704                 goto err;
2705         }
2706
2707         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2708         if (IS_ERR(val)) {
2709                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2710                 ret = PTR_ERR(val);
2711                 goto err;
2712         }
2713
2714         var = create_var(hist_data, file, field_name, val->size, val->type);
2715         if (IS_ERR(var)) {
2716                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2717                 kfree(val);
2718                 ret = PTR_ERR(var);
2719                 goto err;
2720         }
2721
2722         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2723         if (!field_var) {
2724                 kfree(val);
2725                 kfree(var);
2726                 ret =  -ENOMEM;
2727                 goto err;
2728         }
2729
2730         field_var->var = var;
2731         field_var->val = val;
2732  out:
2733         return field_var;
2734  err:
2735         field_var = ERR_PTR(ret);
2736         goto out;
2737 }
2738
2739 /**
2740  * create_target_field_var - Automatically create a variable for a field
2741  * @target_hist_data: The target hist trigger
2742  * @subsys_name: Optional subsystem name
2743  * @event_name: Optional event name
2744  * @var_name: The name of the field (and the resulting variable)
2745  *
2746  * Hist trigger actions fetch data from variables, not directly from
2747  * events.  However, for convenience, users are allowed to directly
2748  * specify an event field in an action, which will be automatically
2749  * converted into a variable on their behalf.
2750
2751  * This function creates a field variable with the name var_name on
2752  * the hist trigger currently being defined on the target event.  If
2753  * subsys_name and event_name are specified, this function simply
2754  * verifies that they do in fact match the target event subsystem and
2755  * event name.
2756  *
2757  * Return: The variable created for the field.
2758  */
2759 static struct field_var *
2760 create_target_field_var(struct hist_trigger_data *target_hist_data,
2761                         char *subsys_name, char *event_name, char *var_name)
2762 {
2763         struct trace_event_file *file = target_hist_data->event_file;
2764
2765         if (subsys_name) {
2766                 struct trace_event_call *call;
2767
2768                 if (!event_name)
2769                         return NULL;
2770
2771                 call = file->event_call;
2772
2773                 if (strcmp(subsys_name, call->class->system) != 0)
2774                         return NULL;
2775
2776                 if (strcmp(event_name, trace_event_name(call)) != 0)
2777                         return NULL;
2778         }
2779
2780         return create_field_var(target_hist_data, file, var_name);
2781 }
2782
2783 static bool check_track_val_max(u64 track_val, u64 var_val)
2784 {
2785         if (var_val <= track_val)
2786                 return false;
2787
2788         return true;
2789 }
2790
2791 static bool check_track_val_changed(u64 track_val, u64 var_val)
2792 {
2793         if (var_val == track_val)
2794                 return false;
2795
2796         return true;
2797 }
2798
2799 static u64 get_track_val(struct hist_trigger_data *hist_data,
2800                          struct tracing_map_elt *elt,
2801                          struct action_data *data)
2802 {
2803         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2804         u64 track_val;
2805
2806         track_val = tracing_map_read_var(elt, track_var_idx);
2807
2808         return track_val;
2809 }
2810
2811 static void save_track_val(struct hist_trigger_data *hist_data,
2812                            struct tracing_map_elt *elt,
2813                            struct action_data *data, u64 var_val)
2814 {
2815         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2816
2817         tracing_map_set_var(elt, track_var_idx, var_val);
2818 }
2819
2820 static void save_track_data(struct hist_trigger_data *hist_data,
2821                             struct tracing_map_elt *elt,
2822                             struct trace_buffer *buffer, void *rec,
2823                             struct ring_buffer_event *rbe, void *key,
2824                             struct action_data *data, u64 *var_ref_vals)
2825 {
2826         if (data->track_data.save_data)
2827                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2828                                            key, data, var_ref_vals);
2829 }
2830
2831 static bool check_track_val(struct tracing_map_elt *elt,
2832                             struct action_data *data,
2833                             u64 var_val)
2834 {
2835         struct hist_trigger_data *hist_data;
2836         u64 track_val;
2837
2838         hist_data = data->track_data.track_var->hist_data;
2839         track_val = get_track_val(hist_data, elt, data);
2840
2841         return data->track_data.check_val(track_val, var_val);
2842 }
2843
2844 #ifdef CONFIG_TRACER_SNAPSHOT
2845 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2846 {
2847         /* called with tr->max_lock held */
2848         struct track_data *track_data = tr->cond_snapshot->cond_data;
2849         struct hist_elt_data *elt_data, *track_elt_data;
2850         struct snapshot_context *context = cond_data;
2851         struct action_data *action;
2852         u64 track_val;
2853
2854         if (!track_data)
2855                 return false;
2856
2857         action = track_data->action_data;
2858
2859         track_val = get_track_val(track_data->hist_data, context->elt,
2860                                   track_data->action_data);
2861
2862         if (!action->track_data.check_val(track_data->track_val, track_val))
2863                 return false;
2864
2865         track_data->track_val = track_val;
2866         memcpy(track_data->key, context->key, track_data->key_len);
2867
2868         elt_data = context->elt->private_data;
2869         track_elt_data = track_data->elt.private_data;
2870         if (elt_data->comm)
2871                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2872
2873         track_data->updated = true;
2874
2875         return true;
2876 }
2877
2878 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2879                                      struct tracing_map_elt *elt,
2880                                      struct trace_buffer *buffer, void *rec,
2881                                      struct ring_buffer_event *rbe, void *key,
2882                                      struct action_data *data,
2883                                      u64 *var_ref_vals)
2884 {
2885         struct trace_event_file *file = hist_data->event_file;
2886         struct snapshot_context context;
2887
2888         context.elt = elt;
2889         context.key = key;
2890
2891         tracing_snapshot_cond(file->tr, &context);
2892 }
2893
2894 static void hist_trigger_print_key(struct seq_file *m,
2895                                    struct hist_trigger_data *hist_data,
2896                                    void *key,
2897                                    struct tracing_map_elt *elt);
2898
2899 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2900 {
2901         unsigned int i;
2902
2903         if (!hist_data->n_actions)
2904                 return NULL;
2905
2906         for (i = 0; i < hist_data->n_actions; i++) {
2907                 struct action_data *data = hist_data->actions[i];
2908
2909                 if (data->action == ACTION_SNAPSHOT)
2910                         return data;
2911         }
2912
2913         return NULL;
2914 }
2915
2916 static void track_data_snapshot_print(struct seq_file *m,
2917                                       struct hist_trigger_data *hist_data)
2918 {
2919         struct trace_event_file *file = hist_data->event_file;
2920         struct track_data *track_data;
2921         struct action_data *action;
2922
2923         track_data = tracing_cond_snapshot_data(file->tr);
2924         if (!track_data)
2925                 return;
2926
2927         if (!track_data->updated)
2928                 return;
2929
2930         action = snapshot_action(hist_data);
2931         if (!action)
2932                 return;
2933
2934         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2935         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2936                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2937                    action->track_data.var_str, track_data->track_val);
2938
2939         seq_puts(m, "\ttriggered by event with key: ");
2940         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2941         seq_putc(m, '\n');
2942 }
2943 #else
2944 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2945 {
2946         return false;
2947 }
2948 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2949                                      struct tracing_map_elt *elt,
2950                                      struct trace_buffer *buffer, void *rec,
2951                                      struct ring_buffer_event *rbe, void *key,
2952                                      struct action_data *data,
2953                                      u64 *var_ref_vals) {}
2954 static void track_data_snapshot_print(struct seq_file *m,
2955                                       struct hist_trigger_data *hist_data) {}
2956 #endif /* CONFIG_TRACER_SNAPSHOT */
2957
2958 static void track_data_print(struct seq_file *m,
2959                              struct hist_trigger_data *hist_data,
2960                              struct tracing_map_elt *elt,
2961                              struct action_data *data)
2962 {
2963         u64 track_val = get_track_val(hist_data, elt, data);
2964         unsigned int i, save_var_idx;
2965
2966         if (data->handler == HANDLER_ONMAX)
2967                 seq_printf(m, "\n\tmax: %10llu", track_val);
2968         else if (data->handler == HANDLER_ONCHANGE)
2969                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2970
2971         if (data->action == ACTION_SNAPSHOT)
2972                 return;
2973
2974         for (i = 0; i < hist_data->n_save_vars; i++) {
2975                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2976                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2977                 u64 val;
2978
2979                 save_var_idx = save_var->var.idx;
2980
2981                 val = tracing_map_read_var(elt, save_var_idx);
2982
2983                 if (save_val->flags & HIST_FIELD_FL_STRING) {
2984                         seq_printf(m, "  %s: %-32s", save_var->var.name,
2985                                    (char *)(uintptr_t)(val));
2986                 } else
2987                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2988         }
2989 }
2990
2991 static void ontrack_action(struct hist_trigger_data *hist_data,
2992                            struct tracing_map_elt *elt,
2993                            struct trace_buffer *buffer, void *rec,
2994                            struct ring_buffer_event *rbe, void *key,
2995                            struct action_data *data, u64 *var_ref_vals)
2996 {
2997         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2998
2999         if (check_track_val(elt, data, var_val)) {
3000                 save_track_val(hist_data, elt, data, var_val);
3001                 save_track_data(hist_data, elt, buffer, rec, rbe,
3002                                 key, data, var_ref_vals);
3003         }
3004 }
3005
3006 static void action_data_destroy(struct action_data *data)
3007 {
3008         unsigned int i;
3009
3010         lockdep_assert_held(&event_mutex);
3011
3012         kfree(data->action_name);
3013
3014         for (i = 0; i < data->n_params; i++)
3015                 kfree(data->params[i]);
3016
3017         if (data->synth_event)
3018                 data->synth_event->ref--;
3019
3020         kfree(data->synth_event_name);
3021
3022         kfree(data);
3023 }
3024
3025 static void track_data_destroy(struct hist_trigger_data *hist_data,
3026                                struct action_data *data)
3027 {
3028         struct trace_event_file *file = hist_data->event_file;
3029
3030         destroy_hist_field(data->track_data.track_var, 0);
3031
3032         if (data->action == ACTION_SNAPSHOT) {
3033                 struct track_data *track_data;
3034
3035                 track_data = tracing_cond_snapshot_data(file->tr);
3036                 if (track_data && track_data->hist_data == hist_data) {
3037                         tracing_snapshot_cond_disable(file->tr);
3038                         track_data_free(track_data);
3039                 }
3040         }
3041
3042         kfree(data->track_data.var_str);
3043
3044         action_data_destroy(data);
3045 }
3046
3047 static int action_create(struct hist_trigger_data *hist_data,
3048                          struct action_data *data);
3049
3050 static int track_data_create(struct hist_trigger_data *hist_data,
3051                              struct action_data *data)
3052 {
3053         struct hist_field *var_field, *ref_field, *track_var = NULL;
3054         struct trace_event_file *file = hist_data->event_file;
3055         struct trace_array *tr = file->tr;
3056         char *track_data_var_str;
3057         int ret = 0;
3058
3059         track_data_var_str = data->track_data.var_str;
3060         if (track_data_var_str[0] != '$') {
3061                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3062                 return -EINVAL;
3063         }
3064         track_data_var_str++;
3065
3066         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3067         if (!var_field) {
3068                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3069                 return -EINVAL;
3070         }
3071
3072         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3073         if (!ref_field)
3074                 return -ENOMEM;
3075
3076         data->track_data.var_ref = ref_field;
3077
3078         if (data->handler == HANDLER_ONMAX)
3079                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3080         if (IS_ERR(track_var)) {
3081                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3082                 ret = PTR_ERR(track_var);
3083                 goto out;
3084         }
3085
3086         if (data->handler == HANDLER_ONCHANGE)
3087                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3088         if (IS_ERR(track_var)) {
3089                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3090                 ret = PTR_ERR(track_var);
3091                 goto out;
3092         }
3093         data->track_data.track_var = track_var;
3094
3095         ret = action_create(hist_data, data);
3096  out:
3097         return ret;
3098 }
3099
3100 static int parse_action_params(struct trace_array *tr, char *params,
3101                                struct action_data *data)
3102 {
3103         char *param, *saved_param;
3104         bool first_param = true;
3105         int ret = 0;
3106
3107         while (params) {
3108                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3109                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3110                         goto out;
3111                 }
3112
3113                 param = strsep(&params, ",");
3114                 if (!param) {
3115                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3116                         ret = -EINVAL;
3117                         goto out;
3118                 }
3119
3120                 param = strstrip(param);
3121                 if (strlen(param) < 2) {
3122                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3123                         ret = -EINVAL;
3124                         goto out;
3125                 }
3126
3127                 saved_param = kstrdup(param, GFP_KERNEL);
3128                 if (!saved_param) {
3129                         ret = -ENOMEM;
3130                         goto out;
3131                 }
3132
3133                 if (first_param && data->use_trace_keyword) {
3134                         data->synth_event_name = saved_param;
3135                         first_param = false;
3136                         continue;
3137                 }
3138                 first_param = false;
3139
3140                 data->params[data->n_params++] = saved_param;
3141         }
3142  out:
3143         return ret;
3144 }
3145
3146 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3147                         enum handler_id handler)
3148 {
3149         char *action_name;
3150         int ret = 0;
3151
3152         strsep(&str, ".");
3153         if (!str) {
3154                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3155                 ret = -EINVAL;
3156                 goto out;
3157         }
3158
3159         action_name = strsep(&str, "(");
3160         if (!action_name || !str) {
3161                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3162                 ret = -EINVAL;
3163                 goto out;
3164         }
3165
3166         if (str_has_prefix(action_name, "save")) {
3167                 char *params = strsep(&str, ")");
3168
3169                 if (!params) {
3170                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3171                         ret = -EINVAL;
3172                         goto out;
3173                 }
3174
3175                 ret = parse_action_params(tr, params, data);
3176                 if (ret)
3177                         goto out;
3178
3179                 if (handler == HANDLER_ONMAX)
3180                         data->track_data.check_val = check_track_val_max;
3181                 else if (handler == HANDLER_ONCHANGE)
3182                         data->track_data.check_val = check_track_val_changed;
3183                 else {
3184                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3185                         ret = -EINVAL;
3186                         goto out;
3187                 }
3188
3189                 data->track_data.save_data = save_track_data_vars;
3190                 data->fn = ontrack_action;
3191                 data->action = ACTION_SAVE;
3192         } else if (str_has_prefix(action_name, "snapshot")) {
3193                 char *params = strsep(&str, ")");
3194
3195                 if (!str) {
3196                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3197                         ret = -EINVAL;
3198                         goto out;
3199                 }
3200
3201                 if (handler == HANDLER_ONMAX)
3202                         data->track_data.check_val = check_track_val_max;
3203                 else if (handler == HANDLER_ONCHANGE)
3204                         data->track_data.check_val = check_track_val_changed;
3205                 else {
3206                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3207                         ret = -EINVAL;
3208                         goto out;
3209                 }
3210
3211                 data->track_data.save_data = save_track_data_snapshot;
3212                 data->fn = ontrack_action;
3213                 data->action = ACTION_SNAPSHOT;
3214         } else {
3215                 char *params = strsep(&str, ")");
3216
3217                 if (str_has_prefix(action_name, "trace"))
3218                         data->use_trace_keyword = true;
3219
3220                 if (params) {
3221                         ret = parse_action_params(tr, params, data);
3222                         if (ret)
3223                                 goto out;
3224                 }
3225
3226                 if (handler == HANDLER_ONMAX)
3227                         data->track_data.check_val = check_track_val_max;
3228                 else if (handler == HANDLER_ONCHANGE)
3229                         data->track_data.check_val = check_track_val_changed;
3230
3231                 if (handler != HANDLER_ONMATCH) {
3232                         data->track_data.save_data = action_trace;
3233                         data->fn = ontrack_action;
3234                 } else
3235                         data->fn = action_trace;
3236
3237                 data->action = ACTION_TRACE;
3238         }
3239
3240         data->action_name = kstrdup(action_name, GFP_KERNEL);
3241         if (!data->action_name) {
3242                 ret = -ENOMEM;
3243                 goto out;
3244         }
3245
3246         data->handler = handler;
3247  out:
3248         return ret;
3249 }
3250
3251 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3252                                             char *str, enum handler_id handler)
3253 {
3254         struct action_data *data;
3255         int ret = -EINVAL;
3256         char *var_str;
3257
3258         data = kzalloc(sizeof(*data), GFP_KERNEL);
3259         if (!data)
3260                 return ERR_PTR(-ENOMEM);
3261
3262         var_str = strsep(&str, ")");
3263         if (!var_str || !str) {
3264                 ret = -EINVAL;
3265                 goto free;
3266         }
3267
3268         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3269         if (!data->track_data.var_str) {
3270                 ret = -ENOMEM;
3271                 goto free;
3272         }
3273
3274         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3275         if (ret)
3276                 goto free;
3277  out:
3278         return data;
3279  free:
3280         track_data_destroy(hist_data, data);
3281         data = ERR_PTR(ret);
3282         goto out;
3283 }
3284
3285 static void onmatch_destroy(struct action_data *data)
3286 {
3287         kfree(data->match_data.event);
3288         kfree(data->match_data.event_system);
3289
3290         action_data_destroy(data);
3291 }
3292
3293 static void destroy_field_var(struct field_var *field_var)
3294 {
3295         if (!field_var)
3296                 return;
3297
3298         destroy_hist_field(field_var->var, 0);
3299         destroy_hist_field(field_var->val, 0);
3300
3301         kfree(field_var);
3302 }
3303
3304 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3305 {
3306         unsigned int i;
3307
3308         for (i = 0; i < hist_data->n_field_vars; i++)
3309                 destroy_field_var(hist_data->field_vars[i]);
3310
3311         for (i = 0; i < hist_data->n_save_vars; i++)
3312                 destroy_field_var(hist_data->save_vars[i]);
3313 }
3314
3315 static void save_field_var(struct hist_trigger_data *hist_data,
3316                            struct field_var *field_var)
3317 {
3318         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3319
3320         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3321                 hist_data->n_field_var_str++;
3322 }
3323
3324
3325 static int check_synth_field(struct synth_event *event,
3326                              struct hist_field *hist_field,
3327                              unsigned int field_pos)
3328 {
3329         struct synth_field *field;
3330
3331         if (field_pos >= event->n_fields)
3332                 return -EINVAL;
3333
3334         field = event->fields[field_pos];
3335
3336         /*
3337          * A dynamic string synth field can accept static or
3338          * dynamic. A static string synth field can only accept a
3339          * same-sized static string, which is checked for later.
3340          */
3341         if (strstr(hist_field->type, "char[") && field->is_string
3342             && field->is_dynamic)
3343                 return 0;
3344
3345         if (strcmp(field->type, hist_field->type) != 0) {
3346                 if (field->size != hist_field->size ||
3347                     field->is_signed != hist_field->is_signed)
3348                         return -EINVAL;
3349         }
3350
3351         return 0;
3352 }
3353
3354 static struct hist_field *
3355 trace_action_find_var(struct hist_trigger_data *hist_data,
3356                       struct action_data *data,
3357                       char *system, char *event, char *var)
3358 {
3359         struct trace_array *tr = hist_data->event_file->tr;
3360         struct hist_field *hist_field;
3361
3362         var++; /* skip '$' */
3363
3364         hist_field = find_target_event_var(hist_data, system, event, var);
3365         if (!hist_field) {
3366                 if (!system && data->handler == HANDLER_ONMATCH) {
3367                         system = data->match_data.event_system;
3368                         event = data->match_data.event;
3369                 }
3370
3371                 hist_field = find_event_var(hist_data, system, event, var);
3372         }
3373
3374         if (!hist_field)
3375                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3376
3377         return hist_field;
3378 }
3379
3380 static struct hist_field *
3381 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3382                               struct action_data *data, char *system,
3383                               char *event, char *var)
3384 {
3385         struct hist_field *hist_field = NULL;
3386         struct field_var *field_var;
3387
3388         /*
3389          * First try to create a field var on the target event (the
3390          * currently being defined).  This will create a variable for
3391          * unqualified fields on the target event, or if qualified,
3392          * target fields that have qualified names matching the target.
3393          */
3394         field_var = create_target_field_var(hist_data, system, event, var);
3395
3396         if (field_var && !IS_ERR(field_var)) {
3397                 save_field_var(hist_data, field_var);
3398                 hist_field = field_var->var;
3399         } else {
3400                 field_var = NULL;
3401                 /*
3402                  * If no explicit system.event is specified, default to
3403                  * looking for fields on the onmatch(system.event.xxx)
3404                  * event.
3405                  */
3406                 if (!system && data->handler == HANDLER_ONMATCH) {
3407                         system = data->match_data.event_system;
3408                         event = data->match_data.event;
3409                 }
3410
3411                 /*
3412                  * At this point, we're looking at a field on another
3413                  * event.  Because we can't modify a hist trigger on
3414                  * another event to add a variable for a field, we need
3415                  * to create a new trigger on that event and create the
3416                  * variable at the same time.
3417                  */
3418                 hist_field = create_field_var_hist(hist_data, system, event, var);
3419                 if (IS_ERR(hist_field))
3420                         goto free;
3421         }
3422  out:
3423         return hist_field;
3424  free:
3425         destroy_field_var(field_var);
3426         hist_field = NULL;
3427         goto out;
3428 }
3429
3430 static int trace_action_create(struct hist_trigger_data *hist_data,
3431                                struct action_data *data)
3432 {
3433         struct trace_array *tr = hist_data->event_file->tr;
3434         char *event_name, *param, *system = NULL;
3435         struct hist_field *hist_field, *var_ref;
3436         unsigned int i;
3437         unsigned int field_pos = 0;
3438         struct synth_event *event;
3439         char *synth_event_name;
3440         int var_ref_idx, ret = 0;
3441
3442         lockdep_assert_held(&event_mutex);
3443
3444         if (data->use_trace_keyword)
3445                 synth_event_name = data->synth_event_name;
3446         else
3447                 synth_event_name = data->action_name;
3448
3449         event = find_synth_event(synth_event_name);
3450         if (!event) {
3451                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3452                 return -EINVAL;
3453         }
3454
3455         event->ref++;
3456
3457         for (i = 0; i < data->n_params; i++) {
3458                 char *p;
3459
3460                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3461                 if (!param) {
3462                         ret = -ENOMEM;
3463                         goto err;
3464                 }
3465
3466                 system = strsep(&param, ".");
3467                 if (!param) {
3468                         param = (char *)system;
3469                         system = event_name = NULL;
3470                 } else {
3471                         event_name = strsep(&param, ".");
3472                         if (!param) {
3473                                 kfree(p);
3474                                 ret = -EINVAL;
3475                                 goto err;
3476                         }
3477                 }
3478
3479                 if (param[0] == '$')
3480                         hist_field = trace_action_find_var(hist_data, data,
3481                                                            system, event_name,
3482                                                            param);
3483                 else
3484                         hist_field = trace_action_create_field_var(hist_data,
3485                                                                    data,
3486                                                                    system,
3487                                                                    event_name,
3488                                                                    param);
3489
3490                 if (!hist_field) {
3491                         kfree(p);
3492                         ret = -EINVAL;
3493                         goto err;
3494                 }
3495
3496                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3497                         var_ref = create_var_ref(hist_data, hist_field,
3498                                                  system, event_name);
3499                         if (!var_ref) {
3500                                 kfree(p);
3501                                 ret = -ENOMEM;
3502                                 goto err;
3503                         }
3504
3505                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3506                         if (WARN_ON(var_ref_idx < 0)) {
3507                                 ret = var_ref_idx;
3508                                 goto err;
3509                         }
3510
3511                         data->var_ref_idx[i] = var_ref_idx;
3512
3513                         field_pos++;
3514                         kfree(p);
3515                         continue;
3516                 }
3517
3518                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3519                 kfree(p);
3520                 ret = -EINVAL;
3521                 goto err;
3522         }
3523
3524         if (field_pos != event->n_fields) {
3525                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3526                 ret = -EINVAL;
3527                 goto err;
3528         }
3529
3530         data->synth_event = event;
3531  out:
3532         return ret;
3533  err:
3534         event->ref--;
3535
3536         goto out;
3537 }
3538
3539 static int action_create(struct hist_trigger_data *hist_data,
3540                          struct action_data *data)
3541 {
3542         struct trace_event_file *file = hist_data->event_file;
3543         struct trace_array *tr = file->tr;
3544         struct track_data *track_data;
3545         struct field_var *field_var;
3546         unsigned int i;
3547         char *param;
3548         int ret = 0;
3549
3550         if (data->action == ACTION_TRACE)
3551                 return trace_action_create(hist_data, data);
3552
3553         if (data->action == ACTION_SNAPSHOT) {
3554                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3555                 if (IS_ERR(track_data)) {
3556                         ret = PTR_ERR(track_data);
3557                         goto out;
3558                 }
3559
3560                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3561                                                    cond_snapshot_update);
3562                 if (ret)
3563                         track_data_free(track_data);
3564
3565                 goto out;
3566         }
3567
3568         if (data->action == ACTION_SAVE) {
3569                 if (hist_data->n_save_vars) {
3570                         ret = -EEXIST;
3571                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3572                         goto out;
3573                 }
3574
3575                 for (i = 0; i < data->n_params; i++) {
3576                         param = kstrdup(data->params[i], GFP_KERNEL);
3577                         if (!param) {
3578                                 ret = -ENOMEM;
3579                                 goto out;
3580                         }
3581
3582                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3583                         if (IS_ERR(field_var)) {
3584                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3585                                          errpos(param));
3586                                 ret = PTR_ERR(field_var);
3587                                 kfree(param);
3588                                 goto out;
3589                         }
3590
3591                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3592                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3593                                 hist_data->n_save_var_str++;
3594                         kfree(param);
3595                 }
3596         }
3597  out:
3598         return ret;
3599 }
3600
3601 static int onmatch_create(struct hist_trigger_data *hist_data,
3602                           struct action_data *data)
3603 {
3604         return action_create(hist_data, data);
3605 }
3606
3607 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3608 {
3609         char *match_event, *match_event_system;
3610         struct action_data *data;
3611         int ret = -EINVAL;
3612
3613         data = kzalloc(sizeof(*data), GFP_KERNEL);
3614         if (!data)
3615                 return ERR_PTR(-ENOMEM);
3616
3617         match_event = strsep(&str, ")");
3618         if (!match_event || !str) {
3619                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3620                 goto free;
3621         }
3622
3623         match_event_system = strsep(&match_event, ".");
3624         if (!match_event) {
3625                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3626                 goto free;
3627         }
3628
3629         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3630                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3631                 goto free;
3632         }
3633
3634         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3635         if (!data->match_data.event) {
3636                 ret = -ENOMEM;
3637                 goto free;
3638         }
3639
3640         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3641         if (!data->match_data.event_system) {
3642                 ret = -ENOMEM;
3643                 goto free;
3644         }
3645
3646         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3647         if (ret)
3648                 goto free;
3649  out:
3650         return data;
3651  free:
3652         onmatch_destroy(data);
3653         data = ERR_PTR(ret);
3654         goto out;
3655 }
3656
3657 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3658 {
3659         hist_data->fields[HITCOUNT_IDX] =
3660                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3661         if (!hist_data->fields[HITCOUNT_IDX])
3662                 return -ENOMEM;
3663
3664         hist_data->n_vals++;
3665         hist_data->n_fields++;
3666
3667         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3668                 return -EINVAL;
3669
3670         return 0;
3671 }
3672
3673 static int __create_val_field(struct hist_trigger_data *hist_data,
3674                               unsigned int val_idx,
3675                               struct trace_event_file *file,
3676                               char *var_name, char *field_str,
3677                               unsigned long flags)
3678 {
3679         struct hist_field *hist_field;
3680         int ret = 0;
3681
3682         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3683         if (IS_ERR(hist_field)) {
3684                 ret = PTR_ERR(hist_field);
3685                 goto out;
3686         }
3687
3688         hist_data->fields[val_idx] = hist_field;
3689
3690         ++hist_data->n_vals;
3691         ++hist_data->n_fields;
3692
3693         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3694                 ret = -EINVAL;
3695  out:
3696         return ret;
3697 }
3698
3699 static int create_val_field(struct hist_trigger_data *hist_data,
3700                             unsigned int val_idx,
3701                             struct trace_event_file *file,
3702                             char *field_str)
3703 {
3704         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3705                 return -EINVAL;
3706
3707         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3708 }
3709
3710 static int create_var_field(struct hist_trigger_data *hist_data,
3711                             unsigned int val_idx,
3712                             struct trace_event_file *file,
3713                             char *var_name, char *expr_str)
3714 {
3715         struct trace_array *tr = hist_data->event_file->tr;
3716         unsigned long flags = 0;
3717         int ret;
3718
3719         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3720                 return -EINVAL;
3721
3722         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3723                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3724                 return -EINVAL;
3725         }
3726
3727         flags |= HIST_FIELD_FL_VAR;
3728         hist_data->n_vars++;
3729         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3730                 return -EINVAL;
3731
3732         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3733
3734         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3735                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3736
3737         return ret;
3738 }
3739
3740 static int create_val_fields(struct hist_trigger_data *hist_data,
3741                              struct trace_event_file *file)
3742 {
3743         char *fields_str, *field_str;
3744         unsigned int i, j = 1;
3745         int ret;
3746
3747         ret = create_hitcount_val(hist_data);
3748         if (ret)
3749                 goto out;
3750
3751         fields_str = hist_data->attrs->vals_str;
3752         if (!fields_str)
3753                 goto out;
3754
3755         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3756                      j < TRACING_MAP_VALS_MAX; i++) {
3757                 field_str = strsep(&fields_str, ",");
3758                 if (!field_str)
3759                         break;
3760
3761                 if (strcmp(field_str, "hitcount") == 0)
3762                         continue;
3763
3764                 ret = create_val_field(hist_data, j++, file, field_str);
3765                 if (ret)
3766                         goto out;
3767         }
3768
3769         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3770                 ret = -EINVAL;
3771  out:
3772         return ret;
3773 }
3774
3775 static int create_key_field(struct hist_trigger_data *hist_data,
3776                             unsigned int key_idx,
3777                             unsigned int key_offset,
3778                             struct trace_event_file *file,
3779                             char *field_str)
3780 {
3781         struct trace_array *tr = hist_data->event_file->tr;
3782         struct hist_field *hist_field = NULL;
3783         unsigned long flags = 0;
3784         unsigned int key_size;
3785         int ret = 0;
3786
3787         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3788                 return -EINVAL;
3789
3790         flags |= HIST_FIELD_FL_KEY;
3791
3792         if (strcmp(field_str, "stacktrace") == 0) {
3793                 flags |= HIST_FIELD_FL_STACKTRACE;
3794                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3795                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3796         } else {
3797                 hist_field = parse_expr(hist_data, file, field_str, flags,
3798                                         NULL, 0);
3799                 if (IS_ERR(hist_field)) {
3800                         ret = PTR_ERR(hist_field);
3801                         goto out;
3802                 }
3803
3804                 if (field_has_hist_vars(hist_field, 0)) {
3805                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3806                         destroy_hist_field(hist_field, 0);
3807                         ret = -EINVAL;
3808                         goto out;
3809                 }
3810
3811                 key_size = hist_field->size;
3812         }
3813
3814         hist_data->fields[key_idx] = hist_field;
3815
3816         key_size = ALIGN(key_size, sizeof(u64));
3817         hist_data->fields[key_idx]->size = key_size;
3818         hist_data->fields[key_idx]->offset = key_offset;
3819
3820         hist_data->key_size += key_size;
3821
3822         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3823                 ret = -EINVAL;
3824                 goto out;
3825         }
3826
3827         hist_data->n_keys++;
3828         hist_data->n_fields++;
3829
3830         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3831                 return -EINVAL;
3832
3833         ret = key_size;
3834  out:
3835         return ret;
3836 }
3837
3838 static int create_key_fields(struct hist_trigger_data *hist_data,
3839                              struct trace_event_file *file)
3840 {
3841         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3842         char *fields_str, *field_str;
3843         int ret = -EINVAL;
3844
3845         fields_str = hist_data->attrs->keys_str;
3846         if (!fields_str)
3847                 goto out;
3848
3849         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3850                 field_str = strsep(&fields_str, ",");
3851                 if (!field_str)
3852                         break;
3853                 ret = create_key_field(hist_data, i, key_offset,
3854                                        file, field_str);
3855                 if (ret < 0)
3856                         goto out;
3857                 key_offset += ret;
3858         }
3859         if (fields_str) {
3860                 ret = -EINVAL;
3861                 goto out;
3862         }
3863         ret = 0;
3864  out:
3865         return ret;
3866 }
3867
3868 static int create_var_fields(struct hist_trigger_data *hist_data,
3869                              struct trace_event_file *file)
3870 {
3871         unsigned int i, j = hist_data->n_vals;
3872         int ret = 0;
3873
3874         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3875
3876         for (i = 0; i < n_vars; i++) {
3877                 char *var_name = hist_data->attrs->var_defs.name[i];
3878                 char *expr = hist_data->attrs->var_defs.expr[i];
3879
3880                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3881                 if (ret)
3882                         goto out;
3883         }
3884  out:
3885         return ret;
3886 }
3887
3888 static void free_var_defs(struct hist_trigger_data *hist_data)
3889 {
3890         unsigned int i;
3891
3892         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3893                 kfree(hist_data->attrs->var_defs.name[i]);
3894                 kfree(hist_data->attrs->var_defs.expr[i]);
3895         }
3896
3897         hist_data->attrs->var_defs.n_vars = 0;
3898 }
3899
3900 static int parse_var_defs(struct hist_trigger_data *hist_data)
3901 {
3902         struct trace_array *tr = hist_data->event_file->tr;
3903         char *s, *str, *var_name, *field_str;
3904         unsigned int i, j, n_vars = 0;
3905         int ret = 0;
3906
3907         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3908                 str = hist_data->attrs->assignment_str[i];
3909                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3910                         field_str = strsep(&str, ",");
3911                         if (!field_str)
3912                                 break;
3913
3914                         var_name = strsep(&field_str, "=");
3915                         if (!var_name || !field_str) {
3916                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3917                                          errpos(var_name));
3918                                 ret = -EINVAL;
3919                                 goto free;
3920                         }
3921
3922                         if (n_vars == TRACING_MAP_VARS_MAX) {
3923                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3924                                 ret = -EINVAL;
3925                                 goto free;
3926                         }
3927
3928                         s = kstrdup(var_name, GFP_KERNEL);
3929                         if (!s) {
3930                                 ret = -ENOMEM;
3931                                 goto free;
3932                         }
3933                         hist_data->attrs->var_defs.name[n_vars] = s;
3934
3935                         s = kstrdup(field_str, GFP_KERNEL);
3936                         if (!s) {
3937                                 ret = -ENOMEM;
3938                                 goto free;
3939                         }
3940                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3941
3942                         hist_data->attrs->var_defs.n_vars = n_vars;
3943                 }
3944         }
3945
3946         return ret;
3947  free:
3948         free_var_defs(hist_data);
3949
3950         return ret;
3951 }
3952
3953 static int create_hist_fields(struct hist_trigger_data *hist_data,
3954                               struct trace_event_file *file)
3955 {
3956         int ret;
3957
3958         ret = parse_var_defs(hist_data);
3959         if (ret)
3960                 goto out;
3961
3962         ret = create_val_fields(hist_data, file);
3963         if (ret)
3964                 goto out;
3965
3966         ret = create_var_fields(hist_data, file);
3967         if (ret)
3968                 goto out;
3969
3970         ret = create_key_fields(hist_data, file);
3971         if (ret)
3972                 goto out;
3973  out:
3974         free_var_defs(hist_data);
3975
3976         return ret;
3977 }
3978
3979 static int is_descending(struct trace_array *tr, const char *str)
3980 {
3981         if (!str)
3982                 return 0;
3983
3984         if (strcmp(str, "descending") == 0)
3985                 return 1;
3986
3987         if (strcmp(str, "ascending") == 0)
3988                 return 0;
3989
3990         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3991
3992         return -EINVAL;
3993 }
3994
3995 static int create_sort_keys(struct hist_trigger_data *hist_data)
3996 {
3997         struct trace_array *tr = hist_data->event_file->tr;
3998         char *fields_str = hist_data->attrs->sort_key_str;
3999         struct tracing_map_sort_key *sort_key;
4000         int descending, ret = 0;
4001         unsigned int i, j, k;
4002
4003         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4004
4005         if (!fields_str)
4006                 goto out;
4007
4008         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4009                 struct hist_field *hist_field;
4010                 char *field_str, *field_name;
4011                 const char *test_name;
4012
4013                 sort_key = &hist_data->sort_keys[i];
4014
4015                 field_str = strsep(&fields_str, ",");
4016                 if (!field_str)
4017                         break;
4018
4019                 if (!*field_str) {
4020                         ret = -EINVAL;
4021                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4022                         break;
4023                 }
4024
4025                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4026                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4027                         ret = -EINVAL;
4028                         break;
4029                 }
4030
4031                 field_name = strsep(&field_str, ".");
4032                 if (!field_name || !*field_name) {
4033                         ret = -EINVAL;
4034                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4035                         break;
4036                 }
4037
4038                 if (strcmp(field_name, "hitcount") == 0) {
4039                         descending = is_descending(tr, field_str);
4040                         if (descending < 0) {
4041                                 ret = descending;
4042                                 break;
4043                         }
4044                         sort_key->descending = descending;
4045                         continue;
4046                 }
4047
4048                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4049                         unsigned int idx;
4050
4051                         hist_field = hist_data->fields[j];
4052                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4053                                 continue;
4054
4055                         idx = k++;
4056
4057                         test_name = hist_field_name(hist_field, 0);
4058
4059                         if (strcmp(field_name, test_name) == 0) {
4060                                 sort_key->field_idx = idx;
4061                                 descending = is_descending(tr, field_str);
4062                                 if (descending < 0) {
4063                                         ret = descending;
4064                                         goto out;
4065                                 }
4066                                 sort_key->descending = descending;
4067                                 break;
4068                         }
4069                 }
4070                 if (j == hist_data->n_fields) {
4071                         ret = -EINVAL;
4072                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4073                         break;
4074                 }
4075         }
4076
4077         hist_data->n_sort_keys = i;
4078  out:
4079         return ret;
4080 }
4081
4082 static void destroy_actions(struct hist_trigger_data *hist_data)
4083 {
4084         unsigned int i;
4085
4086         for (i = 0; i < hist_data->n_actions; i++) {
4087                 struct action_data *data = hist_data->actions[i];
4088
4089                 if (data->handler == HANDLER_ONMATCH)
4090                         onmatch_destroy(data);
4091                 else if (data->handler == HANDLER_ONMAX ||
4092                          data->handler == HANDLER_ONCHANGE)
4093                         track_data_destroy(hist_data, data);
4094                 else
4095                         kfree(data);
4096         }
4097 }
4098
4099 static int parse_actions(struct hist_trigger_data *hist_data)
4100 {
4101         struct trace_array *tr = hist_data->event_file->tr;
4102         struct action_data *data;
4103         unsigned int i;
4104         int ret = 0;
4105         char *str;
4106         int len;
4107
4108         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4109                 str = hist_data->attrs->action_str[i];
4110
4111                 if ((len = str_has_prefix(str, "onmatch("))) {
4112                         char *action_str = str + len;
4113
4114                         data = onmatch_parse(tr, action_str);
4115                         if (IS_ERR(data)) {
4116                                 ret = PTR_ERR(data);
4117                                 break;
4118                         }
4119                 } else if ((len = str_has_prefix(str, "onmax("))) {
4120                         char *action_str = str + len;
4121
4122                         data = track_data_parse(hist_data, action_str,
4123                                                 HANDLER_ONMAX);
4124                         if (IS_ERR(data)) {
4125                                 ret = PTR_ERR(data);
4126                                 break;
4127                         }
4128                 } else if ((len = str_has_prefix(str, "onchange("))) {
4129                         char *action_str = str + len;
4130
4131                         data = track_data_parse(hist_data, action_str,
4132                                                 HANDLER_ONCHANGE);
4133                         if (IS_ERR(data)) {
4134                                 ret = PTR_ERR(data);
4135                                 break;
4136                         }
4137                 } else {
4138                         ret = -EINVAL;
4139                         break;
4140                 }
4141
4142                 hist_data->actions[hist_data->n_actions++] = data;
4143         }
4144
4145         return ret;
4146 }
4147
4148 static int create_actions(struct hist_trigger_data *hist_data)
4149 {
4150         struct action_data *data;
4151         unsigned int i;
4152         int ret = 0;
4153
4154         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4155                 data = hist_data->actions[i];
4156
4157                 if (data->handler == HANDLER_ONMATCH) {
4158                         ret = onmatch_create(hist_data, data);
4159                         if (ret)
4160                                 break;
4161                 } else if (data->handler == HANDLER_ONMAX ||
4162                            data->handler == HANDLER_ONCHANGE) {
4163                         ret = track_data_create(hist_data, data);
4164                         if (ret)
4165                                 break;
4166                 } else {
4167                         ret = -EINVAL;
4168                         break;
4169                 }
4170         }
4171
4172         return ret;
4173 }
4174
4175 static void print_actions(struct seq_file *m,
4176                           struct hist_trigger_data *hist_data,
4177                           struct tracing_map_elt *elt)
4178 {
4179         unsigned int i;
4180
4181         for (i = 0; i < hist_data->n_actions; i++) {
4182                 struct action_data *data = hist_data->actions[i];
4183
4184                 if (data->action == ACTION_SNAPSHOT)
4185                         continue;
4186
4187                 if (data->handler == HANDLER_ONMAX ||
4188                     data->handler == HANDLER_ONCHANGE)
4189                         track_data_print(m, hist_data, elt, data);
4190         }
4191 }
4192
4193 static void print_action_spec(struct seq_file *m,
4194                               struct hist_trigger_data *hist_data,
4195                               struct action_data *data)
4196 {
4197         unsigned int i;
4198
4199         if (data->action == ACTION_SAVE) {
4200                 for (i = 0; i < hist_data->n_save_vars; i++) {
4201                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4202                         if (i < hist_data->n_save_vars - 1)
4203                                 seq_puts(m, ",");
4204                 }
4205         } else if (data->action == ACTION_TRACE) {
4206                 if (data->use_trace_keyword)
4207                         seq_printf(m, "%s", data->synth_event_name);
4208                 for (i = 0; i < data->n_params; i++) {
4209                         if (i || data->use_trace_keyword)
4210                                 seq_puts(m, ",");
4211                         seq_printf(m, "%s", data->params[i]);
4212                 }
4213         }
4214 }
4215
4216 static void print_track_data_spec(struct seq_file *m,
4217                                   struct hist_trigger_data *hist_data,
4218                                   struct action_data *data)
4219 {
4220         if (data->handler == HANDLER_ONMAX)
4221                 seq_puts(m, ":onmax(");
4222         else if (data->handler == HANDLER_ONCHANGE)
4223                 seq_puts(m, ":onchange(");
4224         seq_printf(m, "%s", data->track_data.var_str);
4225         seq_printf(m, ").%s(", data->action_name);
4226
4227         print_action_spec(m, hist_data, data);
4228
4229         seq_puts(m, ")");
4230 }
4231
4232 static void print_onmatch_spec(struct seq_file *m,
4233                                struct hist_trigger_data *hist_data,
4234                                struct action_data *data)
4235 {
4236         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4237                    data->match_data.event);
4238
4239         seq_printf(m, "%s(", data->action_name);
4240
4241         print_action_spec(m, hist_data, data);
4242
4243         seq_puts(m, ")");
4244 }
4245
4246 static bool actions_match(struct hist_trigger_data *hist_data,
4247                           struct hist_trigger_data *hist_data_test)
4248 {
4249         unsigned int i, j;
4250
4251         if (hist_data->n_actions != hist_data_test->n_actions)
4252                 return false;
4253
4254         for (i = 0; i < hist_data->n_actions; i++) {
4255                 struct action_data *data = hist_data->actions[i];
4256                 struct action_data *data_test = hist_data_test->actions[i];
4257                 char *action_name, *action_name_test;
4258
4259                 if (data->handler != data_test->handler)
4260                         return false;
4261                 if (data->action != data_test->action)
4262                         return false;
4263
4264                 if (data->n_params != data_test->n_params)
4265                         return false;
4266
4267                 for (j = 0; j < data->n_params; j++) {
4268                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4269                                 return false;
4270                 }
4271
4272                 if (data->use_trace_keyword)
4273                         action_name = data->synth_event_name;
4274                 else
4275                         action_name = data->action_name;
4276
4277                 if (data_test->use_trace_keyword)
4278                         action_name_test = data_test->synth_event_name;
4279                 else
4280                         action_name_test = data_test->action_name;
4281
4282                 if (strcmp(action_name, action_name_test) != 0)
4283                         return false;
4284
4285                 if (data->handler == HANDLER_ONMATCH) {
4286                         if (strcmp(data->match_data.event_system,
4287                                    data_test->match_data.event_system) != 0)
4288                                 return false;
4289                         if (strcmp(data->match_data.event,
4290                                    data_test->match_data.event) != 0)
4291                                 return false;
4292                 } else if (data->handler == HANDLER_ONMAX ||
4293                            data->handler == HANDLER_ONCHANGE) {
4294                         if (strcmp(data->track_data.var_str,
4295                                    data_test->track_data.var_str) != 0)
4296                                 return false;
4297                 }
4298         }
4299
4300         return true;
4301 }
4302
4303
4304 static void print_actions_spec(struct seq_file *m,
4305                                struct hist_trigger_data *hist_data)
4306 {
4307         unsigned int i;
4308
4309         for (i = 0; i < hist_data->n_actions; i++) {
4310                 struct action_data *data = hist_data->actions[i];
4311
4312                 if (data->handler == HANDLER_ONMATCH)
4313                         print_onmatch_spec(m, hist_data, data);
4314                 else if (data->handler == HANDLER_ONMAX ||
4315                          data->handler == HANDLER_ONCHANGE)
4316                         print_track_data_spec(m, hist_data, data);
4317         }
4318 }
4319
4320 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4321 {
4322         unsigned int i;
4323
4324         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4325                 kfree(hist_data->field_var_hists[i]->cmd);
4326                 kfree(hist_data->field_var_hists[i]);
4327         }
4328 }
4329
4330 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4331 {
4332         if (!hist_data)
4333                 return;
4334
4335         destroy_hist_trigger_attrs(hist_data->attrs);
4336         destroy_hist_fields(hist_data);
4337         tracing_map_destroy(hist_data->map);
4338
4339         destroy_actions(hist_data);
4340         destroy_field_vars(hist_data);
4341         destroy_field_var_hists(hist_data);
4342
4343         kfree(hist_data);
4344 }
4345
4346 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4347 {
4348         struct tracing_map *map = hist_data->map;
4349         struct ftrace_event_field *field;
4350         struct hist_field *hist_field;
4351         int i, idx = 0;
4352
4353         for_each_hist_field(i, hist_data) {
4354                 hist_field = hist_data->fields[i];
4355                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4356                         tracing_map_cmp_fn_t cmp_fn;
4357
4358                         field = hist_field->field;
4359
4360                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4361                                 cmp_fn = tracing_map_cmp_none;
4362                         else if (!field)
4363                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4364                                                              hist_field->is_signed);
4365                         else if (is_string_field(field))
4366                                 cmp_fn = tracing_map_cmp_string;
4367                         else
4368                                 cmp_fn = tracing_map_cmp_num(field->size,
4369                                                              field->is_signed);
4370                         idx = tracing_map_add_key_field(map,
4371                                                         hist_field->offset,
4372                                                         cmp_fn);
4373                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4374                         idx = tracing_map_add_sum_field(map);
4375
4376                 if (idx < 0)
4377                         return idx;
4378
4379                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4380                         idx = tracing_map_add_var(map);
4381                         if (idx < 0)
4382                                 return idx;
4383                         hist_field->var.idx = idx;
4384                         hist_field->var.hist_data = hist_data;
4385                 }
4386         }
4387
4388         return 0;
4389 }
4390
4391 static struct hist_trigger_data *
4392 create_hist_data(unsigned int map_bits,
4393                  struct hist_trigger_attrs *attrs,
4394                  struct trace_event_file *file,
4395                  bool remove)
4396 {
4397         const struct tracing_map_ops *map_ops = NULL;
4398         struct hist_trigger_data *hist_data;
4399         int ret = 0;
4400
4401         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4402         if (!hist_data)
4403                 return ERR_PTR(-ENOMEM);
4404
4405         hist_data->attrs = attrs;
4406         hist_data->remove = remove;
4407         hist_data->event_file = file;
4408
4409         ret = parse_actions(hist_data);
4410         if (ret)
4411                 goto free;
4412
4413         ret = create_hist_fields(hist_data, file);
4414         if (ret)
4415                 goto free;
4416
4417         ret = create_sort_keys(hist_data);
4418         if (ret)
4419                 goto free;
4420
4421         map_ops = &hist_trigger_elt_data_ops;
4422
4423         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4424                                             map_ops, hist_data);
4425         if (IS_ERR(hist_data->map)) {
4426                 ret = PTR_ERR(hist_data->map);
4427                 hist_data->map = NULL;
4428                 goto free;
4429         }
4430
4431         ret = create_tracing_map_fields(hist_data);
4432         if (ret)
4433                 goto free;
4434  out:
4435         return hist_data;
4436  free:
4437         hist_data->attrs = NULL;
4438
4439         destroy_hist_data(hist_data);
4440
4441         hist_data = ERR_PTR(ret);
4442
4443         goto out;
4444 }
4445
4446 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4447                                     struct tracing_map_elt *elt,
4448                                     struct trace_buffer *buffer, void *rec,
4449                                     struct ring_buffer_event *rbe,
4450                                     u64 *var_ref_vals)
4451 {
4452         struct hist_elt_data *elt_data;
4453         struct hist_field *hist_field;
4454         unsigned int i, var_idx;
4455         u64 hist_val;
4456
4457         elt_data = elt->private_data;
4458         elt_data->var_ref_vals = var_ref_vals;
4459
4460         for_each_hist_val_field(i, hist_data) {
4461                 hist_field = hist_data->fields[i];
4462                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4463                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4464                         var_idx = hist_field->var.idx;
4465
4466                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4467                                 unsigned int str_start, var_str_idx, idx;
4468                                 char *str, *val_str;
4469
4470                                 str_start = hist_data->n_field_var_str +
4471                                         hist_data->n_save_var_str;
4472                                 var_str_idx = hist_field->var_str_idx;
4473                                 idx = str_start + var_str_idx;
4474
4475                                 str = elt_data->field_var_str[idx];
4476                                 val_str = (char *)(uintptr_t)hist_val;
4477                                 strscpy(str, val_str, STR_VAR_LEN_MAX);
4478
4479                                 hist_val = (u64)(uintptr_t)str;
4480                         }
4481                         tracing_map_set_var(elt, var_idx, hist_val);
4482                         continue;
4483                 }
4484                 tracing_map_update_sum(elt, i, hist_val);
4485         }
4486
4487         for_each_hist_key_field(i, hist_data) {
4488                 hist_field = hist_data->fields[i];
4489                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4490                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4491                         var_idx = hist_field->var.idx;
4492                         tracing_map_set_var(elt, var_idx, hist_val);
4493                 }
4494         }
4495
4496         update_field_vars(hist_data, elt, buffer, rbe, rec);
4497 }
4498
4499 static inline void add_to_key(char *compound_key, void *key,
4500                               struct hist_field *key_field, void *rec)
4501 {
4502         size_t size = key_field->size;
4503
4504         if (key_field->flags & HIST_FIELD_FL_STRING) {
4505                 struct ftrace_event_field *field;
4506
4507                 field = key_field->field;
4508                 if (field->filter_type == FILTER_DYN_STRING)
4509                         size = *(u32 *)(rec + field->offset) >> 16;
4510                 else if (field->filter_type == FILTER_STATIC_STRING)
4511                         size = field->size;
4512
4513                 /* ensure NULL-termination */
4514                 if (size > key_field->size - 1)
4515                         size = key_field->size - 1;
4516
4517                 strncpy(compound_key + key_field->offset, (char *)key, size);
4518         } else
4519                 memcpy(compound_key + key_field->offset, key, size);
4520 }
4521
4522 static void
4523 hist_trigger_actions(struct hist_trigger_data *hist_data,
4524                      struct tracing_map_elt *elt,
4525                      struct trace_buffer *buffer, void *rec,
4526                      struct ring_buffer_event *rbe, void *key,
4527                      u64 *var_ref_vals)
4528 {
4529         struct action_data *data;
4530         unsigned int i;
4531
4532         for (i = 0; i < hist_data->n_actions; i++) {
4533                 data = hist_data->actions[i];
4534                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4535         }
4536 }
4537
4538 static void event_hist_trigger(struct event_trigger_data *data,
4539                                struct trace_buffer *buffer, void *rec,
4540                                struct ring_buffer_event *rbe)
4541 {
4542         struct hist_trigger_data *hist_data = data->private_data;
4543         bool use_compound_key = (hist_data->n_keys > 1);
4544         unsigned long entries[HIST_STACKTRACE_DEPTH];
4545         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4546         char compound_key[HIST_KEY_SIZE_MAX];
4547         struct tracing_map_elt *elt = NULL;
4548         struct hist_field *key_field;
4549         u64 field_contents;
4550         void *key = NULL;
4551         unsigned int i;
4552
4553         memset(compound_key, 0, hist_data->key_size);
4554
4555         for_each_hist_key_field(i, hist_data) {
4556                 key_field = hist_data->fields[i];
4557
4558                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4559                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4560                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4561                                          HIST_STACKTRACE_SKIP);
4562                         key = entries;
4563                 } else {
4564                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4565                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4566                                 key = (void *)(unsigned long)field_contents;
4567                                 use_compound_key = true;
4568                         } else
4569                                 key = (void *)&field_contents;
4570                 }
4571
4572                 if (use_compound_key)
4573                         add_to_key(compound_key, key, key_field, rec);
4574         }
4575
4576         if (use_compound_key)
4577                 key = compound_key;
4578
4579         if (hist_data->n_var_refs &&
4580             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4581                 return;
4582
4583         elt = tracing_map_insert(hist_data->map, key);
4584         if (!elt)
4585                 return;
4586
4587         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4588
4589         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4590                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4591 }
4592
4593 static void hist_trigger_stacktrace_print(struct seq_file *m,
4594                                           unsigned long *stacktrace_entries,
4595                                           unsigned int max_entries)
4596 {
4597         char str[KSYM_SYMBOL_LEN];
4598         unsigned int spaces = 8;
4599         unsigned int i;
4600
4601         for (i = 0; i < max_entries; i++) {
4602                 if (!stacktrace_entries[i])
4603                         return;
4604
4605                 seq_printf(m, "%*c", 1 + spaces, ' ');
4606                 sprint_symbol(str, stacktrace_entries[i]);
4607                 seq_printf(m, "%s\n", str);
4608         }
4609 }
4610
4611 static void hist_trigger_print_key(struct seq_file *m,
4612                                    struct hist_trigger_data *hist_data,
4613                                    void *key,
4614                                    struct tracing_map_elt *elt)
4615 {
4616         struct hist_field *key_field;
4617         char str[KSYM_SYMBOL_LEN];
4618         bool multiline = false;
4619         const char *field_name;
4620         unsigned int i;
4621         u64 uval;
4622
4623         seq_puts(m, "{ ");
4624
4625         for_each_hist_key_field(i, hist_data) {
4626                 key_field = hist_data->fields[i];
4627
4628                 if (i > hist_data->n_vals)
4629                         seq_puts(m, ", ");
4630
4631                 field_name = hist_field_name(key_field, 0);
4632
4633                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4634                         uval = *(u64 *)(key + key_field->offset);
4635                         seq_printf(m, "%s: %llx", field_name, uval);
4636                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4637                         uval = *(u64 *)(key + key_field->offset);
4638                         sprint_symbol_no_offset(str, uval);
4639                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4640                                    uval, str);
4641                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4642                         uval = *(u64 *)(key + key_field->offset);
4643                         sprint_symbol(str, uval);
4644                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4645                                    uval, str);
4646                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4647                         struct hist_elt_data *elt_data = elt->private_data;
4648                         char *comm;
4649
4650                         if (WARN_ON_ONCE(!elt_data))
4651                                 return;
4652
4653                         comm = elt_data->comm;
4654
4655                         uval = *(u64 *)(key + key_field->offset);
4656                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4657                                    comm, uval);
4658                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4659                         const char *syscall_name;
4660
4661                         uval = *(u64 *)(key + key_field->offset);
4662                         syscall_name = get_syscall_name(uval);
4663                         if (!syscall_name)
4664                                 syscall_name = "unknown_syscall";
4665
4666                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4667                                    syscall_name, uval);
4668                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4669                         seq_puts(m, "stacktrace:\n");
4670                         hist_trigger_stacktrace_print(m,
4671                                                       key + key_field->offset,
4672                                                       HIST_STACKTRACE_DEPTH);
4673                         multiline = true;
4674                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4675                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4676                                    *(u64 *)(key + key_field->offset));
4677                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4678                         seq_printf(m, "%s: %-50s", field_name,
4679                                    (char *)(key + key_field->offset));
4680                 } else {
4681                         uval = *(u64 *)(key + key_field->offset);
4682                         seq_printf(m, "%s: %10llu", field_name, uval);
4683                 }
4684         }
4685
4686         if (!multiline)
4687                 seq_puts(m, " ");
4688
4689         seq_puts(m, "}");
4690 }
4691
4692 static void hist_trigger_entry_print(struct seq_file *m,
4693                                      struct hist_trigger_data *hist_data,
4694                                      void *key,
4695                                      struct tracing_map_elt *elt)
4696 {
4697         const char *field_name;
4698         unsigned int i;
4699
4700         hist_trigger_print_key(m, hist_data, key, elt);
4701
4702         seq_printf(m, " hitcount: %10llu",
4703                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4704
4705         for (i = 1; i < hist_data->n_vals; i++) {
4706                 field_name = hist_field_name(hist_data->fields[i], 0);
4707
4708                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4709                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4710                         continue;
4711
4712                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4713                         seq_printf(m, "  %s: %10llx", field_name,
4714                                    tracing_map_read_sum(elt, i));
4715                 } else {
4716                         seq_printf(m, "  %s: %10llu", field_name,
4717                                    tracing_map_read_sum(elt, i));
4718                 }
4719         }
4720
4721         print_actions(m, hist_data, elt);
4722
4723         seq_puts(m, "\n");
4724 }
4725
4726 static int print_entries(struct seq_file *m,
4727                          struct hist_trigger_data *hist_data)
4728 {
4729         struct tracing_map_sort_entry **sort_entries = NULL;
4730         struct tracing_map *map = hist_data->map;
4731         int i, n_entries;
4732
4733         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4734                                              hist_data->n_sort_keys,
4735                                              &sort_entries);
4736         if (n_entries < 0)
4737                 return n_entries;
4738
4739         for (i = 0; i < n_entries; i++)
4740                 hist_trigger_entry_print(m, hist_data,
4741                                          sort_entries[i]->key,
4742                                          sort_entries[i]->elt);
4743
4744         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4745
4746         return n_entries;
4747 }
4748
4749 static void hist_trigger_show(struct seq_file *m,
4750                               struct event_trigger_data *data, int n)
4751 {
4752         struct hist_trigger_data *hist_data;
4753         int n_entries;
4754
4755         if (n > 0)
4756                 seq_puts(m, "\n\n");
4757
4758         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4759         data->ops->print(m, data->ops, data);
4760         seq_puts(m, "#\n\n");
4761
4762         hist_data = data->private_data;
4763         n_entries = print_entries(m, hist_data);
4764         if (n_entries < 0)
4765                 n_entries = 0;
4766
4767         track_data_snapshot_print(m, hist_data);
4768
4769         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4770                    (u64)atomic64_read(&hist_data->map->hits),
4771                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4772 }
4773
4774 static int hist_show(struct seq_file *m, void *v)
4775 {
4776         struct event_trigger_data *data;
4777         struct trace_event_file *event_file;
4778         int n = 0, ret = 0;
4779
4780         mutex_lock(&event_mutex);
4781
4782         event_file = event_file_data(m->private);
4783         if (unlikely(!event_file)) {
4784                 ret = -ENODEV;
4785                 goto out_unlock;
4786         }
4787
4788         list_for_each_entry(data, &event_file->triggers, list) {
4789                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4790                         hist_trigger_show(m, data, n++);
4791         }
4792
4793  out_unlock:
4794         mutex_unlock(&event_mutex);
4795
4796         return ret;
4797 }
4798
4799 static int event_hist_open(struct inode *inode, struct file *file)
4800 {
4801         int ret;
4802
4803         ret = security_locked_down(LOCKDOWN_TRACEFS);
4804         if (ret)
4805                 return ret;
4806
4807         return single_open(file, hist_show, file);
4808 }
4809
4810 const struct file_operations event_hist_fops = {
4811         .open = event_hist_open,
4812         .read = seq_read,
4813         .llseek = seq_lseek,
4814         .release = single_release,
4815 };
4816
4817 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4818 static void hist_field_debug_show_flags(struct seq_file *m,
4819                                         unsigned long flags)
4820 {
4821         seq_puts(m, "      flags:\n");
4822
4823         if (flags & HIST_FIELD_FL_KEY)
4824                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4825         else if (flags & HIST_FIELD_FL_HITCOUNT)
4826                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4827         else if (flags & HIST_FIELD_FL_VAR)
4828                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4829         else if (flags & HIST_FIELD_FL_VAR_REF)
4830                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4831         else
4832                 seq_puts(m, "        VAL: normal u64 value\n");
4833
4834         if (flags & HIST_FIELD_FL_ALIAS)
4835                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4836 }
4837
4838 static int hist_field_debug_show(struct seq_file *m,
4839                                  struct hist_field *field, unsigned long flags)
4840 {
4841         if ((field->flags & flags) != flags) {
4842                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4843                 return -EINVAL;
4844         }
4845
4846         hist_field_debug_show_flags(m, field->flags);
4847         if (field->field)
4848                 seq_printf(m, "      ftrace_event_field name: %s\n",
4849                            field->field->name);
4850
4851         if (field->flags & HIST_FIELD_FL_VAR) {
4852                 seq_printf(m, "      var.name: %s\n", field->var.name);
4853                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4854                            field->var.idx);
4855         }
4856
4857         if (field->flags & HIST_FIELD_FL_ALIAS)
4858                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4859                            field->var_ref_idx);
4860
4861         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4862                 seq_printf(m, "      name: %s\n", field->name);
4863                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4864                            field->var.idx);
4865                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4866                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4867                            field->var_ref_idx);
4868                 if (field->system)
4869                         seq_printf(m, "      system: %s\n", field->system);
4870                 if (field->event_name)
4871                         seq_printf(m, "      event_name: %s\n", field->event_name);
4872         }
4873
4874         seq_printf(m, "      type: %s\n", field->type);
4875         seq_printf(m, "      size: %u\n", field->size);
4876         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4877
4878         return 0;
4879 }
4880
4881 static int field_var_debug_show(struct seq_file *m,
4882                                 struct field_var *field_var, unsigned int i,
4883                                 bool save_vars)
4884 {
4885         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4886         struct hist_field *field;
4887         int ret = 0;
4888
4889         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4890
4891         field = field_var->var;
4892
4893         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4894
4895         hist_field_debug_show_flags(m, field->flags);
4896         seq_printf(m, "      var.name: %s\n", field->var.name);
4897         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4898                    field->var.idx);
4899
4900         field = field_var->val;
4901
4902         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4903         if (field->field)
4904                 seq_printf(m, "      ftrace_event_field name: %s\n",
4905                            field->field->name);
4906         else {
4907                 ret = -EINVAL;
4908                 goto out;
4909         }
4910
4911         seq_printf(m, "      type: %s\n", field->type);
4912         seq_printf(m, "      size: %u\n", field->size);
4913         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4914 out:
4915         return ret;
4916 }
4917
4918 static int hist_action_debug_show(struct seq_file *m,
4919                                   struct action_data *data, int i)
4920 {
4921         int ret = 0;
4922
4923         if (data->handler == HANDLER_ONMAX ||
4924             data->handler == HANDLER_ONCHANGE) {
4925                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4926                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4927                                             HIST_FIELD_FL_VAR_REF);
4928                 if (ret)
4929                         goto out;
4930
4931                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4932                 ret = hist_field_debug_show(m, data->track_data.track_var,
4933                                             HIST_FIELD_FL_VAR);
4934                 if (ret)
4935                         goto out;
4936         }
4937
4938         if (data->handler == HANDLER_ONMATCH) {
4939                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4940                            i, data->match_data.event_system);
4941                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4942                            i, data->match_data.event);
4943         }
4944 out:
4945         return ret;
4946 }
4947
4948 static int hist_actions_debug_show(struct seq_file *m,
4949                                    struct hist_trigger_data *hist_data)
4950 {
4951         int i, ret = 0;
4952
4953         if (hist_data->n_actions)
4954                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4955
4956         for (i = 0; i < hist_data->n_actions; i++) {
4957                 struct action_data *action = hist_data->actions[i];
4958
4959                 ret = hist_action_debug_show(m, action, i);
4960                 if (ret)
4961                         goto out;
4962         }
4963
4964         if (hist_data->n_save_vars)
4965                 seq_puts(m, "\n  save action variables (save() params):\n");
4966
4967         for (i = 0; i < hist_data->n_save_vars; i++) {
4968                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4969                 if (ret)
4970                         goto out;
4971         }
4972 out:
4973         return ret;
4974 }
4975
4976 static void hist_trigger_debug_show(struct seq_file *m,
4977                                     struct event_trigger_data *data, int n)
4978 {
4979         struct hist_trigger_data *hist_data;
4980         int i, ret;
4981
4982         if (n > 0)
4983                 seq_puts(m, "\n\n");
4984
4985         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4986         data->ops->print(m, data->ops, data);
4987         seq_puts(m, "#\n\n");
4988
4989         hist_data = data->private_data;
4990
4991         seq_printf(m, "hist_data: %p\n\n", hist_data);
4992         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4993         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4994         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4995
4996         seq_puts(m, "\n  val fields:\n\n");
4997
4998         seq_puts(m, "    hist_data->fields[0]:\n");
4999         ret = hist_field_debug_show(m, hist_data->fields[0],
5000                                     HIST_FIELD_FL_HITCOUNT);
5001         if (ret)
5002                 return;
5003
5004         for (i = 1; i < hist_data->n_vals; i++) {
5005                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5006                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5007                 if (ret)
5008                         return;
5009         }
5010
5011         seq_puts(m, "\n  key fields:\n");
5012
5013         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5014                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5015                 ret = hist_field_debug_show(m, hist_data->fields[i],
5016                                             HIST_FIELD_FL_KEY);
5017                 if (ret)
5018                         return;
5019         }
5020
5021         if (hist_data->n_var_refs)
5022                 seq_puts(m, "\n  variable reference fields:\n");
5023
5024         for (i = 0; i < hist_data->n_var_refs; i++) {
5025                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5026                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5027                                             HIST_FIELD_FL_VAR_REF);
5028                 if (ret)
5029                         return;
5030         }
5031
5032         if (hist_data->n_field_vars)
5033                 seq_puts(m, "\n  field variables:\n");
5034
5035         for (i = 0; i < hist_data->n_field_vars; i++) {
5036                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5037                 if (ret)
5038                         return;
5039         }
5040
5041         ret = hist_actions_debug_show(m, hist_data);
5042         if (ret)
5043                 return;
5044 }
5045
5046 static int hist_debug_show(struct seq_file *m, void *v)
5047 {
5048         struct event_trigger_data *data;
5049         struct trace_event_file *event_file;
5050         int n = 0, ret = 0;
5051
5052         mutex_lock(&event_mutex);
5053
5054         event_file = event_file_data(m->private);
5055         if (unlikely(!event_file)) {
5056                 ret = -ENODEV;
5057                 goto out_unlock;
5058         }
5059
5060         list_for_each_entry(data, &event_file->triggers, list) {
5061                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5062                         hist_trigger_debug_show(m, data, n++);
5063         }
5064
5065  out_unlock:
5066         mutex_unlock(&event_mutex);
5067
5068         return ret;
5069 }
5070
5071 static int event_hist_debug_open(struct inode *inode, struct file *file)
5072 {
5073         int ret;
5074
5075         ret = security_locked_down(LOCKDOWN_TRACEFS);
5076         if (ret)
5077                 return ret;
5078
5079         return single_open(file, hist_debug_show, file);
5080 }
5081
5082 const struct file_operations event_hist_debug_fops = {
5083         .open = event_hist_debug_open,
5084         .read = seq_read,
5085         .llseek = seq_lseek,
5086         .release = single_release,
5087 };
5088 #endif
5089
5090 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5091 {
5092         const char *field_name = hist_field_name(hist_field, 0);
5093
5094         if (hist_field->var.name)
5095                 seq_printf(m, "%s=", hist_field->var.name);
5096
5097         if (hist_field->flags & HIST_FIELD_FL_CPU)
5098                 seq_puts(m, "common_cpu");
5099         else if (field_name) {
5100                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5101                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5102                         seq_putc(m, '$');
5103                 seq_printf(m, "%s", field_name);
5104         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5105                 seq_puts(m, "common_timestamp");
5106
5107         if (hist_field->flags) {
5108                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5109                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5110                         const char *flags = get_hist_field_flags(hist_field);
5111
5112                         if (flags)
5113                                 seq_printf(m, ".%s", flags);
5114                 }
5115         }
5116 }
5117
5118 static int event_hist_trigger_print(struct seq_file *m,
5119                                     struct event_trigger_ops *ops,
5120                                     struct event_trigger_data *data)
5121 {
5122         struct hist_trigger_data *hist_data = data->private_data;
5123         struct hist_field *field;
5124         bool have_var = false;
5125         unsigned int i;
5126
5127         seq_puts(m, "hist:");
5128
5129         if (data->name)
5130                 seq_printf(m, "%s:", data->name);
5131
5132         seq_puts(m, "keys=");
5133
5134         for_each_hist_key_field(i, hist_data) {
5135                 field = hist_data->fields[i];
5136
5137                 if (i > hist_data->n_vals)
5138                         seq_puts(m, ",");
5139
5140                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5141                         seq_puts(m, "stacktrace");
5142                 else
5143                         hist_field_print(m, field);
5144         }
5145
5146         seq_puts(m, ":vals=");
5147
5148         for_each_hist_val_field(i, hist_data) {
5149                 field = hist_data->fields[i];
5150                 if (field->flags & HIST_FIELD_FL_VAR) {
5151                         have_var = true;
5152                         continue;
5153                 }
5154
5155                 if (i == HITCOUNT_IDX)
5156                         seq_puts(m, "hitcount");
5157                 else {
5158                         seq_puts(m, ",");
5159                         hist_field_print(m, field);
5160                 }
5161         }
5162
5163         if (have_var) {
5164                 unsigned int n = 0;
5165
5166                 seq_puts(m, ":");
5167
5168                 for_each_hist_val_field(i, hist_data) {
5169                         field = hist_data->fields[i];
5170
5171                         if (field->flags & HIST_FIELD_FL_VAR) {
5172                                 if (n++)
5173                                         seq_puts(m, ",");
5174                                 hist_field_print(m, field);
5175                         }
5176                 }
5177         }
5178
5179         seq_puts(m, ":sort=");
5180
5181         for (i = 0; i < hist_data->n_sort_keys; i++) {
5182                 struct tracing_map_sort_key *sort_key;
5183                 unsigned int idx, first_key_idx;
5184
5185                 /* skip VAR vals */
5186                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5187
5188                 sort_key = &hist_data->sort_keys[i];
5189                 idx = sort_key->field_idx;
5190
5191                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5192                         return -EINVAL;
5193
5194                 if (i > 0)
5195                         seq_puts(m, ",");
5196
5197                 if (idx == HITCOUNT_IDX)
5198                         seq_puts(m, "hitcount");
5199                 else {
5200                         if (idx >= first_key_idx)
5201                                 idx += hist_data->n_vars;
5202                         hist_field_print(m, hist_data->fields[idx]);
5203                 }
5204
5205                 if (sort_key->descending)
5206                         seq_puts(m, ".descending");
5207         }
5208         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5209         if (hist_data->enable_timestamps)
5210                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5211
5212         print_actions_spec(m, hist_data);
5213
5214         if (data->filter_str)
5215                 seq_printf(m, " if %s", data->filter_str);
5216
5217         if (data->paused)
5218                 seq_puts(m, " [paused]");
5219         else
5220                 seq_puts(m, " [active]");
5221
5222         seq_putc(m, '\n');
5223
5224         return 0;
5225 }
5226
5227 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5228                                    struct event_trigger_data *data)
5229 {
5230         struct hist_trigger_data *hist_data = data->private_data;
5231
5232         if (!data->ref && hist_data->attrs->name)
5233                 save_named_trigger(hist_data->attrs->name, data);
5234
5235         data->ref++;
5236
5237         return 0;
5238 }
5239
5240 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5241 {
5242         struct trace_event_file *file;
5243         unsigned int i;
5244         char *cmd;
5245         int ret;
5246
5247         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5248                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5249                 cmd = hist_data->field_var_hists[i]->cmd;
5250                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5251                                               "!hist", "hist", cmd);
5252                 WARN_ON_ONCE(ret < 0);
5253         }
5254 }
5255
5256 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5257                                     struct event_trigger_data *data)
5258 {
5259         struct hist_trigger_data *hist_data = data->private_data;
5260
5261         if (WARN_ON_ONCE(data->ref <= 0))
5262                 return;
5263
5264         data->ref--;
5265         if (!data->ref) {
5266                 if (data->name)
5267                         del_named_trigger(data);
5268
5269                 trigger_data_free(data);
5270
5271                 remove_hist_vars(hist_data);
5272
5273                 unregister_field_var_hists(hist_data);
5274
5275                 destroy_hist_data(hist_data);
5276         }
5277 }
5278
5279 static struct event_trigger_ops event_hist_trigger_ops = {
5280         .func                   = event_hist_trigger,
5281         .print                  = event_hist_trigger_print,
5282         .init                   = event_hist_trigger_init,
5283         .free                   = event_hist_trigger_free,
5284 };
5285
5286 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5287                                          struct event_trigger_data *data)
5288 {
5289         data->ref++;
5290
5291         save_named_trigger(data->named_data->name, data);
5292
5293         event_hist_trigger_init(ops, data->named_data);
5294
5295         return 0;
5296 }
5297
5298 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5299                                           struct event_trigger_data *data)
5300 {
5301         if (WARN_ON_ONCE(data->ref <= 0))
5302                 return;
5303
5304         event_hist_trigger_free(ops, data->named_data);
5305
5306         data->ref--;
5307         if (!data->ref) {
5308                 del_named_trigger(data);
5309                 trigger_data_free(data);
5310         }
5311 }
5312
5313 static struct event_trigger_ops event_hist_trigger_named_ops = {
5314         .func                   = event_hist_trigger,
5315         .print                  = event_hist_trigger_print,
5316         .init                   = event_hist_trigger_named_init,
5317         .free                   = event_hist_trigger_named_free,
5318 };
5319
5320 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5321                                                             char *param)
5322 {
5323         return &event_hist_trigger_ops;
5324 }
5325
5326 static void hist_clear(struct event_trigger_data *data)
5327 {
5328         struct hist_trigger_data *hist_data = data->private_data;
5329
5330         if (data->name)
5331                 pause_named_trigger(data);
5332
5333         tracepoint_synchronize_unregister();
5334
5335         tracing_map_clear(hist_data->map);
5336
5337         if (data->name)
5338                 unpause_named_trigger(data);
5339 }
5340
5341 static bool compatible_field(struct ftrace_event_field *field,
5342                              struct ftrace_event_field *test_field)
5343 {
5344         if (field == test_field)
5345                 return true;
5346         if (field == NULL || test_field == NULL)
5347                 return false;
5348         if (strcmp(field->name, test_field->name) != 0)
5349                 return false;
5350         if (strcmp(field->type, test_field->type) != 0)
5351                 return false;
5352         if (field->size != test_field->size)
5353                 return false;
5354         if (field->is_signed != test_field->is_signed)
5355                 return false;
5356
5357         return true;
5358 }
5359
5360 static bool hist_trigger_match(struct event_trigger_data *data,
5361                                struct event_trigger_data *data_test,
5362                                struct event_trigger_data *named_data,
5363                                bool ignore_filter)
5364 {
5365         struct tracing_map_sort_key *sort_key, *sort_key_test;
5366         struct hist_trigger_data *hist_data, *hist_data_test;
5367         struct hist_field *key_field, *key_field_test;
5368         unsigned int i;
5369
5370         if (named_data && (named_data != data_test) &&
5371             (named_data != data_test->named_data))
5372                 return false;
5373
5374         if (!named_data && is_named_trigger(data_test))
5375                 return false;
5376
5377         hist_data = data->private_data;
5378         hist_data_test = data_test->private_data;
5379
5380         if (hist_data->n_vals != hist_data_test->n_vals ||
5381             hist_data->n_fields != hist_data_test->n_fields ||
5382             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5383                 return false;
5384
5385         if (!ignore_filter) {
5386                 if ((data->filter_str && !data_test->filter_str) ||
5387                    (!data->filter_str && data_test->filter_str))
5388                         return false;
5389         }
5390
5391         for_each_hist_field(i, hist_data) {
5392                 key_field = hist_data->fields[i];
5393                 key_field_test = hist_data_test->fields[i];
5394
5395                 if (key_field->flags != key_field_test->flags)
5396                         return false;
5397                 if (!compatible_field(key_field->field, key_field_test->field))
5398                         return false;
5399                 if (key_field->offset != key_field_test->offset)
5400                         return false;
5401                 if (key_field->size != key_field_test->size)
5402                         return false;
5403                 if (key_field->is_signed != key_field_test->is_signed)
5404                         return false;
5405                 if (!!key_field->var.name != !!key_field_test->var.name)
5406                         return false;
5407                 if (key_field->var.name &&
5408                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5409                         return false;
5410         }
5411
5412         for (i = 0; i < hist_data->n_sort_keys; i++) {
5413                 sort_key = &hist_data->sort_keys[i];
5414                 sort_key_test = &hist_data_test->sort_keys[i];
5415
5416                 if (sort_key->field_idx != sort_key_test->field_idx ||
5417                     sort_key->descending != sort_key_test->descending)
5418                         return false;
5419         }
5420
5421         if (!ignore_filter && data->filter_str &&
5422             (strcmp(data->filter_str, data_test->filter_str) != 0))
5423                 return false;
5424
5425         if (!actions_match(hist_data, hist_data_test))
5426                 return false;
5427
5428         return true;
5429 }
5430
5431 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5432                                  struct event_trigger_data *data,
5433                                  struct trace_event_file *file)
5434 {
5435         struct hist_trigger_data *hist_data = data->private_data;
5436         struct event_trigger_data *test, *named_data = NULL;
5437         struct trace_array *tr = file->tr;
5438         int ret = 0;
5439
5440         if (hist_data->attrs->name) {
5441                 named_data = find_named_trigger(hist_data->attrs->name);
5442                 if (named_data) {
5443                         if (!hist_trigger_match(data, named_data, named_data,
5444                                                 true)) {
5445                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5446                                 ret = -EINVAL;
5447                                 goto out;
5448                         }
5449                 }
5450         }
5451
5452         if (hist_data->attrs->name && !named_data)
5453                 goto new;
5454
5455         lockdep_assert_held(&event_mutex);
5456
5457         list_for_each_entry(test, &file->triggers, list) {
5458                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5459                         if (!hist_trigger_match(data, test, named_data, false))
5460                                 continue;
5461                         if (hist_data->attrs->pause)
5462                                 test->paused = true;
5463                         else if (hist_data->attrs->cont)
5464                                 test->paused = false;
5465                         else if (hist_data->attrs->clear)
5466                                 hist_clear(test);
5467                         else {
5468                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5469                                 ret = -EEXIST;
5470                         }
5471                         goto out;
5472                 }
5473         }
5474  new:
5475         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5476                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5477                 ret = -ENOENT;
5478                 goto out;
5479         }
5480
5481         if (hist_data->attrs->pause)
5482                 data->paused = true;
5483
5484         if (named_data) {
5485                 data->private_data = named_data->private_data;
5486                 set_named_trigger_data(data, named_data);
5487                 data->ops = &event_hist_trigger_named_ops;
5488         }
5489
5490         if (data->ops->init) {
5491                 ret = data->ops->init(data->ops, data);
5492                 if (ret < 0)
5493                         goto out;
5494         }
5495
5496         if (hist_data->enable_timestamps) {
5497                 char *clock = hist_data->attrs->clock;
5498
5499                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5500                 if (ret) {
5501                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5502                         goto out;
5503                 }
5504
5505                 tracing_set_filter_buffering(file->tr, true);
5506         }
5507
5508         if (named_data)
5509                 destroy_hist_data(hist_data);
5510
5511         ret++;
5512  out:
5513         return ret;
5514 }
5515
5516 static int hist_trigger_enable(struct event_trigger_data *data,
5517                                struct trace_event_file *file)
5518 {
5519         int ret = 0;
5520
5521         list_add_tail_rcu(&data->list, &file->triggers);
5522
5523         update_cond_flag(file);
5524
5525         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5526                 list_del_rcu(&data->list);
5527                 update_cond_flag(file);
5528                 ret--;
5529         }
5530
5531         return ret;
5532 }
5533
5534 static bool have_hist_trigger_match(struct event_trigger_data *data,
5535                                     struct trace_event_file *file)
5536 {
5537         struct hist_trigger_data *hist_data = data->private_data;
5538         struct event_trigger_data *test, *named_data = NULL;
5539         bool match = false;
5540
5541         lockdep_assert_held(&event_mutex);
5542
5543         if (hist_data->attrs->name)
5544                 named_data = find_named_trigger(hist_data->attrs->name);
5545
5546         list_for_each_entry(test, &file->triggers, list) {
5547                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5548                         if (hist_trigger_match(data, test, named_data, false)) {
5549                                 match = true;
5550                                 break;
5551                         }
5552                 }
5553         }
5554
5555         return match;
5556 }
5557
5558 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5559                                     struct trace_event_file *file)
5560 {
5561         struct hist_trigger_data *hist_data = data->private_data;
5562         struct event_trigger_data *test, *named_data = NULL;
5563
5564         lockdep_assert_held(&event_mutex);
5565
5566         if (hist_data->attrs->name)
5567                 named_data = find_named_trigger(hist_data->attrs->name);
5568
5569         list_for_each_entry(test, &file->triggers, list) {
5570                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5571                         if (!hist_trigger_match(data, test, named_data, false))
5572                                 continue;
5573                         hist_data = test->private_data;
5574                         if (check_var_refs(hist_data))
5575                                 return true;
5576                         break;
5577                 }
5578         }
5579
5580         return false;
5581 }
5582
5583 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5584                                     struct event_trigger_data *data,
5585                                     struct trace_event_file *file)
5586 {
5587         struct hist_trigger_data *hist_data = data->private_data;
5588         struct event_trigger_data *test, *named_data = NULL;
5589         bool unregistered = false;
5590
5591         lockdep_assert_held(&event_mutex);
5592
5593         if (hist_data->attrs->name)
5594                 named_data = find_named_trigger(hist_data->attrs->name);
5595
5596         list_for_each_entry(test, &file->triggers, list) {
5597                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5598                         if (!hist_trigger_match(data, test, named_data, false))
5599                                 continue;
5600                         unregistered = true;
5601                         list_del_rcu(&test->list);
5602                         trace_event_trigger_enable_disable(file, 0);
5603                         update_cond_flag(file);
5604                         break;
5605                 }
5606         }
5607
5608         if (unregistered && test->ops->free)
5609                 test->ops->free(test->ops, test);
5610
5611         if (hist_data->enable_timestamps) {
5612                 if (!hist_data->remove || unregistered)
5613                         tracing_set_filter_buffering(file->tr, false);
5614         }
5615 }
5616
5617 static bool hist_file_check_refs(struct trace_event_file *file)
5618 {
5619         struct hist_trigger_data *hist_data;
5620         struct event_trigger_data *test;
5621
5622         lockdep_assert_held(&event_mutex);
5623
5624         list_for_each_entry(test, &file->triggers, list) {
5625                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5626                         hist_data = test->private_data;
5627                         if (check_var_refs(hist_data))
5628                                 return true;
5629                 }
5630         }
5631
5632         return false;
5633 }
5634
5635 static void hist_unreg_all(struct trace_event_file *file)
5636 {
5637         struct event_trigger_data *test, *n;
5638         struct hist_trigger_data *hist_data;
5639         struct synth_event *se;
5640         const char *se_name;
5641
5642         lockdep_assert_held(&event_mutex);
5643
5644         if (hist_file_check_refs(file))
5645                 return;
5646
5647         list_for_each_entry_safe(test, n, &file->triggers, list) {
5648                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5649                         hist_data = test->private_data;
5650                         list_del_rcu(&test->list);
5651                         trace_event_trigger_enable_disable(file, 0);
5652
5653                         se_name = trace_event_name(file->event_call);
5654                         se = find_synth_event(se_name);
5655                         if (se)
5656                                 se->ref--;
5657
5658                         update_cond_flag(file);
5659                         if (hist_data->enable_timestamps)
5660                                 tracing_set_filter_buffering(file->tr, false);
5661                         if (test->ops->free)
5662                                 test->ops->free(test->ops, test);
5663                 }
5664         }
5665 }
5666
5667 static int event_hist_trigger_func(struct event_command *cmd_ops,
5668                                    struct trace_event_file *file,
5669                                    char *glob, char *cmd, char *param)
5670 {
5671         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5672         struct event_trigger_data *trigger_data;
5673         struct hist_trigger_attrs *attrs;
5674         struct event_trigger_ops *trigger_ops;
5675         struct hist_trigger_data *hist_data;
5676         struct synth_event *se;
5677         const char *se_name;
5678         bool remove = false;
5679         char *trigger, *p;
5680         int ret = 0;
5681
5682         lockdep_assert_held(&event_mutex);
5683
5684         if (glob && strlen(glob)) {
5685                 hist_err_clear();
5686                 last_cmd_set(file, param);
5687         }
5688
5689         if (!param)
5690                 return -EINVAL;
5691
5692         if (glob[0] == '!')
5693                 remove = true;
5694
5695         /*
5696          * separate the trigger from the filter (k:v [if filter])
5697          * allowing for whitespace in the trigger
5698          */
5699         p = trigger = param;
5700         do {
5701                 p = strstr(p, "if");
5702                 if (!p)
5703                         break;
5704                 if (p == param)
5705                         return -EINVAL;
5706                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5707                         p++;
5708                         continue;
5709                 }
5710                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5711                         return -EINVAL;
5712                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5713                         p++;
5714                         continue;
5715                 }
5716                 break;
5717         } while (p);
5718
5719         if (!p)
5720                 param = NULL;
5721         else {
5722                 *(p - 1) = '\0';
5723                 param = strstrip(p);
5724                 trigger = strstrip(trigger);
5725         }
5726
5727         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5728         if (IS_ERR(attrs))
5729                 return PTR_ERR(attrs);
5730
5731         if (attrs->map_bits)
5732                 hist_trigger_bits = attrs->map_bits;
5733
5734         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5735         if (IS_ERR(hist_data)) {
5736                 destroy_hist_trigger_attrs(attrs);
5737                 return PTR_ERR(hist_data);
5738         }
5739
5740         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5741
5742         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5743         if (!trigger_data) {
5744                 ret = -ENOMEM;
5745                 goto out_free;
5746         }
5747
5748         trigger_data->count = -1;
5749         trigger_data->ops = trigger_ops;
5750         trigger_data->cmd_ops = cmd_ops;
5751
5752         INIT_LIST_HEAD(&trigger_data->list);
5753         RCU_INIT_POINTER(trigger_data->filter, NULL);
5754
5755         trigger_data->private_data = hist_data;
5756
5757         /* if param is non-empty, it's supposed to be a filter */
5758         if (param && cmd_ops->set_filter) {
5759                 ret = cmd_ops->set_filter(param, trigger_data, file);
5760                 if (ret < 0)
5761                         goto out_free;
5762         }
5763
5764         if (remove) {
5765                 if (!have_hist_trigger_match(trigger_data, file))
5766                         goto out_free;
5767
5768                 if (hist_trigger_check_refs(trigger_data, file)) {
5769                         ret = -EBUSY;
5770                         goto out_free;
5771                 }
5772
5773                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5774                 se_name = trace_event_name(file->event_call);
5775                 se = find_synth_event(se_name);
5776                 if (se)
5777                         se->ref--;
5778                 ret = 0;
5779                 goto out_free;
5780         }
5781
5782         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5783         /*
5784          * The above returns on success the # of triggers registered,
5785          * but if it didn't register any it returns zero.  Consider no
5786          * triggers registered a failure too.
5787          */
5788         if (!ret) {
5789                 if (!(attrs->pause || attrs->cont || attrs->clear))
5790                         ret = -ENOENT;
5791                 goto out_free;
5792         } else if (ret < 0)
5793                 goto out_free;
5794
5795         if (get_named_trigger_data(trigger_data))
5796                 goto enable;
5797
5798         if (has_hist_vars(hist_data))
5799                 save_hist_vars(hist_data);
5800
5801         ret = create_actions(hist_data);
5802         if (ret)
5803                 goto out_unreg;
5804
5805         ret = tracing_map_init(hist_data->map);
5806         if (ret)
5807                 goto out_unreg;
5808 enable:
5809         ret = hist_trigger_enable(trigger_data, file);
5810         if (ret)
5811                 goto out_unreg;
5812
5813         se_name = trace_event_name(file->event_call);
5814         se = find_synth_event(se_name);
5815         if (se)
5816                 se->ref++;
5817         /* Just return zero, not the number of registered triggers */
5818         ret = 0;
5819  out:
5820         if (ret == 0)
5821                 hist_err_clear();
5822
5823         return ret;
5824  out_unreg:
5825         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5826  out_free:
5827         if (cmd_ops->set_filter)
5828                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5829
5830         remove_hist_vars(hist_data);
5831
5832         kfree(trigger_data);
5833
5834         destroy_hist_data(hist_data);
5835         goto out;
5836 }
5837
5838 static struct event_command trigger_hist_cmd = {
5839         .name                   = "hist",
5840         .trigger_type           = ETT_EVENT_HIST,
5841         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5842         .func                   = event_hist_trigger_func,
5843         .reg                    = hist_register_trigger,
5844         .unreg                  = hist_unregister_trigger,
5845         .unreg_all              = hist_unreg_all,
5846         .get_trigger_ops        = event_hist_get_trigger_ops,
5847         .set_filter             = set_trigger_filter,
5848 };
5849
5850 __init int register_trigger_hist_cmd(void)
5851 {
5852         int ret;
5853
5854         ret = register_event_command(&trigger_hist_cmd);
5855         WARN_ON(ret < 0);
5856
5857         return ret;
5858 }
5859
5860 static void
5861 hist_enable_trigger(struct event_trigger_data *data,
5862                     struct trace_buffer *buffer,  void *rec,
5863                     struct ring_buffer_event *event)
5864 {
5865         struct enable_trigger_data *enable_data = data->private_data;
5866         struct event_trigger_data *test;
5867
5868         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5869                                 lockdep_is_held(&event_mutex)) {
5870                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5871                         if (enable_data->enable)
5872                                 test->paused = false;
5873                         else
5874                                 test->paused = true;
5875                 }
5876         }
5877 }
5878
5879 static void
5880 hist_enable_count_trigger(struct event_trigger_data *data,
5881                           struct trace_buffer *buffer,  void *rec,
5882                           struct ring_buffer_event *event)
5883 {
5884         if (!data->count)
5885                 return;
5886
5887         if (data->count != -1)
5888                 (data->count)--;
5889
5890         hist_enable_trigger(data, buffer, rec, event);
5891 }
5892
5893 static struct event_trigger_ops hist_enable_trigger_ops = {
5894         .func                   = hist_enable_trigger,
5895         .print                  = event_enable_trigger_print,
5896         .init                   = event_trigger_init,
5897         .free                   = event_enable_trigger_free,
5898 };
5899
5900 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5901         .func                   = hist_enable_count_trigger,
5902         .print                  = event_enable_trigger_print,
5903         .init                   = event_trigger_init,
5904         .free                   = event_enable_trigger_free,
5905 };
5906
5907 static struct event_trigger_ops hist_disable_trigger_ops = {
5908         .func                   = hist_enable_trigger,
5909         .print                  = event_enable_trigger_print,
5910         .init                   = event_trigger_init,
5911         .free                   = event_enable_trigger_free,
5912 };
5913
5914 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5915         .func                   = hist_enable_count_trigger,
5916         .print                  = event_enable_trigger_print,
5917         .init                   = event_trigger_init,
5918         .free                   = event_enable_trigger_free,
5919 };
5920
5921 static struct event_trigger_ops *
5922 hist_enable_get_trigger_ops(char *cmd, char *param)
5923 {
5924         struct event_trigger_ops *ops;
5925         bool enable;
5926
5927         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5928
5929         if (enable)
5930                 ops = param ? &hist_enable_count_trigger_ops :
5931                         &hist_enable_trigger_ops;
5932         else
5933                 ops = param ? &hist_disable_count_trigger_ops :
5934                         &hist_disable_trigger_ops;
5935
5936         return ops;
5937 }
5938
5939 static void hist_enable_unreg_all(struct trace_event_file *file)
5940 {
5941         struct event_trigger_data *test, *n;
5942
5943         list_for_each_entry_safe(test, n, &file->triggers, list) {
5944                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5945                         list_del_rcu(&test->list);
5946                         update_cond_flag(file);
5947                         trace_event_trigger_enable_disable(file, 0);
5948                         if (test->ops->free)
5949                                 test->ops->free(test->ops, test);
5950                 }
5951         }
5952 }
5953
5954 static struct event_command trigger_hist_enable_cmd = {
5955         .name                   = ENABLE_HIST_STR,
5956         .trigger_type           = ETT_HIST_ENABLE,
5957         .func                   = event_enable_trigger_func,
5958         .reg                    = event_enable_register_trigger,
5959         .unreg                  = event_enable_unregister_trigger,
5960         .unreg_all              = hist_enable_unreg_all,
5961         .get_trigger_ops        = hist_enable_get_trigger_ops,
5962         .set_filter             = set_trigger_filter,
5963 };
5964
5965 static struct event_command trigger_hist_disable_cmd = {
5966         .name                   = DISABLE_HIST_STR,
5967         .trigger_type           = ETT_HIST_ENABLE,
5968         .func                   = event_enable_trigger_func,
5969         .reg                    = event_enable_register_trigger,
5970         .unreg                  = event_enable_unregister_trigger,
5971         .unreg_all              = hist_enable_unreg_all,
5972         .get_trigger_ops        = hist_enable_get_trigger_ops,
5973         .set_filter             = set_trigger_filter,
5974 };
5975
5976 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5977 {
5978         unregister_event_command(&trigger_hist_enable_cmd);
5979         unregister_event_command(&trigger_hist_disable_cmd);
5980 }
5981
5982 __init int register_trigger_hist_enable_disable_cmds(void)
5983 {
5984         int ret;
5985
5986         ret = register_event_command(&trigger_hist_enable_cmd);
5987         if (WARN_ON(ret < 0))
5988                 return ret;
5989         ret = register_event_command(&trigger_hist_disable_cmd);
5990         if (WARN_ON(ret < 0))
5991                 unregister_trigger_hist_enable_disable_cmds();
5992
5993         return ret;
5994 }