kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  69
  70 #undef C
  71 #define C(a, b)         HIST_ERR_##a
  72
  73 enum { ERRORS };
  74
  75 #undef C
  76 #define C(a, b)         b
  77
  78 static const char *err_text[] = { ERRORS };
  79
  80 struct hist_field;
  81
  82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  83                                 struct tracing_map_elt *elt,
  84                                 struct ring_buffer_event *rbe,
  85                                 void *event);
  86
  87 #define HIST_FIELD_OPERANDS_MAX 2
  88 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  89 #define HIST_ACTIONS_MAX        8
  90
  91 enum field_op_id {
  92         FIELD_OP_NONE,
  93         FIELD_OP_PLUS,
  94         FIELD_OP_MINUS,
  95         FIELD_OP_UNARY_MINUS,
  96 };
  97
  98 /*
  99  * A hist_var (histogram variable) contains variable information for
 100  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 101  * flag set.  A hist_var has a variable name e.g. ts0, and is
 102  * associated with a given histogram trigger, as specified by
 103  * hist_data.  The hist_var idx is the unique index assigned to the
 104  * variable by the hist trigger's tracing_map.  The idx is what is
 105  * used to set a variable's value and, by a variable reference, to
 106  * retrieve it.
 107  */
 108 struct hist_var {
 109         char                            *name;
 110         struct hist_trigger_data        *hist_data;
 111         unsigned int                    idx;
 112 };
 113
 114 struct hist_field {
 115         struct ftrace_event_field       *field;
 116         unsigned long                   flags;
 117         hist_field_fn_t                 fn;
 118         unsigned int                    ref;
 119         unsigned int                    size;
 120         unsigned int                    offset;
 121         unsigned int                    is_signed;
 122         const char                      *type;
 123         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 124         struct hist_trigger_data        *hist_data;
 125
 126         /*
 127          * Variable fields contain variable-specific info in var.
 128          */
 129         struct hist_var                 var;
 130         enum field_op_id                operator;
 131         char                            *system;
 132         char                            *event_name;
 133
 134         /*
 135          * The name field is used for EXPR and VAR_REF fields.  VAR
 136          * fields contain the variable name in var.name.
 137          */
 138         char                            *name;
 139
 140         /*
 141          * When a histogram trigger is hit, if it has any references
 142          * to variables, the values of those variables are collected
 143          * into a var_ref_vals array by resolve_var_refs().  The
 144          * current value of each variable is read from the tracing_map
 145          * using the hist field's hist_var.idx and entered into the
 146          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 147          */
 148         unsigned int                    var_ref_idx;
 149         bool                            read_once;
 150 };
 151
 152 static u64 hist_field_none(struct hist_field *field,
 153                            struct tracing_map_elt *elt,
 154                            struct ring_buffer_event *rbe,
 155                            void *event)
 156 {
 157         return 0;
 158 }
 159
 160 static u64 hist_field_counter(struct hist_field *field,
 161                               struct tracing_map_elt *elt,
 162                               struct ring_buffer_event *rbe,
 163                               void *event)
 164 {
 165         return 1;
 166 }
 167
 168 static u64 hist_field_string(struct hist_field *hist_field,
 169                              struct tracing_map_elt *elt,
 170                              struct ring_buffer_event *rbe,
 171                              void *event)
 172 {
 173         char *addr = (char *)(event + hist_field->field->offset);
 174
 175         return (u64)(unsigned long)addr;
 176 }
 177
 178 static u64 hist_field_dynstring(struct hist_field *hist_field,
 179                                 struct tracing_map_elt *elt,
 180                                 struct ring_buffer_event *rbe,
 181                                 void *event)
 182 {
 183         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 184         int str_loc = str_item & 0xffff;
 185         char *addr = (char *)(event + str_loc);
 186
 187         return (u64)(unsigned long)addr;
 188 }
 189
 190 static u64 hist_field_pstring(struct hist_field *hist_field,
 191                               struct tracing_map_elt *elt,
 192                               struct ring_buffer_event *rbe,
 193                               void *event)
 194 {
 195         char **addr = (char **)(event + hist_field->field->offset);
 196
 197         return (u64)(unsigned long)*addr;
 198 }
 199
 200 static u64 hist_field_log2(struct hist_field *hist_field,
 201                            struct tracing_map_elt *elt,
 202                            struct ring_buffer_event *rbe,
 203                            void *event)
 204 {
 205         struct hist_field *operand = hist_field->operands[0];
 206
 207         u64 val = operand->fn(operand, elt, rbe, event);
 208
 209         return (u64) ilog2(roundup_pow_of_two(val));
 210 }
 211
 212 static u64 hist_field_plus(struct hist_field *hist_field,
 213                            struct tracing_map_elt *elt,
 214                            struct ring_buffer_event *rbe,
 215                            void *event)
 216 {
 217         struct hist_field *operand1 = hist_field->operands[0];
 218         struct hist_field *operand2 = hist_field->operands[1];
 219
 220         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 221         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 222
 223         return val1 + val2;
 224 }
 225
 226 static u64 hist_field_minus(struct hist_field *hist_field,
 227                             struct tracing_map_elt *elt,
 228                             struct ring_buffer_event *rbe,
 229                             void *event)
 230 {
 231         struct hist_field *operand1 = hist_field->operands[0];
 232         struct hist_field *operand2 = hist_field->operands[1];
 233
 234         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 235         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 236
 237         return val1 - val2;
 238 }
 239
 240 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 241                                   struct tracing_map_elt *elt,
 242                                   struct ring_buffer_event *rbe,
 243                                   void *event)
 244 {
 245         struct hist_field *operand = hist_field->operands[0];
 246
 247         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 248         u64 val = (u64)-sval;
 249
 250         return val;
 251 }
 252
 253 #define DEFINE_HIST_FIELD_FN(type)                                      \
 254         static u64 hist_field_##type(struct hist_field *hist_field,     \
 255                                      struct tracing_map_elt *elt,       \
 256                                      struct ring_buffer_event *rbe,     \
 257                                      void *event)                       \
 258 {                                                                       \
 259         type *addr = (type *)(event + hist_field->field->offset);       \
 260                                                                         \
 261         return (u64)(unsigned long)*addr;                               \
 262 }
 263
 264 DEFINE_HIST_FIELD_FN(s64);
 265 DEFINE_HIST_FIELD_FN(u64);
 266 DEFINE_HIST_FIELD_FN(s32);
 267 DEFINE_HIST_FIELD_FN(u32);
 268 DEFINE_HIST_FIELD_FN(s16);
 269 DEFINE_HIST_FIELD_FN(u16);
 270 DEFINE_HIST_FIELD_FN(s8);
 271 DEFINE_HIST_FIELD_FN(u8);
 272
 273 #define for_each_hist_field(i, hist_data)       \
 274         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 275
 276 #define for_each_hist_val_field(i, hist_data)   \
 277         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 278
 279 #define for_each_hist_key_field(i, hist_data)   \
 280         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 281
 282 #define HIST_STACKTRACE_DEPTH   16
 283 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 284 #define HIST_STACKTRACE_SKIP    5
 285
 286 #define HITCOUNT_IDX            0
 287 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 288
 289 enum hist_field_flags {
 290         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 291         HIST_FIELD_FL_KEY               = 1 << 1,
 292         HIST_FIELD_FL_STRING            = 1 << 2,
 293         HIST_FIELD_FL_HEX               = 1 << 3,
 294         HIST_FIELD_FL_SYM               = 1 << 4,
 295         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 296         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 297         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 298         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 299         HIST_FIELD_FL_LOG2              = 1 << 9,
 300         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 301         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 302         HIST_FIELD_FL_VAR               = 1 << 12,
 303         HIST_FIELD_FL_EXPR              = 1 << 13,
 304         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 305         HIST_FIELD_FL_CPU               = 1 << 15,
 306         HIST_FIELD_FL_ALIAS             = 1 << 16,
 307 };
 308
 309 struct var_defs {
 310         unsigned int    n_vars;
 311         char            *name[TRACING_MAP_VARS_MAX];
 312         char            *expr[TRACING_MAP_VARS_MAX];
 313 };
 314
 315 struct hist_trigger_attrs {
 316         char            *keys_str;
 317         char            *vals_str;
 318         char            *sort_key_str;
 319         char            *name;
 320         char            *clock;
 321         bool            pause;
 322         bool            cont;
 323         bool            clear;
 324         bool            ts_in_usecs;
 325         unsigned int    map_bits;
 326
 327         char            *assignment_str[TRACING_MAP_VARS_MAX];
 328         unsigned int    n_assignments;
 329
 330         char            *action_str[HIST_ACTIONS_MAX];
 331         unsigned int    n_actions;
 332
 333         struct var_defs var_defs;
 334 };
 335
 336 struct field_var {
 337         struct hist_field       *var;
 338         struct hist_field       *val;
 339 };
 340
 341 struct field_var_hist {
 342         struct hist_trigger_data        *hist_data;
 343         char                            *cmd;
 344 };
 345
 346 struct hist_trigger_data {
 347         struct hist_field               *fields[HIST_FIELDS_MAX];
 348         unsigned int                    n_vals;
 349         unsigned int                    n_keys;
 350         unsigned int                    n_fields;
 351         unsigned int                    n_vars;
 352         unsigned int                    key_size;
 353         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 354         unsigned int                    n_sort_keys;
 355         struct trace_event_file         *event_file;
 356         struct hist_trigger_attrs       *attrs;
 357         struct tracing_map              *map;
 358         bool                            enable_timestamps;
 359         bool                            remove;
 360         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 361         unsigned int                    n_var_refs;
 362
 363         struct action_data              *actions[HIST_ACTIONS_MAX];
 364         unsigned int                    n_actions;
 365
 366         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 367         unsigned int                    n_field_vars;
 368         unsigned int                    n_field_var_str;
 369         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 370         unsigned int                    n_field_var_hists;
 371
 372         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 373         unsigned int                    n_save_vars;
 374         unsigned int                    n_save_var_str;
 375 };
 376
 377 struct action_data;
 378
 379 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 380                              struct tracing_map_elt *elt, void *rec,
 381                              struct ring_buffer_event *rbe, void *key,
 382                              struct action_data *data, u64 *var_ref_vals);
 383
 384 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 385
 386 enum handler_id {
 387         HANDLER_ONMATCH = 1,
 388         HANDLER_ONMAX,
 389         HANDLER_ONCHANGE,
 390 };
 391
 392 enum action_id {
 393         ACTION_SAVE = 1,
 394         ACTION_TRACE,
 395         ACTION_SNAPSHOT,
 396 };
 397
 398 struct action_data {
 399         enum handler_id         handler;
 400         enum action_id          action;
 401         char                    *action_name;
 402         action_fn_t             fn;
 403
 404         unsigned int            n_params;
 405         char                    *params[SYNTH_FIELDS_MAX];
 406
 407         /*
 408          * When a histogram trigger is hit, the values of any
 409          * references to variables, including variables being passed
 410          * as parameters to synthetic events, are collected into a
 411          * var_ref_vals array.  This var_ref_idx array is an array of
 412          * indices into the var_ref_vals array, one for each synthetic
 413          * event param, and is passed to the synthetic event
 414          * invocation.
 415          */
 416         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 417         struct synth_event      *synth_event;
 418         bool                    use_trace_keyword;
 419         char                    *synth_event_name;
 420
 421         union {
 422                 struct {
 423                         char                    *event;
 424                         char                    *event_system;
 425                 } match_data;
 426
 427                 struct {
 428                         /*
 429                          * var_str contains the $-unstripped variable
 430                          * name referenced by var_ref, and used when
 431                          * printing the action.  Because var_ref
 432                          * creation is deferred to create_actions(),
 433                          * we need a per-action way to save it until
 434                          * then, thus var_str.
 435                          */
 436                         char                    *var_str;
 437
 438                         /*
 439                          * var_ref refers to the variable being
 440                          * tracked e.g onmax($var).
 441                          */
 442                         struct hist_field       *var_ref;
 443
 444                         /*
 445                          * track_var contains the 'invisible' tracking
 446                          * variable created to keep the current
 447                          * e.g. max value.
 448                          */
 449                         struct hist_field       *track_var;
 450
 451                         check_track_val_fn_t    check_val;
 452                         action_fn_t             save_data;
 453                 } track_data;
 454         };
 455 };
 456
 457 struct track_data {
 458         u64                             track_val;
 459         bool                            updated;
 460
 461         unsigned int                    key_len;
 462         void                            *key;
 463         struct tracing_map_elt          elt;
 464
 465         struct action_data              *action_data;
 466         struct hist_trigger_data        *hist_data;
 467 };
 468
 469 struct hist_elt_data {
 470         char *comm;
 471         u64 *var_ref_vals;
 472         char *field_var_str[SYNTH_FIELDS_MAX];
 473 };
 474
 475 struct snapshot_context {
 476         struct tracing_map_elt  *elt;
 477         void                    *key;
 478 };
 479
 480 static void track_data_free(struct track_data *track_data)
 481 {
 482         struct hist_elt_data *elt_data;
 483
 484         if (!track_data)
 485                 return;
 486
 487         kfree(track_data->key);
 488
 489         elt_data = track_data->elt.private_data;
 490         if (elt_data) {
 491                 kfree(elt_data->comm);
 492                 kfree(elt_data);
 493         }
 494
 495         kfree(track_data);
 496 }
 497
 498 static struct track_data *track_data_alloc(unsigned int key_len,
 499                                            struct action_data *action_data,
 500                                            struct hist_trigger_data *hist_data)
 501 {
 502         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 503         struct hist_elt_data *elt_data;
 504
 505         if (!data)
 506                 return ERR_PTR(-ENOMEM);
 507
 508         data->key = kzalloc(key_len, GFP_KERNEL);
 509         if (!data->key) {
 510                 track_data_free(data);
 511                 return ERR_PTR(-ENOMEM);
 512         }
 513
 514         data->key_len = key_len;
 515         data->action_data = action_data;
 516         data->hist_data = hist_data;
 517
 518         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 519         if (!elt_data) {
 520                 track_data_free(data);
 521                 return ERR_PTR(-ENOMEM);
 522         }
 523
 524         data->elt.private_data = elt_data;
 525
 526         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 527         if (!elt_data->comm) {
 528                 track_data_free(data);
 529                 return ERR_PTR(-ENOMEM);
 530         }
 531
 532         return data;
 533 }
 534
 535 static char last_cmd[MAX_FILTER_STR_VAL];
 536 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 537
 538 static int errpos(char *str)
 539 {
 540         return err_pos(last_cmd, str);
 541 }
 542
 543 static void last_cmd_set(struct trace_event_file *file, char *str)
 544 {
 545         const char *system = NULL, *name = NULL;
 546         struct trace_event_call *call;
 547
 548         if (!str)
 549                 return;
 550
 551         strcpy(last_cmd, "hist:");
 552         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 553
 554         if (file) {
 555                 call = file->event_call;
 556                 system = call->class->system;
 557                 if (system) {
 558                         name = trace_event_name(call);
 559                         if (!name)
 560                                 system = NULL;
 561                 }
 562         }
 563
 564         if (system)
 565                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 566 }
 567
 568 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 569 {
 570         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 571                         err_type, err_pos);
 572 }
 573
 574 static void hist_err_clear(void)
 575 {
 576         last_cmd[0] = '\0';
 577         last_cmd_loc[0] = '\0';
 578 }
 579
 580 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 581                                     unsigned int *var_ref_idx);
 582
 583 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 584                                unsigned int *var_ref_idx)
 585 {
 586         struct tracepoint *tp = event->tp;
 587
 588         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 589                 struct tracepoint_func *probe_func_ptr;
 590                 synth_probe_func_t probe_func;
 591                 void *__data;
 592
 593                 if (!(cpu_online(raw_smp_processor_id())))
 594                         return;
 595
 596                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 597                 if (probe_func_ptr) {
 598                         do {
 599                                 probe_func = probe_func_ptr->func;
 600                                 __data = probe_func_ptr->data;
 601                                 probe_func(__data, var_ref_vals, var_ref_idx);
 602                         } while ((++probe_func_ptr)->func);
 603                 }
 604         }
 605 }
 606
 607 static void action_trace(struct hist_trigger_data *hist_data,
 608                          struct tracing_map_elt *elt, void *rec,
 609                          struct ring_buffer_event *rbe, void *key,
 610                          struct action_data *data, u64 *var_ref_vals)
 611 {
 612         struct synth_event *event = data->synth_event;
 613
 614         trace_synth(event, var_ref_vals, data->var_ref_idx);
 615 }
 616
 617 struct hist_var_data {
 618         struct list_head list;
 619         struct hist_trigger_data *hist_data;
 620 };
 621
 622 static u64 hist_field_timestamp(struct hist_field *hist_field,
 623                                 struct tracing_map_elt *elt,
 624                                 struct ring_buffer_event *rbe,
 625                                 void *event)
 626 {
 627         struct hist_trigger_data *hist_data = hist_field->hist_data;
 628         struct trace_array *tr = hist_data->event_file->tr;
 629
 630         u64 ts = ring_buffer_event_time_stamp(rbe);
 631
 632         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 633                 ts = ns2usecs(ts);
 634
 635         return ts;
 636 }
 637
 638 static u64 hist_field_cpu(struct hist_field *hist_field,
 639                           struct tracing_map_elt *elt,
 640                           struct ring_buffer_event *rbe,
 641                           void *event)
 642 {
 643         int cpu = smp_processor_id();
 644
 645         return cpu;
 646 }
 647
 648 /**
 649  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 650  * @hist_field: The VAR_REF field to check
 651  * @var_data: The hist trigger that owns the variable
 652  * @var_idx: The trigger variable identifier
 653  *
 654  * Check the given VAR_REF field to see whether or not it references
 655  * the given variable associated with the given trigger.
 656  *
 657  * Return: The VAR_REF field if it does reference the variable, NULL if not
 658  */
 659 static struct hist_field *
 660 check_field_for_var_ref(struct hist_field *hist_field,
 661                         struct hist_trigger_data *var_data,
 662                         unsigned int var_idx)
 663 {
 664         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 665
 666         if (hist_field && hist_field->var.idx == var_idx &&
 667             hist_field->var.hist_data == var_data)
 668                 return hist_field;
 669
 670         return NULL;
 671 }
 672
 673 /**
 674  * find_var_ref - Check if a trigger has a reference to a trigger variable
 675  * @hist_data: The hist trigger that might have a reference to the variable
 676  * @var_data: The hist trigger that owns the variable
 677  * @var_idx: The trigger variable identifier
 678  *
 679  * Check the list of var_refs[] on the first hist trigger to see
 680  * whether any of them are references to the variable on the second
 681  * trigger.
 682  *
 683  * Return: The VAR_REF field referencing the variable if so, NULL if not
 684  */
 685 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 686                                        struct hist_trigger_data *var_data,
 687                                        unsigned int var_idx)
 688 {
 689         struct hist_field *hist_field;
 690         unsigned int i;
 691
 692         for (i = 0; i < hist_data->n_var_refs; i++) {
 693                 hist_field = hist_data->var_refs[i];
 694                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 695                         return hist_field;
 696         }
 697
 698         return NULL;
 699 }
 700
 701 /**
 702  * find_any_var_ref - Check if there is a reference to a given trigger variable
 703  * @hist_data: The hist trigger
 704  * @var_idx: The trigger variable identifier
 705  *
 706  * Check to see whether the given variable is currently referenced by
 707  * any other trigger.
 708  *
 709  * The trigger the variable is defined on is explicitly excluded - the
 710  * assumption being that a self-reference doesn't prevent a trigger
 711  * from being removed.
 712  *
 713  * Return: The VAR_REF field referencing the variable if so, NULL if not
 714  */
 715 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 716                                            unsigned int var_idx)
 717 {
 718         struct trace_array *tr = hist_data->event_file->tr;
 719         struct hist_field *found = NULL;
 720         struct hist_var_data *var_data;
 721
 722         list_for_each_entry(var_data, &tr->hist_vars, list) {
 723                 if (var_data->hist_data == hist_data)
 724                         continue;
 725                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 726                 if (found)
 727                         break;
 728         }
 729
 730         return found;
 731 }
 732
 733 /**
 734  * check_var_refs - Check if there is a reference to any of trigger's variables
 735  * @hist_data: The hist trigger
 736  *
 737  * A trigger can define one or more variables.  If any one of them is
 738  * currently referenced by any other trigger, this function will
 739  * determine that.
 740
 741  * Typically used to determine whether or not a trigger can be removed
 742  * - if there are any references to a trigger's variables, it cannot.
 743  *
 744  * Return: True if there is a reference to any of trigger's variables
 745  */
 746 static bool check_var_refs(struct hist_trigger_data *hist_data)
 747 {
 748         struct hist_field *field;
 749         bool found = false;
 750         int i;
 751
 752         for_each_hist_field(i, hist_data) {
 753                 field = hist_data->fields[i];
 754                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 755                         if (find_any_var_ref(hist_data, field->var.idx)) {
 756                                 found = true;
 757                                 break;
 758                         }
 759                 }
 760         }
 761
 762         return found;
 763 }
 764
 765 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 766 {
 767         struct trace_array *tr = hist_data->event_file->tr;
 768         struct hist_var_data *var_data, *found = NULL;
 769
 770         list_for_each_entry(var_data, &tr->hist_vars, list) {
 771                 if (var_data->hist_data == hist_data) {
 772                         found = var_data;
 773                         break;
 774                 }
 775         }
 776
 777         return found;
 778 }
 779
 780 static bool field_has_hist_vars(struct hist_field *hist_field,
 781                                 unsigned int level)
 782 {
 783         int i;
 784
 785         if (level > 3)
 786                 return false;
 787
 788         if (!hist_field)
 789                 return false;
 790
 791         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 792             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 793                 return true;
 794
 795         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 796                 struct hist_field *operand;
 797
 798                 operand = hist_field->operands[i];
 799                 if (field_has_hist_vars(operand, level + 1))
 800                         return true;
 801         }
 802
 803         return false;
 804 }
 805
 806 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 807 {
 808         struct hist_field *hist_field;
 809         int i;
 810
 811         for_each_hist_field(i, hist_data) {
 812                 hist_field = hist_data->fields[i];
 813                 if (field_has_hist_vars(hist_field, 0))
 814                         return true;
 815         }
 816
 817         return false;
 818 }
 819
 820 static int save_hist_vars(struct hist_trigger_data *hist_data)
 821 {
 822         struct trace_array *tr = hist_data->event_file->tr;
 823         struct hist_var_data *var_data;
 824
 825         var_data = find_hist_vars(hist_data);
 826         if (var_data)
 827                 return 0;
 828
 829         if (tracing_check_open_get_tr(tr))
 830                 return -ENODEV;
 831
 832         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 833         if (!var_data) {
 834                 trace_array_put(tr);
 835                 return -ENOMEM;
 836         }
 837
 838         var_data->hist_data = hist_data;
 839         list_add(&var_data->list, &tr->hist_vars);
 840
 841         return 0;
 842 }
 843
 844 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 845 {
 846         struct trace_array *tr = hist_data->event_file->tr;
 847         struct hist_var_data *var_data;
 848
 849         var_data = find_hist_vars(hist_data);
 850         if (!var_data)
 851                 return;
 852
 853         if (WARN_ON(check_var_refs(hist_data)))
 854                 return;
 855
 856         list_del(&var_data->list);
 857
 858         kfree(var_data);
 859
 860         trace_array_put(tr);
 861 }
 862
 863 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 864                                          const char *var_name)
 865 {
 866         struct hist_field *hist_field, *found = NULL;
 867         int i;
 868
 869         for_each_hist_field(i, hist_data) {
 870                 hist_field = hist_data->fields[i];
 871                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 872                     strcmp(hist_field->var.name, var_name) == 0) {
 873                         found = hist_field;
 874                         break;
 875                 }
 876         }
 877
 878         return found;
 879 }
 880
 881 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 882                                    struct trace_event_file *file,
 883                                    const char *var_name)
 884 {
 885         struct hist_trigger_data *test_data;
 886         struct event_trigger_data *test;
 887         struct hist_field *hist_field;
 888
 889         lockdep_assert_held(&event_mutex);
 890
 891         hist_field = find_var_field(hist_data, var_name);
 892         if (hist_field)
 893                 return hist_field;
 894
 895         list_for_each_entry(test, &file->triggers, list) {
 896                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 897                         test_data = test->private_data;
 898                         hist_field = find_var_field(test_data, var_name);
 899                         if (hist_field)
 900                                 return hist_field;
 901                 }
 902         }
 903
 904         return NULL;
 905 }
 906
 907 static struct trace_event_file *find_var_file(struct trace_array *tr,
 908                                               char *system,
 909                                               char *event_name,
 910                                               char *var_name)
 911 {
 912         struct hist_trigger_data *var_hist_data;
 913         struct hist_var_data *var_data;
 914         struct trace_event_file *file, *found = NULL;
 915
 916         if (system)
 917                 return find_event_file(tr, system, event_name);
 918
 919         list_for_each_entry(var_data, &tr->hist_vars, list) {
 920                 var_hist_data = var_data->hist_data;
 921                 file = var_hist_data->event_file;
 922                 if (file == found)
 923                         continue;
 924
 925                 if (find_var_field(var_hist_data, var_name)) {
 926                         if (found) {
 927                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 928                                 return NULL;
 929                         }
 930
 931                         found = file;
 932                 }
 933         }
 934
 935         return found;
 936 }
 937
 938 static struct hist_field *find_file_var(struct trace_event_file *file,
 939                                         const char *var_name)
 940 {
 941         struct hist_trigger_data *test_data;
 942         struct event_trigger_data *test;
 943         struct hist_field *hist_field;
 944
 945         lockdep_assert_held(&event_mutex);
 946
 947         list_for_each_entry(test, &file->triggers, list) {
 948                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 949                         test_data = test->private_data;
 950                         hist_field = find_var_field(test_data, var_name);
 951                         if (hist_field)
 952                                 return hist_field;
 953                 }
 954         }
 955
 956         return NULL;
 957 }
 958
 959 static struct hist_field *
 960 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 961 {
 962         struct trace_array *tr = hist_data->event_file->tr;
 963         struct hist_field *hist_field, *found = NULL;
 964         struct trace_event_file *file;
 965         unsigned int i;
 966
 967         for (i = 0; i < hist_data->n_actions; i++) {
 968                 struct action_data *data = hist_data->actions[i];
 969
 970                 if (data->handler == HANDLER_ONMATCH) {
 971                         char *system = data->match_data.event_system;
 972                         char *event_name = data->match_data.event;
 973
 974                         file = find_var_file(tr, system, event_name, var_name);
 975                         if (!file)
 976                                 continue;
 977                         hist_field = find_file_var(file, var_name);
 978                         if (hist_field) {
 979                                 if (found) {
 980                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
 981                                                  errpos(var_name));
 982                                         return ERR_PTR(-EINVAL);
 983                                 }
 984
 985                                 found = hist_field;
 986                         }
 987                 }
 988         }
 989         return found;
 990 }
 991
 992 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
 993                                          char *system,
 994                                          char *event_name,
 995                                          char *var_name)
 996 {
 997         struct trace_array *tr = hist_data->event_file->tr;
 998         struct hist_field *hist_field = NULL;
 999         struct trace_event_file *file;
1000
1001         if (!system || !event_name) {
1002                 hist_field = find_match_var(hist_data, var_name);
1003                 if (IS_ERR(hist_field))
1004                         return NULL;
1005                 if (hist_field)
1006                         return hist_field;
1007         }
1008
1009         file = find_var_file(tr, system, event_name, var_name);
1010         if (!file)
1011                 return NULL;
1012
1013         hist_field = find_file_var(file, var_name);
1014
1015         return hist_field;
1016 }
1017
1018 static u64 hist_field_var_ref(struct hist_field *hist_field,
1019                               struct tracing_map_elt *elt,
1020                               struct ring_buffer_event *rbe,
1021                               void *event)
1022 {
1023         struct hist_elt_data *elt_data;
1024         u64 var_val = 0;
1025
1026         if (WARN_ON_ONCE(!elt))
1027                 return var_val;
1028
1029         elt_data = elt->private_data;
1030         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1031
1032         return var_val;
1033 }
1034
1035 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1036                              u64 *var_ref_vals, bool self)
1037 {
1038         struct hist_trigger_data *var_data;
1039         struct tracing_map_elt *var_elt;
1040         struct hist_field *hist_field;
1041         unsigned int i, var_idx;
1042         bool resolved = true;
1043         u64 var_val = 0;
1044
1045         for (i = 0; i < hist_data->n_var_refs; i++) {
1046                 hist_field = hist_data->var_refs[i];
1047                 var_idx = hist_field->var.idx;
1048                 var_data = hist_field->var.hist_data;
1049
1050                 if (var_data == NULL) {
1051                         resolved = false;
1052                         break;
1053                 }
1054
1055                 if ((self && var_data != hist_data) ||
1056                     (!self && var_data == hist_data))
1057                         continue;
1058
1059                 var_elt = tracing_map_lookup(var_data->map, key);
1060                 if (!var_elt) {
1061                         resolved = false;
1062                         break;
1063                 }
1064
1065                 if (!tracing_map_var_set(var_elt, var_idx)) {
1066                         resolved = false;
1067                         break;
1068                 }
1069
1070                 if (self || !hist_field->read_once)
1071                         var_val = tracing_map_read_var(var_elt, var_idx);
1072                 else
1073                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1074
1075                 var_ref_vals[i] = var_val;
1076         }
1077
1078         return resolved;
1079 }
1080
1081 static const char *hist_field_name(struct hist_field *field,
1082                                    unsigned int level)
1083 {
1084         const char *field_name = "";
1085
1086         if (level > 1)
1087                 return field_name;
1088
1089         if (field->field)
1090                 field_name = field->field->name;
1091         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1092                  field->flags & HIST_FIELD_FL_ALIAS)
1093                 field_name = hist_field_name(field->operands[0], ++level);
1094         else if (field->flags & HIST_FIELD_FL_CPU)
1095                 field_name = "cpu";
1096         else if (field->flags & HIST_FIELD_FL_EXPR ||
1097                  field->flags & HIST_FIELD_FL_VAR_REF) {
1098                 if (field->system) {
1099                         static char full_name[MAX_FILTER_STR_VAL];
1100
1101                         strcat(full_name, field->system);
1102                         strcat(full_name, ".");
1103                         strcat(full_name, field->event_name);
1104                         strcat(full_name, ".");
1105                         strcat(full_name, field->name);
1106                         field_name = full_name;
1107                 } else
1108                         field_name = field->name;
1109         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1110                 field_name = "common_timestamp";
1111
1112         if (field_name == NULL)
1113                 field_name = "";
1114
1115         return field_name;
1116 }
1117
1118 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1119 {
1120         hist_field_fn_t fn = NULL;
1121
1122         switch (field_size) {
1123         case 8:
1124                 if (field_is_signed)
1125                         fn = hist_field_s64;
1126                 else
1127                         fn = hist_field_u64;
1128                 break;
1129         case 4:
1130                 if (field_is_signed)
1131                         fn = hist_field_s32;
1132                 else
1133                         fn = hist_field_u32;
1134                 break;
1135         case 2:
1136                 if (field_is_signed)
1137                         fn = hist_field_s16;
1138                 else
1139                         fn = hist_field_u16;
1140                 break;
1141         case 1:
1142                 if (field_is_signed)
1143                         fn = hist_field_s8;
1144                 else
1145                         fn = hist_field_u8;
1146                 break;
1147         }
1148
1149         return fn;
1150 }
1151
1152 static int parse_map_size(char *str)
1153 {
1154         unsigned long size, map_bits;
1155         int ret;
1156
1157         ret = kstrtoul(str, 0, &size);
1158         if (ret)
1159                 goto out;
1160
1161         map_bits = ilog2(roundup_pow_of_two(size));
1162         if (map_bits < TRACING_MAP_BITS_MIN ||
1163             map_bits > TRACING_MAP_BITS_MAX)
1164                 ret = -EINVAL;
1165         else
1166                 ret = map_bits;
1167  out:
1168         return ret;
1169 }
1170
1171 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1172 {
1173         unsigned int i;
1174
1175         if (!attrs)
1176                 return;
1177
1178         for (i = 0; i < attrs->n_assignments; i++)
1179                 kfree(attrs->assignment_str[i]);
1180
1181         for (i = 0; i < attrs->n_actions; i++)
1182                 kfree(attrs->action_str[i]);
1183
1184         kfree(attrs->name);
1185         kfree(attrs->sort_key_str);
1186         kfree(attrs->keys_str);
1187         kfree(attrs->vals_str);
1188         kfree(attrs->clock);
1189         kfree(attrs);
1190 }
1191
1192 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1193 {
1194         int ret = -EINVAL;
1195
1196         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1197                 return ret;
1198
1199         if ((str_has_prefix(str, "onmatch(")) ||
1200             (str_has_prefix(str, "onmax(")) ||
1201             (str_has_prefix(str, "onchange("))) {
1202                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1203                 if (!attrs->action_str[attrs->n_actions]) {
1204                         ret = -ENOMEM;
1205                         return ret;
1206                 }
1207                 attrs->n_actions++;
1208                 ret = 0;
1209         }
1210         return ret;
1211 }
1212
1213 static int parse_assignment(struct trace_array *tr,
1214                             char *str, struct hist_trigger_attrs *attrs)
1215 {
1216         int len, ret = 0;
1217
1218         if ((len = str_has_prefix(str, "key=")) ||
1219             (len = str_has_prefix(str, "keys="))) {
1220                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1221                 if (!attrs->keys_str) {
1222                         ret = -ENOMEM;
1223                         goto out;
1224                 }
1225         } else if ((len = str_has_prefix(str, "val=")) ||
1226                    (len = str_has_prefix(str, "vals=")) ||
1227                    (len = str_has_prefix(str, "values="))) {
1228                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1229                 if (!attrs->vals_str) {
1230                         ret = -ENOMEM;
1231                         goto out;
1232                 }
1233         } else if ((len = str_has_prefix(str, "sort="))) {
1234                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1235                 if (!attrs->sort_key_str) {
1236                         ret = -ENOMEM;
1237                         goto out;
1238                 }
1239         } else if (str_has_prefix(str, "name=")) {
1240                 attrs->name = kstrdup(str, GFP_KERNEL);
1241                 if (!attrs->name) {
1242                         ret = -ENOMEM;
1243                         goto out;
1244                 }
1245         } else if ((len = str_has_prefix(str, "clock="))) {
1246                 str += len;
1247
1248                 str = strstrip(str);
1249                 attrs->clock = kstrdup(str, GFP_KERNEL);
1250                 if (!attrs->clock) {
1251                         ret = -ENOMEM;
1252                         goto out;
1253                 }
1254         } else if ((len = str_has_prefix(str, "size="))) {
1255                 int map_bits = parse_map_size(str + len);
1256
1257                 if (map_bits < 0) {
1258                         ret = map_bits;
1259                         goto out;
1260                 }
1261                 attrs->map_bits = map_bits;
1262         } else {
1263                 char *assignment;
1264
1265                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1266                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1267                         ret = -EINVAL;
1268                         goto out;
1269                 }
1270
1271                 assignment = kstrdup(str, GFP_KERNEL);
1272                 if (!assignment) {
1273                         ret = -ENOMEM;
1274                         goto out;
1275                 }
1276
1277                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1278         }
1279  out:
1280         return ret;
1281 }
1282
1283 static struct hist_trigger_attrs *
1284 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1285 {
1286         struct hist_trigger_attrs *attrs;
1287         int ret = 0;
1288
1289         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1290         if (!attrs)
1291                 return ERR_PTR(-ENOMEM);
1292
1293         while (trigger_str) {
1294                 char *str = strsep(&trigger_str, ":");
1295                 char *rhs;
1296
1297                 rhs = strchr(str, '=');
1298                 if (rhs) {
1299                         if (!strlen(++rhs)) {
1300                                 ret = -EINVAL;
1301                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1302                                 goto free;
1303                         }
1304                         ret = parse_assignment(tr, str, attrs);
1305                         if (ret)
1306                                 goto free;
1307                 } else if (strcmp(str, "pause") == 0)
1308                         attrs->pause = true;
1309                 else if ((strcmp(str, "cont") == 0) ||
1310                          (strcmp(str, "continue") == 0))
1311                         attrs->cont = true;
1312                 else if (strcmp(str, "clear") == 0)
1313                         attrs->clear = true;
1314                 else {
1315                         ret = parse_action(str, attrs);
1316                         if (ret)
1317                                 goto free;
1318                 }
1319         }
1320
1321         if (!attrs->keys_str) {
1322                 ret = -EINVAL;
1323                 goto free;
1324         }
1325
1326         if (!attrs->clock) {
1327                 attrs->clock = kstrdup("global", GFP_KERNEL);
1328                 if (!attrs->clock) {
1329                         ret = -ENOMEM;
1330                         goto free;
1331                 }
1332         }
1333
1334         return attrs;
1335  free:
1336         destroy_hist_trigger_attrs(attrs);
1337
1338         return ERR_PTR(ret);
1339 }
1340
1341 static inline void save_comm(char *comm, struct task_struct *task)
1342 {
1343         if (!task->pid) {
1344                 strcpy(comm, "<idle>");
1345                 return;
1346         }
1347
1348         if (WARN_ON_ONCE(task->pid < 0)) {
1349                 strcpy(comm, "<XXX>");
1350                 return;
1351         }
1352
1353         strncpy(comm, task->comm, TASK_COMM_LEN);
1354 }
1355
1356 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1357 {
1358         unsigned int i;
1359
1360         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1361                 kfree(elt_data->field_var_str[i]);
1362
1363         kfree(elt_data->comm);
1364         kfree(elt_data);
1365 }
1366
1367 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1368 {
1369         struct hist_elt_data *elt_data = elt->private_data;
1370
1371         hist_elt_data_free(elt_data);
1372 }
1373
1374 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1375 {
1376         struct hist_trigger_data *hist_data = elt->map->private_data;
1377         unsigned int size = TASK_COMM_LEN;
1378         struct hist_elt_data *elt_data;
1379         struct hist_field *key_field;
1380         unsigned int i, n_str;
1381
1382         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1383         if (!elt_data)
1384                 return -ENOMEM;
1385
1386         for_each_hist_key_field(i, hist_data) {
1387                 key_field = hist_data->fields[i];
1388
1389                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1390                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1391                         if (!elt_data->comm) {
1392                                 kfree(elt_data);
1393                                 return -ENOMEM;
1394                         }
1395                         break;
1396                 }
1397         }
1398
1399         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
1400
1401         size = STR_VAR_LEN_MAX;
1402
1403         for (i = 0; i < n_str; i++) {
1404                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1405                 if (!elt_data->field_var_str[i]) {
1406                         hist_elt_data_free(elt_data);
1407                         return -ENOMEM;
1408                 }
1409         }
1410
1411         elt->private_data = elt_data;
1412
1413         return 0;
1414 }
1415
1416 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1417 {
1418         struct hist_elt_data *elt_data = elt->private_data;
1419
1420         if (elt_data->comm)
1421                 save_comm(elt_data->comm, current);
1422 }
1423
1424 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1425         .elt_alloc      = hist_trigger_elt_data_alloc,
1426         .elt_free       = hist_trigger_elt_data_free,
1427         .elt_init       = hist_trigger_elt_data_init,
1428 };
1429
1430 static const char *get_hist_field_flags(struct hist_field *hist_field)
1431 {
1432         const char *flags_str = NULL;
1433
1434         if (hist_field->flags & HIST_FIELD_FL_HEX)
1435                 flags_str = "hex";
1436         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1437                 flags_str = "sym";
1438         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1439                 flags_str = "sym-offset";
1440         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1441                 flags_str = "execname";
1442         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1443                 flags_str = "syscall";
1444         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1445                 flags_str = "log2";
1446         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1447                 flags_str = "usecs";
1448
1449         return flags_str;
1450 }
1451
1452 static void expr_field_str(struct hist_field *field, char *expr)
1453 {
1454         if (field->flags & HIST_FIELD_FL_VAR_REF)
1455                 strcat(expr, "$");
1456
1457         strcat(expr, hist_field_name(field, 0));
1458
1459         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1460                 const char *flags_str = get_hist_field_flags(field);
1461
1462                 if (flags_str) {
1463                         strcat(expr, ".");
1464                         strcat(expr, flags_str);
1465                 }
1466         }
1467 }
1468
1469 static char *expr_str(struct hist_field *field, unsigned int level)
1470 {
1471         char *expr;
1472
1473         if (level > 1)
1474                 return NULL;
1475
1476         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1477         if (!expr)
1478                 return NULL;
1479
1480         if (!field->operands[0]) {
1481                 expr_field_str(field, expr);
1482                 return expr;
1483         }
1484
1485         if (field->operator == FIELD_OP_UNARY_MINUS) {
1486                 char *subexpr;
1487
1488                 strcat(expr, "-(");
1489                 subexpr = expr_str(field->operands[0], ++level);
1490                 if (!subexpr) {
1491                         kfree(expr);
1492                         return NULL;
1493                 }
1494                 strcat(expr, subexpr);
1495                 strcat(expr, ")");
1496
1497                 kfree(subexpr);
1498
1499                 return expr;
1500         }
1501
1502         expr_field_str(field->operands[0], expr);
1503
1504         switch (field->operator) {
1505         case FIELD_OP_MINUS:
1506                 strcat(expr, "-");
1507                 break;
1508         case FIELD_OP_PLUS:
1509                 strcat(expr, "+");
1510                 break;
1511         default:
1512                 kfree(expr);
1513                 return NULL;
1514         }
1515
1516         expr_field_str(field->operands[1], expr);
1517
1518         return expr;
1519 }
1520
1521 static int contains_operator(char *str)
1522 {
1523         enum field_op_id field_op = FIELD_OP_NONE;
1524         char *op;
1525
1526         op = strpbrk(str, "+-");
1527         if (!op)
1528                 return FIELD_OP_NONE;
1529
1530         switch (*op) {
1531         case '-':
1532                 if (*str == '-')
1533                         field_op = FIELD_OP_UNARY_MINUS;
1534                 else
1535                         field_op = FIELD_OP_MINUS;
1536                 break;
1537         case '+':
1538                 field_op = FIELD_OP_PLUS;
1539                 break;
1540         default:
1541                 break;
1542         }
1543
1544         return field_op;
1545 }
1546
1547 static void get_hist_field(struct hist_field *hist_field)
1548 {
1549         hist_field->ref++;
1550 }
1551
1552 static void __destroy_hist_field(struct hist_field *hist_field)
1553 {
1554         if (--hist_field->ref > 1)
1555                 return;
1556
1557         kfree(hist_field->var.name);
1558         kfree(hist_field->name);
1559         kfree(hist_field->type);
1560
1561         kfree(hist_field->system);
1562         kfree(hist_field->event_name);
1563
1564         kfree(hist_field);
1565 }
1566
1567 static void destroy_hist_field(struct hist_field *hist_field,
1568                                unsigned int level)
1569 {
1570         unsigned int i;
1571
1572         if (level > 3)
1573                 return;
1574
1575         if (!hist_field)
1576                 return;
1577
1578         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1579                 return; /* var refs will be destroyed separately */
1580
1581         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1582                 destroy_hist_field(hist_field->operands[i], level + 1);
1583
1584         __destroy_hist_field(hist_field);
1585 }
1586
1587 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1588                                             struct ftrace_event_field *field,
1589                                             unsigned long flags,
1590                                             char *var_name)
1591 {
1592         struct hist_field *hist_field;
1593
1594         if (field && is_function_field(field))
1595                 return NULL;
1596
1597         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1598         if (!hist_field)
1599                 return NULL;
1600
1601         hist_field->ref = 1;
1602
1603         hist_field->hist_data = hist_data;
1604
1605         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1606                 goto out; /* caller will populate */
1607
1608         if (flags & HIST_FIELD_FL_VAR_REF) {
1609                 hist_field->fn = hist_field_var_ref;
1610                 goto out;
1611         }
1612
1613         if (flags & HIST_FIELD_FL_HITCOUNT) {
1614                 hist_field->fn = hist_field_counter;
1615                 hist_field->size = sizeof(u64);
1616                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1617                 if (!hist_field->type)
1618                         goto free;
1619                 goto out;
1620         }
1621
1622         if (flags & HIST_FIELD_FL_STACKTRACE) {
1623                 hist_field->fn = hist_field_none;
1624                 goto out;
1625         }
1626
1627         if (flags & HIST_FIELD_FL_LOG2) {
1628                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1629                 hist_field->fn = hist_field_log2;
1630                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1631                 hist_field->size = hist_field->operands[0]->size;
1632                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1633                 if (!hist_field->type)
1634                         goto free;
1635                 goto out;
1636         }
1637
1638         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1639                 hist_field->fn = hist_field_timestamp;
1640                 hist_field->size = sizeof(u64);
1641                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1642                 if (!hist_field->type)
1643                         goto free;
1644                 goto out;
1645         }
1646
1647         if (flags & HIST_FIELD_FL_CPU) {
1648                 hist_field->fn = hist_field_cpu;
1649                 hist_field->size = sizeof(int);
1650                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1651                 if (!hist_field->type)
1652                         goto free;
1653                 goto out;
1654         }
1655
1656         if (WARN_ON_ONCE(!field))
1657                 goto out;
1658
1659         if (is_string_field(field)) {
1660                 flags |= HIST_FIELD_FL_STRING;
1661
1662                 hist_field->size = MAX_FILTER_STR_VAL;
1663                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1664                 if (!hist_field->type)
1665                         goto free;
1666
1667                 if (field->filter_type == FILTER_STATIC_STRING)
1668                         hist_field->fn = hist_field_string;
1669                 else if (field->filter_type == FILTER_DYN_STRING)
1670                         hist_field->fn = hist_field_dynstring;
1671                 else
1672                         hist_field->fn = hist_field_pstring;
1673         } else {
1674                 hist_field->size = field->size;
1675                 hist_field->is_signed = field->is_signed;
1676                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1677                 if (!hist_field->type)
1678                         goto free;
1679
1680                 hist_field->fn = select_value_fn(field->size,
1681                                                  field->is_signed);
1682                 if (!hist_field->fn) {
1683                         destroy_hist_field(hist_field, 0);
1684                         return NULL;
1685                 }
1686         }
1687  out:
1688         hist_field->field = field;
1689         hist_field->flags = flags;
1690
1691         if (var_name) {
1692                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1693                 if (!hist_field->var.name)
1694                         goto free;
1695         }
1696
1697         return hist_field;
1698  free:
1699         destroy_hist_field(hist_field, 0);
1700         return NULL;
1701 }
1702
1703 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1704 {
1705         unsigned int i;
1706
1707         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1708                 if (hist_data->fields[i]) {
1709                         destroy_hist_field(hist_data->fields[i], 0);
1710                         hist_data->fields[i] = NULL;
1711                 }
1712         }
1713
1714         for (i = 0; i < hist_data->n_var_refs; i++) {
1715                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1716                 __destroy_hist_field(hist_data->var_refs[i]);
1717                 hist_data->var_refs[i] = NULL;
1718         }
1719 }
1720
1721 static int init_var_ref(struct hist_field *ref_field,
1722                         struct hist_field *var_field,
1723                         char *system, char *event_name)
1724 {
1725         int err = 0;
1726
1727         ref_field->var.idx = var_field->var.idx;
1728         ref_field->var.hist_data = var_field->hist_data;
1729         ref_field->size = var_field->size;
1730         ref_field->is_signed = var_field->is_signed;
1731         ref_field->flags |= var_field->flags &
1732                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1733
1734         if (system) {
1735                 ref_field->system = kstrdup(system, GFP_KERNEL);
1736                 if (!ref_field->system)
1737                         return -ENOMEM;
1738         }
1739
1740         if (event_name) {
1741                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1742                 if (!ref_field->event_name) {
1743                         err = -ENOMEM;
1744                         goto free;
1745                 }
1746         }
1747
1748         if (var_field->var.name) {
1749                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1750                 if (!ref_field->name) {
1751                         err = -ENOMEM;
1752                         goto free;
1753                 }
1754         } else if (var_field->name) {
1755                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1756                 if (!ref_field->name) {
1757                         err = -ENOMEM;
1758                         goto free;
1759                 }
1760         }
1761
1762         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1763         if (!ref_field->type) {
1764                 err = -ENOMEM;
1765                 goto free;
1766         }
1767  out:
1768         return err;
1769  free:
1770         kfree(ref_field->system);
1771         kfree(ref_field->event_name);
1772         kfree(ref_field->name);
1773
1774         goto out;
1775 }
1776
1777 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1778                             struct hist_field *var_field)
1779 {
1780         struct hist_field *ref_field;
1781         int i;
1782
1783         for (i = 0; i < hist_data->n_var_refs; i++) {
1784                 ref_field = hist_data->var_refs[i];
1785                 if (ref_field->var.idx == var_field->var.idx &&
1786                     ref_field->var.hist_data == var_field->hist_data)
1787                         return i;
1788         }
1789
1790         return -ENOENT;
1791 }
1792
1793 /**
1794  * create_var_ref - Create a variable reference and attach it to trigger
1795  * @hist_data: The trigger that will be referencing the variable
1796  * @var_field: The VAR field to create a reference to
1797  * @system: The optional system string
1798  * @event_name: The optional event_name string
1799  *
1800  * Given a variable hist_field, create a VAR_REF hist_field that
1801  * represents a reference to it.
1802  *
1803  * This function also adds the reference to the trigger that
1804  * now references the variable.
1805  *
1806  * Return: The VAR_REF field if successful, NULL if not
1807  */
1808 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1809                                          struct hist_field *var_field,
1810                                          char *system, char *event_name)
1811 {
1812         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1813         struct hist_field *ref_field;
1814         int i;
1815
1816         /* Check if the variable already exists */
1817         for (i = 0; i < hist_data->n_var_refs; i++) {
1818                 ref_field = hist_data->var_refs[i];
1819                 if (ref_field->var.idx == var_field->var.idx &&
1820                     ref_field->var.hist_data == var_field->hist_data) {
1821                         get_hist_field(ref_field);
1822                         return ref_field;
1823                 }
1824         }
1825
1826         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1827         if (ref_field) {
1828                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1829                         destroy_hist_field(ref_field, 0);
1830                         return NULL;
1831                 }
1832
1833                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1834                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1835         }
1836
1837         return ref_field;
1838 }
1839
1840 static bool is_var_ref(char *var_name)
1841 {
1842         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1843                 return false;
1844
1845         return true;
1846 }
1847
1848 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1849                                  char *var_name)
1850 {
1851         char *name, *field;
1852         unsigned int i;
1853
1854         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1855                 name = hist_data->attrs->var_defs.name[i];
1856
1857                 if (strcmp(var_name, name) == 0) {
1858                         field = hist_data->attrs->var_defs.expr[i];
1859                         if (contains_operator(field) || is_var_ref(field))
1860                                 continue;
1861                         return field;
1862                 }
1863         }
1864
1865         return NULL;
1866 }
1867
1868 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1869                                  char *system, char *event_name,
1870                                  char *var_name)
1871 {
1872         struct trace_event_call *call;
1873
1874         if (system && event_name) {
1875                 call = hist_data->event_file->event_call;
1876
1877                 if (strcmp(system, call->class->system) != 0)
1878                         return NULL;
1879
1880                 if (strcmp(event_name, trace_event_name(call)) != 0)
1881                         return NULL;
1882         }
1883
1884         if (!!system != !!event_name)
1885                 return NULL;
1886
1887         if (!is_var_ref(var_name))
1888                 return NULL;
1889
1890         var_name++;
1891
1892         return field_name_from_var(hist_data, var_name);
1893 }
1894
1895 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1896                                         char *system, char *event_name,
1897                                         char *var_name)
1898 {
1899         struct hist_field *var_field = NULL, *ref_field = NULL;
1900         struct trace_array *tr = hist_data->event_file->tr;
1901
1902         if (!is_var_ref(var_name))
1903                 return NULL;
1904
1905         var_name++;
1906
1907         var_field = find_event_var(hist_data, system, event_name, var_name);
1908         if (var_field)
1909                 ref_field = create_var_ref(hist_data, var_field,
1910                                            system, event_name);
1911
1912         if (!ref_field)
1913                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1914
1915         return ref_field;
1916 }
1917
1918 static struct ftrace_event_field *
1919 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1920             char *field_str, unsigned long *flags)
1921 {
1922         struct ftrace_event_field *field = NULL;
1923         char *field_name, *modifier, *str;
1924         struct trace_array *tr = file->tr;
1925
1926         modifier = str = kstrdup(field_str, GFP_KERNEL);
1927         if (!modifier)
1928                 return ERR_PTR(-ENOMEM);
1929
1930         field_name = strsep(&modifier, ".");
1931         if (modifier) {
1932                 if (strcmp(modifier, "hex") == 0)
1933                         *flags |= HIST_FIELD_FL_HEX;
1934                 else if (strcmp(modifier, "sym") == 0)
1935                         *flags |= HIST_FIELD_FL_SYM;
1936                 else if (strcmp(modifier, "sym-offset") == 0)
1937                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1938                 else if ((strcmp(modifier, "execname") == 0) &&
1939                          (strcmp(field_name, "common_pid") == 0))
1940                         *flags |= HIST_FIELD_FL_EXECNAME;
1941                 else if (strcmp(modifier, "syscall") == 0)
1942                         *flags |= HIST_FIELD_FL_SYSCALL;
1943                 else if (strcmp(modifier, "log2") == 0)
1944                         *flags |= HIST_FIELD_FL_LOG2;
1945                 else if (strcmp(modifier, "usecs") == 0)
1946                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1947                 else {
1948                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1949                         field = ERR_PTR(-EINVAL);
1950                         goto out;
1951                 }
1952         }
1953
1954         if (strcmp(field_name, "common_timestamp") == 0) {
1955                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1956                 hist_data->enable_timestamps = true;
1957                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1958                         hist_data->attrs->ts_in_usecs = true;
1959         } else if (strcmp(field_name, "cpu") == 0)
1960                 *flags |= HIST_FIELD_FL_CPU;
1961         else {
1962                 field = trace_find_event_field(file->event_call, field_name);
1963                 if (!field || !field->size) {
1964                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1965                         field = ERR_PTR(-EINVAL);
1966                         goto out;
1967                 }
1968         }
1969  out:
1970         kfree(str);
1971
1972         return field;
1973 }
1974
1975 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
1976                                        struct hist_field *var_ref,
1977                                        char *var_name)
1978 {
1979         struct hist_field *alias = NULL;
1980         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
1981
1982         alias = create_hist_field(hist_data, NULL, flags, var_name);
1983         if (!alias)
1984                 return NULL;
1985
1986         alias->fn = var_ref->fn;
1987         alias->operands[0] = var_ref;
1988
1989         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
1990                 destroy_hist_field(alias, 0);
1991                 return NULL;
1992         }
1993
1994         alias->var_ref_idx = var_ref->var_ref_idx;
1995
1996         return alias;
1997 }
1998
1999 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2000                                      struct trace_event_file *file, char *str,
2001                                      unsigned long *flags, char *var_name)
2002 {
2003         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2004         struct ftrace_event_field *field = NULL;
2005         struct hist_field *hist_field = NULL;
2006         int ret = 0;
2007
2008         s = strchr(str, '.');
2009         if (s) {
2010                 s = strchr(++s, '.');
2011                 if (s) {
2012                         ref_system = strsep(&str, ".");
2013                         if (!str) {
2014                                 ret = -EINVAL;
2015                                 goto out;
2016                         }
2017                         ref_event = strsep(&str, ".");
2018                         if (!str) {
2019                                 ret = -EINVAL;
2020                                 goto out;
2021                         }
2022                         ref_var = str;
2023                 }
2024         }
2025
2026         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2027         if (!s) {
2028                 hist_field = parse_var_ref(hist_data, ref_system,
2029                                            ref_event, ref_var);
2030                 if (hist_field) {
2031                         if (var_name) {
2032                                 hist_field = create_alias(hist_data, hist_field, var_name);
2033                                 if (!hist_field) {
2034                                         ret = -ENOMEM;
2035                                         goto out;
2036                                 }
2037                         }
2038                         return hist_field;
2039                 }
2040         } else
2041                 str = s;
2042
2043         field = parse_field(hist_data, file, str, flags);
2044         if (IS_ERR(field)) {
2045                 ret = PTR_ERR(field);
2046                 goto out;
2047         }
2048
2049         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2050         if (!hist_field) {
2051                 ret = -ENOMEM;
2052                 goto out;
2053         }
2054
2055         return hist_field;
2056  out:
2057         return ERR_PTR(ret);
2058 }
2059
2060 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2061                                      struct trace_event_file *file,
2062                                      char *str, unsigned long flags,
2063                                      char *var_name, unsigned int level);
2064
2065 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2066                                       struct trace_event_file *file,
2067                                       char *str, unsigned long flags,
2068                                       char *var_name, unsigned int level)
2069 {
2070         struct hist_field *operand1, *expr = NULL;
2071         unsigned long operand_flags;
2072         int ret = 0;
2073         char *s;
2074
2075         /* we support only -(xxx) i.e. explicit parens required */
2076
2077         if (level > 3) {
2078                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2079                 ret = -EINVAL;
2080                 goto free;
2081         }
2082
2083         str++; /* skip leading '-' */
2084
2085         s = strchr(str, '(');
2086         if (s)
2087                 str++;
2088         else {
2089                 ret = -EINVAL;
2090                 goto free;
2091         }
2092
2093         s = strrchr(str, ')');
2094         if (s)
2095                 *s = '\0';
2096         else {
2097                 ret = -EINVAL; /* no closing ')' */
2098                 goto free;
2099         }
2100
2101         flags |= HIST_FIELD_FL_EXPR;
2102         expr = create_hist_field(hist_data, NULL, flags, var_name);
2103         if (!expr) {
2104                 ret = -ENOMEM;
2105                 goto free;
2106         }
2107
2108         operand_flags = 0;
2109         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2110         if (IS_ERR(operand1)) {
2111                 ret = PTR_ERR(operand1);
2112                 goto free;
2113         }
2114
2115         expr->flags |= operand1->flags &
2116                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2117         expr->fn = hist_field_unary_minus;
2118         expr->operands[0] = operand1;
2119         expr->operator = FIELD_OP_UNARY_MINUS;
2120         expr->name = expr_str(expr, 0);
2121         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2122         if (!expr->type) {
2123                 ret = -ENOMEM;
2124                 goto free;
2125         }
2126
2127         return expr;
2128  free:
2129         destroy_hist_field(expr, 0);
2130         return ERR_PTR(ret);
2131 }
2132
2133 static int check_expr_operands(struct trace_array *tr,
2134                                struct hist_field *operand1,
2135                                struct hist_field *operand2)
2136 {
2137         unsigned long operand1_flags = operand1->flags;
2138         unsigned long operand2_flags = operand2->flags;
2139
2140         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2141             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2142                 struct hist_field *var;
2143
2144                 var = find_var_field(operand1->var.hist_data, operand1->name);
2145                 if (!var)
2146                         return -EINVAL;
2147                 operand1_flags = var->flags;
2148         }
2149
2150         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2151             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2152                 struct hist_field *var;
2153
2154                 var = find_var_field(operand2->var.hist_data, operand2->name);
2155                 if (!var)
2156                         return -EINVAL;
2157                 operand2_flags = var->flags;
2158         }
2159
2160         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2161             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2162                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2163                 return -EINVAL;
2164         }
2165
2166         return 0;
2167 }
2168
2169 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2170                                      struct trace_event_file *file,
2171                                      char *str, unsigned long flags,
2172                                      char *var_name, unsigned int level)
2173 {
2174         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2175         unsigned long operand_flags;
2176         int field_op, ret = -EINVAL;
2177         char *sep, *operand1_str;
2178
2179         if (level > 3) {
2180                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2181                 return ERR_PTR(-EINVAL);
2182         }
2183
2184         field_op = contains_operator(str);
2185
2186         if (field_op == FIELD_OP_NONE)
2187                 return parse_atom(hist_data, file, str, &flags, var_name);
2188
2189         if (field_op == FIELD_OP_UNARY_MINUS)
2190                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2191
2192         switch (field_op) {
2193         case FIELD_OP_MINUS:
2194                 sep = "-";
2195                 break;
2196         case FIELD_OP_PLUS:
2197                 sep = "+";
2198                 break;
2199         default:
2200                 goto free;
2201         }
2202
2203         operand1_str = strsep(&str, sep);
2204         if (!operand1_str || !str)
2205                 goto free;
2206
2207         operand_flags = 0;
2208         operand1 = parse_atom(hist_data, file, operand1_str,
2209                               &operand_flags, NULL);
2210         if (IS_ERR(operand1)) {
2211                 ret = PTR_ERR(operand1);
2212                 operand1 = NULL;
2213                 goto free;
2214         }
2215
2216         /* rest of string could be another expression e.g. b+c in a+b+c */
2217         operand_flags = 0;
2218         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2219         if (IS_ERR(operand2)) {
2220                 ret = PTR_ERR(operand2);
2221                 operand2 = NULL;
2222                 goto free;
2223         }
2224
2225         ret = check_expr_operands(file->tr, operand1, operand2);
2226         if (ret)
2227                 goto free;
2228
2229         flags |= HIST_FIELD_FL_EXPR;
2230
2231         flags |= operand1->flags &
2232                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2233
2234         expr = create_hist_field(hist_data, NULL, flags, var_name);
2235         if (!expr) {
2236                 ret = -ENOMEM;
2237                 goto free;
2238         }
2239
2240         operand1->read_once = true;
2241         operand2->read_once = true;
2242
2243         expr->operands[0] = operand1;
2244         expr->operands[1] = operand2;
2245         expr->operator = field_op;
2246         expr->name = expr_str(expr, 0);
2247         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2248         if (!expr->type) {
2249                 ret = -ENOMEM;
2250                 goto free;
2251         }
2252
2253         switch (field_op) {
2254         case FIELD_OP_MINUS:
2255                 expr->fn = hist_field_minus;
2256                 break;
2257         case FIELD_OP_PLUS:
2258                 expr->fn = hist_field_plus;
2259                 break;
2260         default:
2261                 ret = -EINVAL;
2262                 goto free;
2263         }
2264
2265         return expr;
2266  free:
2267         destroy_hist_field(operand1, 0);
2268         destroy_hist_field(operand2, 0);
2269         destroy_hist_field(expr, 0);
2270
2271         return ERR_PTR(ret);
2272 }
2273
2274 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2275                                  struct trace_event_file *file)
2276 {
2277         struct event_trigger_data *test;
2278
2279         lockdep_assert_held(&event_mutex);
2280
2281         list_for_each_entry(test, &file->triggers, list) {
2282                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2283                         if (test->private_data == hist_data)
2284                                 return test->filter_str;
2285                 }
2286         }
2287
2288         return NULL;
2289 }
2290
2291 static struct event_command trigger_hist_cmd;
2292 static int event_hist_trigger_func(struct event_command *cmd_ops,
2293                                    struct trace_event_file *file,
2294                                    char *glob, char *cmd, char *param);
2295
2296 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2297                             struct hist_trigger_data *hist_data,
2298                             unsigned int n_keys)
2299 {
2300         struct hist_field *target_hist_field, *hist_field;
2301         unsigned int n, i, j;
2302
2303         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2304                 return false;
2305
2306         i = hist_data->n_vals;
2307         j = target_hist_data->n_vals;
2308
2309         for (n = 0; n < n_keys; n++) {
2310                 hist_field = hist_data->fields[i + n];
2311                 target_hist_field = target_hist_data->fields[j + n];
2312
2313                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2314                         return false;
2315                 if (hist_field->size != target_hist_field->size)
2316                         return false;
2317                 if (hist_field->is_signed != target_hist_field->is_signed)
2318                         return false;
2319         }
2320
2321         return true;
2322 }
2323
2324 static struct hist_trigger_data *
2325 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2326                      struct trace_event_file *file)
2327 {
2328         struct hist_trigger_data *hist_data;
2329         struct event_trigger_data *test;
2330         unsigned int n_keys;
2331
2332         lockdep_assert_held(&event_mutex);
2333
2334         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2335
2336         list_for_each_entry(test, &file->triggers, list) {
2337                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2338                         hist_data = test->private_data;
2339
2340                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2341                                 return hist_data;
2342                 }
2343         }
2344
2345         return NULL;
2346 }
2347
2348 static struct trace_event_file *event_file(struct trace_array *tr,
2349                                            char *system, char *event_name)
2350 {
2351         struct trace_event_file *file;
2352
2353         file = __find_event_file(tr, system, event_name);
2354         if (!file)
2355                 return ERR_PTR(-EINVAL);
2356
2357         return file;
2358 }
2359
2360 static struct hist_field *
2361 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2362                          char *system, char *event_name, char *field_name)
2363 {
2364         struct hist_field *event_var;
2365         char *synthetic_name;
2366
2367         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2368         if (!synthetic_name)
2369                 return ERR_PTR(-ENOMEM);
2370
2371         strcpy(synthetic_name, "synthetic_");
2372         strcat(synthetic_name, field_name);
2373
2374         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2375
2376         kfree(synthetic_name);
2377
2378         return event_var;
2379 }
2380
2381 /**
2382  * create_field_var_hist - Automatically create a histogram and var for a field
2383  * @target_hist_data: The target hist trigger
2384  * @subsys_name: Optional subsystem name
2385  * @event_name: Optional event name
2386  * @field_name: The name of the field (and the resulting variable)
2387  *
2388  * Hist trigger actions fetch data from variables, not directly from
2389  * events.  However, for convenience, users are allowed to directly
2390  * specify an event field in an action, which will be automatically
2391  * converted into a variable on their behalf.
2392
2393  * If a user specifies a field on an event that isn't the event the
2394  * histogram currently being defined (the target event histogram), the
2395  * only way that can be accomplished is if a new hist trigger is
2396  * created and the field variable defined on that.
2397  *
2398  * This function creates a new histogram compatible with the target
2399  * event (meaning a histogram with the same key as the target
2400  * histogram), and creates a variable for the specified field, but
2401  * with 'synthetic_' prepended to the variable name in order to avoid
2402  * collision with normal field variables.
2403  *
2404  * Return: The variable created for the field.
2405  */
2406 static struct hist_field *
2407 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2408                       char *subsys_name, char *event_name, char *field_name)
2409 {
2410         struct trace_array *tr = target_hist_data->event_file->tr;
2411         struct hist_field *event_var = ERR_PTR(-EINVAL);
2412         struct hist_trigger_data *hist_data;
2413         unsigned int i, n, first = true;
2414         struct field_var_hist *var_hist;
2415         struct trace_event_file *file;
2416         struct hist_field *key_field;
2417         char *saved_filter;
2418         char *cmd;
2419         int ret;
2420
2421         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2422                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2423                 return ERR_PTR(-EINVAL);
2424         }
2425
2426         file = event_file(tr, subsys_name, event_name);
2427
2428         if (IS_ERR(file)) {
2429                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2430                 ret = PTR_ERR(file);
2431                 return ERR_PTR(ret);
2432         }
2433
2434         /*
2435          * Look for a histogram compatible with target.  We'll use the
2436          * found histogram specification to create a new matching
2437          * histogram with our variable on it.  target_hist_data is not
2438          * yet a registered histogram so we can't use that.
2439          */
2440         hist_data = find_compatible_hist(target_hist_data, file);
2441         if (!hist_data) {
2442                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2443                 return ERR_PTR(-EINVAL);
2444         }
2445
2446         /* See if a synthetic field variable has already been created */
2447         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2448                                              event_name, field_name);
2449         if (!IS_ERR_OR_NULL(event_var))
2450                 return event_var;
2451
2452         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2453         if (!var_hist)
2454                 return ERR_PTR(-ENOMEM);
2455
2456         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2457         if (!cmd) {
2458                 kfree(var_hist);
2459                 return ERR_PTR(-ENOMEM);
2460         }
2461
2462         /* Use the same keys as the compatible histogram */
2463         strcat(cmd, "keys=");
2464
2465         for_each_hist_key_field(i, hist_data) {
2466                 key_field = hist_data->fields[i];
2467                 if (!first)
2468                         strcat(cmd, ",");
2469                 strcat(cmd, key_field->field->name);
2470                 first = false;
2471         }
2472
2473         /* Create the synthetic field variable specification */
2474         strcat(cmd, ":synthetic_");
2475         strcat(cmd, field_name);
2476         strcat(cmd, "=");
2477         strcat(cmd, field_name);
2478
2479         /* Use the same filter as the compatible histogram */
2480         saved_filter = find_trigger_filter(hist_data, file);
2481         if (saved_filter) {
2482                 strcat(cmd, " if ");
2483                 strcat(cmd, saved_filter);
2484         }
2485
2486         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2487         if (!var_hist->cmd) {
2488                 kfree(cmd);
2489                 kfree(var_hist);
2490                 return ERR_PTR(-ENOMEM);
2491         }
2492
2493         /* Save the compatible histogram information */
2494         var_hist->hist_data = hist_data;
2495
2496         /* Create the new histogram with our variable */
2497         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2498                                       "", "hist", cmd);
2499         if (ret) {
2500                 kfree(cmd);
2501                 kfree(var_hist->cmd);
2502                 kfree(var_hist);
2503                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2504                 return ERR_PTR(ret);
2505         }
2506
2507         kfree(cmd);
2508
2509         /* If we can't find the variable, something went wrong */
2510         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2511                                              event_name, field_name);
2512         if (IS_ERR_OR_NULL(event_var)) {
2513                 kfree(var_hist->cmd);
2514                 kfree(var_hist);
2515                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2516                 return ERR_PTR(-EINVAL);
2517         }
2518
2519         n = target_hist_data->n_field_var_hists;
2520         target_hist_data->field_var_hists[n] = var_hist;
2521         target_hist_data->n_field_var_hists++;
2522
2523         return event_var;
2524 }
2525
2526 static struct hist_field *
2527 find_target_event_var(struct hist_trigger_data *hist_data,
2528                       char *subsys_name, char *event_name, char *var_name)
2529 {
2530         struct trace_event_file *file = hist_data->event_file;
2531         struct hist_field *hist_field = NULL;
2532
2533         if (subsys_name) {
2534                 struct trace_event_call *call;
2535
2536                 if (!event_name)
2537                         return NULL;
2538
2539                 call = file->event_call;
2540
2541                 if (strcmp(subsys_name, call->class->system) != 0)
2542                         return NULL;
2543
2544                 if (strcmp(event_name, trace_event_name(call)) != 0)
2545                         return NULL;
2546         }
2547
2548         hist_field = find_var_field(hist_data, var_name);
2549
2550         return hist_field;
2551 }
2552
2553 static inline void __update_field_vars(struct tracing_map_elt *elt,
2554                                        struct ring_buffer_event *rbe,
2555                                        void *rec,
2556                                        struct field_var **field_vars,
2557                                        unsigned int n_field_vars,
2558                                        unsigned int field_var_str_start)
2559 {
2560         struct hist_elt_data *elt_data = elt->private_data;
2561         unsigned int i, j, var_idx;
2562         u64 var_val;
2563
2564         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2565                 struct field_var *field_var = field_vars[i];
2566                 struct hist_field *var = field_var->var;
2567                 struct hist_field *val = field_var->val;
2568
2569                 var_val = val->fn(val, elt, rbe, rec);
2570                 var_idx = var->var.idx;
2571
2572                 if (val->flags & HIST_FIELD_FL_STRING) {
2573                         char *str = elt_data->field_var_str[j++];
2574                         char *val_str = (char *)(uintptr_t)var_val;
2575
2576                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2577                         var_val = (u64)(uintptr_t)str;
2578                 }
2579                 tracing_map_set_var(elt, var_idx, var_val);
2580         }
2581 }
2582
2583 static void update_field_vars(struct hist_trigger_data *hist_data,
2584                               struct tracing_map_elt *elt,
2585                               struct ring_buffer_event *rbe,
2586                               void *rec)
2587 {
2588         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
2589                             hist_data->n_field_vars, 0);
2590 }
2591
2592 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2593                                  struct tracing_map_elt *elt, void *rec,
2594                                  struct ring_buffer_event *rbe, void *key,
2595                                  struct action_data *data, u64 *var_ref_vals)
2596 {
2597         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
2598                             hist_data->n_save_vars, hist_data->n_field_var_str);
2599 }
2600
2601 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2602                                      struct trace_event_file *file,
2603                                      char *name, int size, const char *type)
2604 {
2605         struct hist_field *var;
2606         int idx;
2607
2608         if (find_var(hist_data, file, name) && !hist_data->remove) {
2609                 var = ERR_PTR(-EINVAL);
2610                 goto out;
2611         }
2612
2613         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2614         if (!var) {
2615                 var = ERR_PTR(-ENOMEM);
2616                 goto out;
2617         }
2618
2619         idx = tracing_map_add_var(hist_data->map);
2620         if (idx < 0) {
2621                 kfree(var);
2622                 var = ERR_PTR(-EINVAL);
2623                 goto out;
2624         }
2625
2626         var->ref = 1;
2627         var->flags = HIST_FIELD_FL_VAR;
2628         var->var.idx = idx;
2629         var->var.hist_data = var->hist_data = hist_data;
2630         var->size = size;
2631         var->var.name = kstrdup(name, GFP_KERNEL);
2632         var->type = kstrdup(type, GFP_KERNEL);
2633         if (!var->var.name || !var->type) {
2634                 kfree(var->var.name);
2635                 kfree(var->type);
2636                 kfree(var);
2637                 var = ERR_PTR(-ENOMEM);
2638         }
2639  out:
2640         return var;
2641 }
2642
2643 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2644                                           struct trace_event_file *file,
2645                                           char *field_name)
2646 {
2647         struct hist_field *val = NULL, *var = NULL;
2648         unsigned long flags = HIST_FIELD_FL_VAR;
2649         struct trace_array *tr = file->tr;
2650         struct field_var *field_var;
2651         int ret = 0;
2652
2653         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2654                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2655                 ret = -EINVAL;
2656                 goto err;
2657         }
2658
2659         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2660         if (IS_ERR(val)) {
2661                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2662                 ret = PTR_ERR(val);
2663                 goto err;
2664         }
2665
2666         var = create_var(hist_data, file, field_name, val->size, val->type);
2667         if (IS_ERR(var)) {
2668                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2669                 kfree(val);
2670                 ret = PTR_ERR(var);
2671                 goto err;
2672         }
2673
2674         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2675         if (!field_var) {
2676                 kfree(val);
2677                 kfree(var);
2678                 ret =  -ENOMEM;
2679                 goto err;
2680         }
2681
2682         field_var->var = var;
2683         field_var->val = val;
2684  out:
2685         return field_var;
2686  err:
2687         field_var = ERR_PTR(ret);
2688         goto out;
2689 }
2690
2691 /**
2692  * create_target_field_var - Automatically create a variable for a field
2693  * @target_hist_data: The target hist trigger
2694  * @subsys_name: Optional subsystem name
2695  * @event_name: Optional event name
2696  * @var_name: The name of the field (and the resulting variable)
2697  *
2698  * Hist trigger actions fetch data from variables, not directly from
2699  * events.  However, for convenience, users are allowed to directly
2700  * specify an event field in an action, which will be automatically
2701  * converted into a variable on their behalf.
2702
2703  * This function creates a field variable with the name var_name on
2704  * the hist trigger currently being defined on the target event.  If
2705  * subsys_name and event_name are specified, this function simply
2706  * verifies that they do in fact match the target event subsystem and
2707  * event name.
2708  *
2709  * Return: The variable created for the field.
2710  */
2711 static struct field_var *
2712 create_target_field_var(struct hist_trigger_data *target_hist_data,
2713                         char *subsys_name, char *event_name, char *var_name)
2714 {
2715         struct trace_event_file *file = target_hist_data->event_file;
2716
2717         if (subsys_name) {
2718                 struct trace_event_call *call;
2719
2720                 if (!event_name)
2721                         return NULL;
2722
2723                 call = file->event_call;
2724
2725                 if (strcmp(subsys_name, call->class->system) != 0)
2726                         return NULL;
2727
2728                 if (strcmp(event_name, trace_event_name(call)) != 0)
2729                         return NULL;
2730         }
2731
2732         return create_field_var(target_hist_data, file, var_name);
2733 }
2734
2735 static bool check_track_val_max(u64 track_val, u64 var_val)
2736 {
2737         if (var_val <= track_val)
2738                 return false;
2739
2740         return true;
2741 }
2742
2743 static bool check_track_val_changed(u64 track_val, u64 var_val)
2744 {
2745         if (var_val == track_val)
2746                 return false;
2747
2748         return true;
2749 }
2750
2751 static u64 get_track_val(struct hist_trigger_data *hist_data,
2752                          struct tracing_map_elt *elt,
2753                          struct action_data *data)
2754 {
2755         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2756         u64 track_val;
2757
2758         track_val = tracing_map_read_var(elt, track_var_idx);
2759
2760         return track_val;
2761 }
2762
2763 static void save_track_val(struct hist_trigger_data *hist_data,
2764                            struct tracing_map_elt *elt,
2765                            struct action_data *data, u64 var_val)
2766 {
2767         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2768
2769         tracing_map_set_var(elt, track_var_idx, var_val);
2770 }
2771
2772 static void save_track_data(struct hist_trigger_data *hist_data,
2773                             struct tracing_map_elt *elt, void *rec,
2774                             struct ring_buffer_event *rbe, void *key,
2775                             struct action_data *data, u64 *var_ref_vals)
2776 {
2777         if (data->track_data.save_data)
2778                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2779 }
2780
2781 static bool check_track_val(struct tracing_map_elt *elt,
2782                             struct action_data *data,
2783                             u64 var_val)
2784 {
2785         struct hist_trigger_data *hist_data;
2786         u64 track_val;
2787
2788         hist_data = data->track_data.track_var->hist_data;
2789         track_val = get_track_val(hist_data, elt, data);
2790
2791         return data->track_data.check_val(track_val, var_val);
2792 }
2793
2794 #ifdef CONFIG_TRACER_SNAPSHOT
2795 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2796 {
2797         /* called with tr->max_lock held */
2798         struct track_data *track_data = tr->cond_snapshot->cond_data;
2799         struct hist_elt_data *elt_data, *track_elt_data;
2800         struct snapshot_context *context = cond_data;
2801         struct action_data *action;
2802         u64 track_val;
2803
2804         if (!track_data)
2805                 return false;
2806
2807         action = track_data->action_data;
2808
2809         track_val = get_track_val(track_data->hist_data, context->elt,
2810                                   track_data->action_data);
2811
2812         if (!action->track_data.check_val(track_data->track_val, track_val))
2813                 return false;
2814
2815         track_data->track_val = track_val;
2816         memcpy(track_data->key, context->key, track_data->key_len);
2817
2818         elt_data = context->elt->private_data;
2819         track_elt_data = track_data->elt.private_data;
2820         if (elt_data->comm)
2821                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2822
2823         track_data->updated = true;
2824
2825         return true;
2826 }
2827
2828 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2829                                      struct tracing_map_elt *elt, void *rec,
2830                                      struct ring_buffer_event *rbe, void *key,
2831                                      struct action_data *data,
2832                                      u64 *var_ref_vals)
2833 {
2834         struct trace_event_file *file = hist_data->event_file;
2835         struct snapshot_context context;
2836
2837         context.elt = elt;
2838         context.key = key;
2839
2840         tracing_snapshot_cond(file->tr, &context);
2841 }
2842
2843 static void hist_trigger_print_key(struct seq_file *m,
2844                                    struct hist_trigger_data *hist_data,
2845                                    void *key,
2846                                    struct tracing_map_elt *elt);
2847
2848 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2849 {
2850         unsigned int i;
2851
2852         if (!hist_data->n_actions)
2853                 return NULL;
2854
2855         for (i = 0; i < hist_data->n_actions; i++) {
2856                 struct action_data *data = hist_data->actions[i];
2857
2858                 if (data->action == ACTION_SNAPSHOT)
2859                         return data;
2860         }
2861
2862         return NULL;
2863 }
2864
2865 static void track_data_snapshot_print(struct seq_file *m,
2866                                       struct hist_trigger_data *hist_data)
2867 {
2868         struct trace_event_file *file = hist_data->event_file;
2869         struct track_data *track_data;
2870         struct action_data *action;
2871
2872         track_data = tracing_cond_snapshot_data(file->tr);
2873         if (!track_data)
2874                 return;
2875
2876         if (!track_data->updated)
2877                 return;
2878
2879         action = snapshot_action(hist_data);
2880         if (!action)
2881                 return;
2882
2883         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2884         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2885                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2886                    action->track_data.var_str, track_data->track_val);
2887
2888         seq_puts(m, "\ttriggered by event with key: ");
2889         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2890         seq_putc(m, '\n');
2891 }
2892 #else
2893 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2894 {
2895         return false;
2896 }
2897 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2898                                      struct tracing_map_elt *elt, void *rec,
2899                                      struct ring_buffer_event *rbe, void *key,
2900                                      struct action_data *data,
2901                                      u64 *var_ref_vals) {}
2902 static void track_data_snapshot_print(struct seq_file *m,
2903                                       struct hist_trigger_data *hist_data) {}
2904 #endif /* CONFIG_TRACER_SNAPSHOT */
2905
2906 static void track_data_print(struct seq_file *m,
2907                              struct hist_trigger_data *hist_data,
2908                              struct tracing_map_elt *elt,
2909                              struct action_data *data)
2910 {
2911         u64 track_val = get_track_val(hist_data, elt, data);
2912         unsigned int i, save_var_idx;
2913
2914         if (data->handler == HANDLER_ONMAX)
2915                 seq_printf(m, "\n\tmax: %10llu", track_val);
2916         else if (data->handler == HANDLER_ONCHANGE)
2917                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2918
2919         if (data->action == ACTION_SNAPSHOT)
2920                 return;
2921
2922         for (i = 0; i < hist_data->n_save_vars; i++) {
2923                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2924                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2925                 u64 val;
2926
2927                 save_var_idx = save_var->var.idx;
2928
2929                 val = tracing_map_read_var(elt, save_var_idx);
2930
2931                 if (save_val->flags & HIST_FIELD_FL_STRING) {
2932                         seq_printf(m, "  %s: %-32s", save_var->var.name,
2933                                    (char *)(uintptr_t)(val));
2934                 } else
2935                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2936         }
2937 }
2938
2939 static void ontrack_action(struct hist_trigger_data *hist_data,
2940                            struct tracing_map_elt *elt, void *rec,
2941                            struct ring_buffer_event *rbe, void *key,
2942                            struct action_data *data, u64 *var_ref_vals)
2943 {
2944         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2945
2946         if (check_track_val(elt, data, var_val)) {
2947                 save_track_val(hist_data, elt, data, var_val);
2948                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2949         }
2950 }
2951
2952 static void action_data_destroy(struct action_data *data)
2953 {
2954         unsigned int i;
2955
2956         lockdep_assert_held(&event_mutex);
2957
2958         kfree(data->action_name);
2959
2960         for (i = 0; i < data->n_params; i++)
2961                 kfree(data->params[i]);
2962
2963         if (data->synth_event)
2964                 data->synth_event->ref--;
2965
2966         kfree(data->synth_event_name);
2967
2968         kfree(data);
2969 }
2970
2971 static void track_data_destroy(struct hist_trigger_data *hist_data,
2972                                struct action_data *data)
2973 {
2974         struct trace_event_file *file = hist_data->event_file;
2975
2976         destroy_hist_field(data->track_data.track_var, 0);
2977
2978         if (data->action == ACTION_SNAPSHOT) {
2979                 struct track_data *track_data;
2980
2981                 track_data = tracing_cond_snapshot_data(file->tr);
2982                 if (track_data && track_data->hist_data == hist_data) {
2983                         tracing_snapshot_cond_disable(file->tr);
2984                         track_data_free(track_data);
2985                 }
2986         }
2987
2988         kfree(data->track_data.var_str);
2989
2990         action_data_destroy(data);
2991 }
2992
2993 static int action_create(struct hist_trigger_data *hist_data,
2994                          struct action_data *data);
2995
2996 static int track_data_create(struct hist_trigger_data *hist_data,
2997                              struct action_data *data)
2998 {
2999         struct hist_field *var_field, *ref_field, *track_var = NULL;
3000         struct trace_event_file *file = hist_data->event_file;
3001         struct trace_array *tr = file->tr;
3002         char *track_data_var_str;
3003         int ret = 0;
3004
3005         track_data_var_str = data->track_data.var_str;
3006         if (track_data_var_str[0] != '$') {
3007                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3008                 return -EINVAL;
3009         }
3010         track_data_var_str++;
3011
3012         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3013         if (!var_field) {
3014                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3015                 return -EINVAL;
3016         }
3017
3018         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3019         if (!ref_field)
3020                 return -ENOMEM;
3021
3022         data->track_data.var_ref = ref_field;
3023
3024         if (data->handler == HANDLER_ONMAX)
3025                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3026         if (IS_ERR(track_var)) {
3027                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3028                 ret = PTR_ERR(track_var);
3029                 goto out;
3030         }
3031
3032         if (data->handler == HANDLER_ONCHANGE)
3033                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3034         if (IS_ERR(track_var)) {
3035                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3036                 ret = PTR_ERR(track_var);
3037                 goto out;
3038         }
3039         data->track_data.track_var = track_var;
3040
3041         ret = action_create(hist_data, data);
3042  out:
3043         return ret;
3044 }
3045
3046 static int parse_action_params(struct trace_array *tr, char *params,
3047                                struct action_data *data)
3048 {
3049         char *param, *saved_param;
3050         bool first_param = true;
3051         int ret = 0;
3052
3053         while (params) {
3054                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3055                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3056                         goto out;
3057                 }
3058
3059                 param = strsep(&params, ",");
3060                 if (!param) {
3061                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3062                         ret = -EINVAL;
3063                         goto out;
3064                 }
3065
3066                 param = strstrip(param);
3067                 if (strlen(param) < 2) {
3068                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3069                         ret = -EINVAL;
3070                         goto out;
3071                 }
3072
3073                 saved_param = kstrdup(param, GFP_KERNEL);
3074                 if (!saved_param) {
3075                         ret = -ENOMEM;
3076                         goto out;
3077                 }
3078
3079                 if (first_param && data->use_trace_keyword) {
3080                         data->synth_event_name = saved_param;
3081                         first_param = false;
3082                         continue;
3083                 }
3084                 first_param = false;
3085
3086                 data->params[data->n_params++] = saved_param;
3087         }
3088  out:
3089         return ret;
3090 }
3091
3092 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3093                         enum handler_id handler)
3094 {
3095         char *action_name;
3096         int ret = 0;
3097
3098         strsep(&str, ".");
3099         if (!str) {
3100                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3101                 ret = -EINVAL;
3102                 goto out;
3103         }
3104
3105         action_name = strsep(&str, "(");
3106         if (!action_name || !str) {
3107                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3108                 ret = -EINVAL;
3109                 goto out;
3110         }
3111
3112         if (str_has_prefix(action_name, "save")) {
3113                 char *params = strsep(&str, ")");
3114
3115                 if (!params) {
3116                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3117                         ret = -EINVAL;
3118                         goto out;
3119                 }
3120
3121                 ret = parse_action_params(tr, params, data);
3122                 if (ret)
3123                         goto out;
3124
3125                 if (handler == HANDLER_ONMAX)
3126                         data->track_data.check_val = check_track_val_max;
3127                 else if (handler == HANDLER_ONCHANGE)
3128                         data->track_data.check_val = check_track_val_changed;
3129                 else {
3130                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3131                         ret = -EINVAL;
3132                         goto out;
3133                 }
3134
3135                 data->track_data.save_data = save_track_data_vars;
3136                 data->fn = ontrack_action;
3137                 data->action = ACTION_SAVE;
3138         } else if (str_has_prefix(action_name, "snapshot")) {
3139                 char *params = strsep(&str, ")");
3140
3141                 if (!str) {
3142                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3143                         ret = -EINVAL;
3144                         goto out;
3145                 }
3146
3147                 if (handler == HANDLER_ONMAX)
3148                         data->track_data.check_val = check_track_val_max;
3149                 else if (handler == HANDLER_ONCHANGE)
3150                         data->track_data.check_val = check_track_val_changed;
3151                 else {
3152                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3153                         ret = -EINVAL;
3154                         goto out;
3155                 }
3156
3157                 data->track_data.save_data = save_track_data_snapshot;
3158                 data->fn = ontrack_action;
3159                 data->action = ACTION_SNAPSHOT;
3160         } else {
3161                 char *params = strsep(&str, ")");
3162
3163                 if (str_has_prefix(action_name, "trace"))
3164                         data->use_trace_keyword = true;
3165
3166                 if (params) {
3167                         ret = parse_action_params(tr, params, data);
3168                         if (ret)
3169                                 goto out;
3170                 }
3171
3172                 if (handler == HANDLER_ONMAX)
3173                         data->track_data.check_val = check_track_val_max;
3174                 else if (handler == HANDLER_ONCHANGE)
3175                         data->track_data.check_val = check_track_val_changed;
3176
3177                 if (handler != HANDLER_ONMATCH) {
3178                         data->track_data.save_data = action_trace;
3179                         data->fn = ontrack_action;
3180                 } else
3181                         data->fn = action_trace;
3182
3183                 data->action = ACTION_TRACE;
3184         }
3185
3186         data->action_name = kstrdup(action_name, GFP_KERNEL);
3187         if (!data->action_name) {
3188                 ret = -ENOMEM;
3189                 goto out;
3190         }
3191
3192         data->handler = handler;
3193  out:
3194         return ret;
3195 }
3196
3197 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3198                                             char *str, enum handler_id handler)
3199 {
3200         struct action_data *data;
3201         int ret = -EINVAL;
3202         char *var_str;
3203
3204         data = kzalloc(sizeof(*data), GFP_KERNEL);
3205         if (!data)
3206                 return ERR_PTR(-ENOMEM);
3207
3208         var_str = strsep(&str, ")");
3209         if (!var_str || !str) {
3210                 ret = -EINVAL;
3211                 goto free;
3212         }
3213
3214         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3215         if (!data->track_data.var_str) {
3216                 ret = -ENOMEM;
3217                 goto free;
3218         }
3219
3220         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3221         if (ret)
3222                 goto free;
3223  out:
3224         return data;
3225  free:
3226         track_data_destroy(hist_data, data);
3227         data = ERR_PTR(ret);
3228         goto out;
3229 }
3230
3231 static void onmatch_destroy(struct action_data *data)
3232 {
3233         kfree(data->match_data.event);
3234         kfree(data->match_data.event_system);
3235
3236         action_data_destroy(data);
3237 }
3238
3239 static void destroy_field_var(struct field_var *field_var)
3240 {
3241         if (!field_var)
3242                 return;
3243
3244         destroy_hist_field(field_var->var, 0);
3245         destroy_hist_field(field_var->val, 0);
3246
3247         kfree(field_var);
3248 }
3249
3250 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3251 {
3252         unsigned int i;
3253
3254         for (i = 0; i < hist_data->n_field_vars; i++)
3255                 destroy_field_var(hist_data->field_vars[i]);
3256
3257         for (i = 0; i < hist_data->n_save_vars; i++)
3258                 destroy_field_var(hist_data->save_vars[i]);
3259 }
3260
3261 static void save_field_var(struct hist_trigger_data *hist_data,
3262                            struct field_var *field_var)
3263 {
3264         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3265
3266         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3267                 hist_data->n_field_var_str++;
3268 }
3269
3270
3271 static int check_synth_field(struct synth_event *event,
3272                              struct hist_field *hist_field,
3273                              unsigned int field_pos)
3274 {
3275         struct synth_field *field;
3276
3277         if (field_pos >= event->n_fields)
3278                 return -EINVAL;
3279
3280         field = event->fields[field_pos];
3281
3282         if (strcmp(field->type, hist_field->type) != 0) {
3283                 if (field->size != hist_field->size ||
3284                     field->is_signed != hist_field->is_signed)
3285                         return -EINVAL;
3286         }
3287
3288         return 0;
3289 }
3290
3291 static struct hist_field *
3292 trace_action_find_var(struct hist_trigger_data *hist_data,
3293                       struct action_data *data,
3294                       char *system, char *event, char *var)
3295 {
3296         struct trace_array *tr = hist_data->event_file->tr;
3297         struct hist_field *hist_field;
3298
3299         var++; /* skip '$' */
3300
3301         hist_field = find_target_event_var(hist_data, system, event, var);
3302         if (!hist_field) {
3303                 if (!system && data->handler == HANDLER_ONMATCH) {
3304                         system = data->match_data.event_system;
3305                         event = data->match_data.event;
3306                 }
3307
3308                 hist_field = find_event_var(hist_data, system, event, var);
3309         }
3310
3311         if (!hist_field)
3312                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3313
3314         return hist_field;
3315 }
3316
3317 static struct hist_field *
3318 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3319                               struct action_data *data, char *system,
3320                               char *event, char *var)
3321 {
3322         struct hist_field *hist_field = NULL;
3323         struct field_var *field_var;
3324
3325         /*
3326          * First try to create a field var on the target event (the
3327          * currently being defined).  This will create a variable for
3328          * unqualified fields on the target event, or if qualified,
3329          * target fields that have qualified names matching the target.
3330          */
3331         field_var = create_target_field_var(hist_data, system, event, var);
3332
3333         if (field_var && !IS_ERR(field_var)) {
3334                 save_field_var(hist_data, field_var);
3335                 hist_field = field_var->var;
3336         } else {
3337                 field_var = NULL;
3338                 /*
3339                  * If no explicit system.event is specfied, default to
3340                  * looking for fields on the onmatch(system.event.xxx)
3341                  * event.
3342                  */
3343                 if (!system && data->handler == HANDLER_ONMATCH) {
3344                         system = data->match_data.event_system;
3345                         event = data->match_data.event;
3346                 }
3347
3348                 /*
3349                  * At this point, we're looking at a field on another
3350                  * event.  Because we can't modify a hist trigger on
3351                  * another event to add a variable for a field, we need
3352                  * to create a new trigger on that event and create the
3353                  * variable at the same time.
3354                  */
3355                 hist_field = create_field_var_hist(hist_data, system, event, var);
3356                 if (IS_ERR(hist_field))
3357                         goto free;
3358         }
3359  out:
3360         return hist_field;
3361  free:
3362         destroy_field_var(field_var);
3363         hist_field = NULL;
3364         goto out;
3365 }
3366
3367 static int trace_action_create(struct hist_trigger_data *hist_data,
3368                                struct action_data *data)
3369 {
3370         struct trace_array *tr = hist_data->event_file->tr;
3371         char *event_name, *param, *system = NULL;
3372         struct hist_field *hist_field, *var_ref;
3373         unsigned int i;
3374         unsigned int field_pos = 0;
3375         struct synth_event *event;
3376         char *synth_event_name;
3377         int var_ref_idx, ret = 0;
3378
3379         lockdep_assert_held(&event_mutex);
3380
3381         if (data->use_trace_keyword)
3382                 synth_event_name = data->synth_event_name;
3383         else
3384                 synth_event_name = data->action_name;
3385
3386         event = find_synth_event(synth_event_name);
3387         if (!event) {
3388                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3389                 return -EINVAL;
3390         }
3391
3392         event->ref++;
3393
3394         for (i = 0; i < data->n_params; i++) {
3395                 char *p;
3396
3397                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3398                 if (!param) {
3399                         ret = -ENOMEM;
3400                         goto err;
3401                 }
3402
3403                 system = strsep(&param, ".");
3404                 if (!param) {
3405                         param = (char *)system;
3406                         system = event_name = NULL;
3407                 } else {
3408                         event_name = strsep(&param, ".");
3409                         if (!param) {
3410                                 kfree(p);
3411                                 ret = -EINVAL;
3412                                 goto err;
3413                         }
3414                 }
3415
3416                 if (param[0] == '$')
3417                         hist_field = trace_action_find_var(hist_data, data,
3418                                                            system, event_name,
3419                                                            param);
3420                 else
3421                         hist_field = trace_action_create_field_var(hist_data,
3422                                                                    data,
3423                                                                    system,
3424                                                                    event_name,
3425                                                                    param);
3426
3427                 if (!hist_field) {
3428                         kfree(p);
3429                         ret = -EINVAL;
3430                         goto err;
3431                 }
3432
3433                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3434                         var_ref = create_var_ref(hist_data, hist_field,
3435                                                  system, event_name);
3436                         if (!var_ref) {
3437                                 kfree(p);
3438                                 ret = -ENOMEM;
3439                                 goto err;
3440                         }
3441
3442                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3443                         if (WARN_ON(var_ref_idx < 0)) {
3444                                 ret = var_ref_idx;
3445                                 goto err;
3446                         }
3447
3448                         data->var_ref_idx[i] = var_ref_idx;
3449
3450                         field_pos++;
3451                         kfree(p);
3452                         continue;
3453                 }
3454
3455                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3456                 kfree(p);
3457                 ret = -EINVAL;
3458                 goto err;
3459         }
3460
3461         if (field_pos != event->n_fields) {
3462                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3463                 ret = -EINVAL;
3464                 goto err;
3465         }
3466
3467         data->synth_event = event;
3468  out:
3469         return ret;
3470  err:
3471         event->ref--;
3472
3473         goto out;
3474 }
3475
3476 static int action_create(struct hist_trigger_data *hist_data,
3477                          struct action_data *data)
3478 {
3479         struct trace_event_file *file = hist_data->event_file;
3480         struct trace_array *tr = file->tr;
3481         struct track_data *track_data;
3482         struct field_var *field_var;
3483         unsigned int i;
3484         char *param;
3485         int ret = 0;
3486
3487         if (data->action == ACTION_TRACE)
3488                 return trace_action_create(hist_data, data);
3489
3490         if (data->action == ACTION_SNAPSHOT) {
3491                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3492                 if (IS_ERR(track_data)) {
3493                         ret = PTR_ERR(track_data);
3494                         goto out;
3495                 }
3496
3497                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3498                                                    cond_snapshot_update);
3499                 if (ret)
3500                         track_data_free(track_data);
3501
3502                 goto out;
3503         }
3504
3505         if (data->action == ACTION_SAVE) {
3506                 if (hist_data->n_save_vars) {
3507                         ret = -EEXIST;
3508                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3509                         goto out;
3510                 }
3511
3512                 for (i = 0; i < data->n_params; i++) {
3513                         param = kstrdup(data->params[i], GFP_KERNEL);
3514                         if (!param) {
3515                                 ret = -ENOMEM;
3516                                 goto out;
3517                         }
3518
3519                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3520                         if (IS_ERR(field_var)) {
3521                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3522                                          errpos(param));
3523                                 ret = PTR_ERR(field_var);
3524                                 kfree(param);
3525                                 goto out;
3526                         }
3527
3528                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3529                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3530                                 hist_data->n_save_var_str++;
3531                         kfree(param);
3532                 }
3533         }
3534  out:
3535         return ret;
3536 }
3537
3538 static int onmatch_create(struct hist_trigger_data *hist_data,
3539                           struct action_data *data)
3540 {
3541         return action_create(hist_data, data);
3542 }
3543
3544 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3545 {
3546         char *match_event, *match_event_system;
3547         struct action_data *data;
3548         int ret = -EINVAL;
3549
3550         data = kzalloc(sizeof(*data), GFP_KERNEL);
3551         if (!data)
3552                 return ERR_PTR(-ENOMEM);
3553
3554         match_event = strsep(&str, ")");
3555         if (!match_event || !str) {
3556                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3557                 goto free;
3558         }
3559
3560         match_event_system = strsep(&match_event, ".");
3561         if (!match_event) {
3562                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3563                 goto free;
3564         }
3565
3566         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3567                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3568                 goto free;
3569         }
3570
3571         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3572         if (!data->match_data.event) {
3573                 ret = -ENOMEM;
3574                 goto free;
3575         }
3576
3577         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3578         if (!data->match_data.event_system) {
3579                 ret = -ENOMEM;
3580                 goto free;
3581         }
3582
3583         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3584         if (ret)
3585                 goto free;
3586  out:
3587         return data;
3588  free:
3589         onmatch_destroy(data);
3590         data = ERR_PTR(ret);
3591         goto out;
3592 }
3593
3594 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3595 {
3596         hist_data->fields[HITCOUNT_IDX] =
3597                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3598         if (!hist_data->fields[HITCOUNT_IDX])
3599                 return -ENOMEM;
3600
3601         hist_data->n_vals++;
3602         hist_data->n_fields++;
3603
3604         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3605                 return -EINVAL;
3606
3607         return 0;
3608 }
3609
3610 static int __create_val_field(struct hist_trigger_data *hist_data,
3611                               unsigned int val_idx,
3612                               struct trace_event_file *file,
3613                               char *var_name, char *field_str,
3614                               unsigned long flags)
3615 {
3616         struct hist_field *hist_field;
3617         int ret = 0;
3618
3619         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3620         if (IS_ERR(hist_field)) {
3621                 ret = PTR_ERR(hist_field);
3622                 goto out;
3623         }
3624
3625         hist_data->fields[val_idx] = hist_field;
3626
3627         ++hist_data->n_vals;
3628         ++hist_data->n_fields;
3629
3630         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3631                 ret = -EINVAL;
3632  out:
3633         return ret;
3634 }
3635
3636 static int create_val_field(struct hist_trigger_data *hist_data,
3637                             unsigned int val_idx,
3638                             struct trace_event_file *file,
3639                             char *field_str)
3640 {
3641         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3642                 return -EINVAL;
3643
3644         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3645 }
3646
3647 static int create_var_field(struct hist_trigger_data *hist_data,
3648                             unsigned int val_idx,
3649                             struct trace_event_file *file,
3650                             char *var_name, char *expr_str)
3651 {
3652         struct trace_array *tr = hist_data->event_file->tr;
3653         unsigned long flags = 0;
3654
3655         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3656                 return -EINVAL;
3657
3658         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3659                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3660                 return -EINVAL;
3661         }
3662
3663         flags |= HIST_FIELD_FL_VAR;
3664         hist_data->n_vars++;
3665         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3666                 return -EINVAL;
3667
3668         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3669 }
3670
3671 static int create_val_fields(struct hist_trigger_data *hist_data,
3672                              struct trace_event_file *file)
3673 {
3674         char *fields_str, *field_str;
3675         unsigned int i, j = 1;
3676         int ret;
3677
3678         ret = create_hitcount_val(hist_data);
3679         if (ret)
3680                 goto out;
3681
3682         fields_str = hist_data->attrs->vals_str;
3683         if (!fields_str)
3684                 goto out;
3685
3686         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3687                      j < TRACING_MAP_VALS_MAX; i++) {
3688                 field_str = strsep(&fields_str, ",");
3689                 if (!field_str)
3690                         break;
3691
3692                 if (strcmp(field_str, "hitcount") == 0)
3693                         continue;
3694
3695                 ret = create_val_field(hist_data, j++, file, field_str);
3696                 if (ret)
3697                         goto out;
3698         }
3699
3700         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3701                 ret = -EINVAL;
3702  out:
3703         return ret;
3704 }
3705
3706 static int create_key_field(struct hist_trigger_data *hist_data,
3707                             unsigned int key_idx,
3708                             unsigned int key_offset,
3709                             struct trace_event_file *file,
3710                             char *field_str)
3711 {
3712         struct trace_array *tr = hist_data->event_file->tr;
3713         struct hist_field *hist_field = NULL;
3714         unsigned long flags = 0;
3715         unsigned int key_size;
3716         int ret = 0;
3717
3718         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3719                 return -EINVAL;
3720
3721         flags |= HIST_FIELD_FL_KEY;
3722
3723         if (strcmp(field_str, "stacktrace") == 0) {
3724                 flags |= HIST_FIELD_FL_STACKTRACE;
3725                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3726                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3727         } else {
3728                 hist_field = parse_expr(hist_data, file, field_str, flags,
3729                                         NULL, 0);
3730                 if (IS_ERR(hist_field)) {
3731                         ret = PTR_ERR(hist_field);
3732                         goto out;
3733                 }
3734
3735                 if (field_has_hist_vars(hist_field, 0)) {
3736                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3737                         destroy_hist_field(hist_field, 0);
3738                         ret = -EINVAL;
3739                         goto out;
3740                 }
3741
3742                 key_size = hist_field->size;
3743         }
3744
3745         hist_data->fields[key_idx] = hist_field;
3746
3747         key_size = ALIGN(key_size, sizeof(u64));
3748         hist_data->fields[key_idx]->size = key_size;
3749         hist_data->fields[key_idx]->offset = key_offset;
3750
3751         hist_data->key_size += key_size;
3752
3753         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3754                 ret = -EINVAL;
3755                 goto out;
3756         }
3757
3758         hist_data->n_keys++;
3759         hist_data->n_fields++;
3760
3761         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3762                 return -EINVAL;
3763
3764         ret = key_size;
3765  out:
3766         return ret;
3767 }
3768
3769 static int create_key_fields(struct hist_trigger_data *hist_data,
3770                              struct trace_event_file *file)
3771 {
3772         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3773         char *fields_str, *field_str;
3774         int ret = -EINVAL;
3775
3776         fields_str = hist_data->attrs->keys_str;
3777         if (!fields_str)
3778                 goto out;
3779
3780         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3781                 field_str = strsep(&fields_str, ",");
3782                 if (!field_str)
3783                         break;
3784                 ret = create_key_field(hist_data, i, key_offset,
3785                                        file, field_str);
3786                 if (ret < 0)
3787                         goto out;
3788                 key_offset += ret;
3789         }
3790         if (fields_str) {
3791                 ret = -EINVAL;
3792                 goto out;
3793         }
3794         ret = 0;
3795  out:
3796         return ret;
3797 }
3798
3799 static int create_var_fields(struct hist_trigger_data *hist_data,
3800                              struct trace_event_file *file)
3801 {
3802         unsigned int i, j = hist_data->n_vals;
3803         int ret = 0;
3804
3805         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3806
3807         for (i = 0; i < n_vars; i++) {
3808                 char *var_name = hist_data->attrs->var_defs.name[i];
3809                 char *expr = hist_data->attrs->var_defs.expr[i];
3810
3811                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3812                 if (ret)
3813                         goto out;
3814         }
3815  out:
3816         return ret;
3817 }
3818
3819 static void free_var_defs(struct hist_trigger_data *hist_data)
3820 {
3821         unsigned int i;
3822
3823         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3824                 kfree(hist_data->attrs->var_defs.name[i]);
3825                 kfree(hist_data->attrs->var_defs.expr[i]);
3826         }
3827
3828         hist_data->attrs->var_defs.n_vars = 0;
3829 }
3830
3831 static int parse_var_defs(struct hist_trigger_data *hist_data)
3832 {
3833         struct trace_array *tr = hist_data->event_file->tr;
3834         char *s, *str, *var_name, *field_str;
3835         unsigned int i, j, n_vars = 0;
3836         int ret = 0;
3837
3838         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3839                 str = hist_data->attrs->assignment_str[i];
3840                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3841                         field_str = strsep(&str, ",");
3842                         if (!field_str)
3843                                 break;
3844
3845                         var_name = strsep(&field_str, "=");
3846                         if (!var_name || !field_str) {
3847                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3848                                          errpos(var_name));
3849                                 ret = -EINVAL;
3850                                 goto free;
3851                         }
3852
3853                         if (n_vars == TRACING_MAP_VARS_MAX) {
3854                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3855                                 ret = -EINVAL;
3856                                 goto free;
3857                         }
3858
3859                         s = kstrdup(var_name, GFP_KERNEL);
3860                         if (!s) {
3861                                 ret = -ENOMEM;
3862                                 goto free;
3863                         }
3864                         hist_data->attrs->var_defs.name[n_vars] = s;
3865
3866                         s = kstrdup(field_str, GFP_KERNEL);
3867                         if (!s) {
3868                                 ret = -ENOMEM;
3869                                 goto free;
3870                         }
3871                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3872
3873                         hist_data->attrs->var_defs.n_vars = n_vars;
3874                 }
3875         }
3876
3877         return ret;
3878  free:
3879         free_var_defs(hist_data);
3880
3881         return ret;
3882 }
3883
3884 static int create_hist_fields(struct hist_trigger_data *hist_data,
3885                               struct trace_event_file *file)
3886 {
3887         int ret;
3888
3889         ret = parse_var_defs(hist_data);
3890         if (ret)
3891                 goto out;
3892
3893         ret = create_val_fields(hist_data, file);
3894         if (ret)
3895                 goto out;
3896
3897         ret = create_var_fields(hist_data, file);
3898         if (ret)
3899                 goto out;
3900
3901         ret = create_key_fields(hist_data, file);
3902         if (ret)
3903                 goto out;
3904  out:
3905         free_var_defs(hist_data);
3906
3907         return ret;
3908 }
3909
3910 static int is_descending(struct trace_array *tr, const char *str)
3911 {
3912         if (!str)
3913                 return 0;
3914
3915         if (strcmp(str, "descending") == 0)
3916                 return 1;
3917
3918         if (strcmp(str, "ascending") == 0)
3919                 return 0;
3920
3921         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3922
3923         return -EINVAL;
3924 }
3925
3926 static int create_sort_keys(struct hist_trigger_data *hist_data)
3927 {
3928         struct trace_array *tr = hist_data->event_file->tr;
3929         char *fields_str = hist_data->attrs->sort_key_str;
3930         struct tracing_map_sort_key *sort_key;
3931         int descending, ret = 0;
3932         unsigned int i, j, k;
3933
3934         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3935
3936         if (!fields_str)
3937                 goto out;
3938
3939         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3940                 struct hist_field *hist_field;
3941                 char *field_str, *field_name;
3942                 const char *test_name;
3943
3944                 sort_key = &hist_data->sort_keys[i];
3945
3946                 field_str = strsep(&fields_str, ",");
3947                 if (!field_str)
3948                         break;
3949
3950                 if (!*field_str) {
3951                         ret = -EINVAL;
3952                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3953                         break;
3954                 }
3955
3956                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
3957                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
3958                         ret = -EINVAL;
3959                         break;
3960                 }
3961
3962                 field_name = strsep(&field_str, ".");
3963                 if (!field_name || !*field_name) {
3964                         ret = -EINVAL;
3965                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3966                         break;
3967                 }
3968
3969                 if (strcmp(field_name, "hitcount") == 0) {
3970                         descending = is_descending(tr, field_str);
3971                         if (descending < 0) {
3972                                 ret = descending;
3973                                 break;
3974                         }
3975                         sort_key->descending = descending;
3976                         continue;
3977                 }
3978
3979                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
3980                         unsigned int idx;
3981
3982                         hist_field = hist_data->fields[j];
3983                         if (hist_field->flags & HIST_FIELD_FL_VAR)
3984                                 continue;
3985
3986                         idx = k++;
3987
3988                         test_name = hist_field_name(hist_field, 0);
3989
3990                         if (strcmp(field_name, test_name) == 0) {
3991                                 sort_key->field_idx = idx;
3992                                 descending = is_descending(tr, field_str);
3993                                 if (descending < 0) {
3994                                         ret = descending;
3995                                         goto out;
3996                                 }
3997                                 sort_key->descending = descending;
3998                                 break;
3999                         }
4000                 }
4001                 if (j == hist_data->n_fields) {
4002                         ret = -EINVAL;
4003                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4004                         break;
4005                 }
4006         }
4007
4008         hist_data->n_sort_keys = i;
4009  out:
4010         return ret;
4011 }
4012
4013 static void destroy_actions(struct hist_trigger_data *hist_data)
4014 {
4015         unsigned int i;
4016
4017         for (i = 0; i < hist_data->n_actions; i++) {
4018                 struct action_data *data = hist_data->actions[i];
4019
4020                 if (data->handler == HANDLER_ONMATCH)
4021                         onmatch_destroy(data);
4022                 else if (data->handler == HANDLER_ONMAX ||
4023                          data->handler == HANDLER_ONCHANGE)
4024                         track_data_destroy(hist_data, data);
4025                 else
4026                         kfree(data);
4027         }
4028 }
4029
4030 static int parse_actions(struct hist_trigger_data *hist_data)
4031 {
4032         struct trace_array *tr = hist_data->event_file->tr;
4033         struct action_data *data;
4034         unsigned int i;
4035         int ret = 0;
4036         char *str;
4037         int len;
4038
4039         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4040                 str = hist_data->attrs->action_str[i];
4041
4042                 if ((len = str_has_prefix(str, "onmatch("))) {
4043                         char *action_str = str + len;
4044
4045                         data = onmatch_parse(tr, action_str);
4046                         if (IS_ERR(data)) {
4047                                 ret = PTR_ERR(data);
4048                                 break;
4049                         }
4050                 } else if ((len = str_has_prefix(str, "onmax("))) {
4051                         char *action_str = str + len;
4052
4053                         data = track_data_parse(hist_data, action_str,
4054                                                 HANDLER_ONMAX);
4055                         if (IS_ERR(data)) {
4056                                 ret = PTR_ERR(data);
4057                                 break;
4058                         }
4059                 } else if ((len = str_has_prefix(str, "onchange("))) {
4060                         char *action_str = str + len;
4061
4062                         data = track_data_parse(hist_data, action_str,
4063                                                 HANDLER_ONCHANGE);
4064                         if (IS_ERR(data)) {
4065                                 ret = PTR_ERR(data);
4066                                 break;
4067                         }
4068                 } else {
4069                         ret = -EINVAL;
4070                         break;
4071                 }
4072
4073                 hist_data->actions[hist_data->n_actions++] = data;
4074         }
4075
4076         return ret;
4077 }
4078
4079 static int create_actions(struct hist_trigger_data *hist_data)
4080 {
4081         struct action_data *data;
4082         unsigned int i;
4083         int ret = 0;
4084
4085         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4086                 data = hist_data->actions[i];
4087
4088                 if (data->handler == HANDLER_ONMATCH) {
4089                         ret = onmatch_create(hist_data, data);
4090                         if (ret)
4091                                 break;
4092                 } else if (data->handler == HANDLER_ONMAX ||
4093                            data->handler == HANDLER_ONCHANGE) {
4094                         ret = track_data_create(hist_data, data);
4095                         if (ret)
4096                                 break;
4097                 } else {
4098                         ret = -EINVAL;
4099                         break;
4100                 }
4101         }
4102
4103         return ret;
4104 }
4105
4106 static void print_actions(struct seq_file *m,
4107                           struct hist_trigger_data *hist_data,
4108                           struct tracing_map_elt *elt)
4109 {
4110         unsigned int i;
4111
4112         for (i = 0; i < hist_data->n_actions; i++) {
4113                 struct action_data *data = hist_data->actions[i];
4114
4115                 if (data->action == ACTION_SNAPSHOT)
4116                         continue;
4117
4118                 if (data->handler == HANDLER_ONMAX ||
4119                     data->handler == HANDLER_ONCHANGE)
4120                         track_data_print(m, hist_data, elt, data);
4121         }
4122 }
4123
4124 static void print_action_spec(struct seq_file *m,
4125                               struct hist_trigger_data *hist_data,
4126                               struct action_data *data)
4127 {
4128         unsigned int i;
4129
4130         if (data->action == ACTION_SAVE) {
4131                 for (i = 0; i < hist_data->n_save_vars; i++) {
4132                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4133                         if (i < hist_data->n_save_vars - 1)
4134                                 seq_puts(m, ",");
4135                 }
4136         } else if (data->action == ACTION_TRACE) {
4137                 if (data->use_trace_keyword)
4138                         seq_printf(m, "%s", data->synth_event_name);
4139                 for (i = 0; i < data->n_params; i++) {
4140                         if (i || data->use_trace_keyword)
4141                                 seq_puts(m, ",");
4142                         seq_printf(m, "%s", data->params[i]);
4143                 }
4144         }
4145 }
4146
4147 static void print_track_data_spec(struct seq_file *m,
4148                                   struct hist_trigger_data *hist_data,
4149                                   struct action_data *data)
4150 {
4151         if (data->handler == HANDLER_ONMAX)
4152                 seq_puts(m, ":onmax(");
4153         else if (data->handler == HANDLER_ONCHANGE)
4154                 seq_puts(m, ":onchange(");
4155         seq_printf(m, "%s", data->track_data.var_str);
4156         seq_printf(m, ").%s(", data->action_name);
4157
4158         print_action_spec(m, hist_data, data);
4159
4160         seq_puts(m, ")");
4161 }
4162
4163 static void print_onmatch_spec(struct seq_file *m,
4164                                struct hist_trigger_data *hist_data,
4165                                struct action_data *data)
4166 {
4167         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4168                    data->match_data.event);
4169
4170         seq_printf(m, "%s(", data->action_name);
4171
4172         print_action_spec(m, hist_data, data);
4173
4174         seq_puts(m, ")");
4175 }
4176
4177 static bool actions_match(struct hist_trigger_data *hist_data,
4178                           struct hist_trigger_data *hist_data_test)
4179 {
4180         unsigned int i, j;
4181
4182         if (hist_data->n_actions != hist_data_test->n_actions)
4183                 return false;
4184
4185         for (i = 0; i < hist_data->n_actions; i++) {
4186                 struct action_data *data = hist_data->actions[i];
4187                 struct action_data *data_test = hist_data_test->actions[i];
4188                 char *action_name, *action_name_test;
4189
4190                 if (data->handler != data_test->handler)
4191                         return false;
4192                 if (data->action != data_test->action)
4193                         return false;
4194
4195                 if (data->n_params != data_test->n_params)
4196                         return false;
4197
4198                 for (j = 0; j < data->n_params; j++) {
4199                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4200                                 return false;
4201                 }
4202
4203                 if (data->use_trace_keyword)
4204                         action_name = data->synth_event_name;
4205                 else
4206                         action_name = data->action_name;
4207
4208                 if (data_test->use_trace_keyword)
4209                         action_name_test = data_test->synth_event_name;
4210                 else
4211                         action_name_test = data_test->action_name;
4212
4213                 if (strcmp(action_name, action_name_test) != 0)
4214                         return false;
4215
4216                 if (data->handler == HANDLER_ONMATCH) {
4217                         if (strcmp(data->match_data.event_system,
4218                                    data_test->match_data.event_system) != 0)
4219                                 return false;
4220                         if (strcmp(data->match_data.event,
4221                                    data_test->match_data.event) != 0)
4222                                 return false;
4223                 } else if (data->handler == HANDLER_ONMAX ||
4224                            data->handler == HANDLER_ONCHANGE) {
4225                         if (strcmp(data->track_data.var_str,
4226                                    data_test->track_data.var_str) != 0)
4227                                 return false;
4228                 }
4229         }
4230
4231         return true;
4232 }
4233
4234
4235 static void print_actions_spec(struct seq_file *m,
4236                                struct hist_trigger_data *hist_data)
4237 {
4238         unsigned int i;
4239
4240         for (i = 0; i < hist_data->n_actions; i++) {
4241                 struct action_data *data = hist_data->actions[i];
4242
4243                 if (data->handler == HANDLER_ONMATCH)
4244                         print_onmatch_spec(m, hist_data, data);
4245                 else if (data->handler == HANDLER_ONMAX ||
4246                          data->handler == HANDLER_ONCHANGE)
4247                         print_track_data_spec(m, hist_data, data);
4248         }
4249 }
4250
4251 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4252 {
4253         unsigned int i;
4254
4255         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4256                 kfree(hist_data->field_var_hists[i]->cmd);
4257                 kfree(hist_data->field_var_hists[i]);
4258         }
4259 }
4260
4261 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4262 {
4263         if (!hist_data)
4264                 return;
4265
4266         destroy_hist_trigger_attrs(hist_data->attrs);
4267         destroy_hist_fields(hist_data);
4268         tracing_map_destroy(hist_data->map);
4269
4270         destroy_actions(hist_data);
4271         destroy_field_vars(hist_data);
4272         destroy_field_var_hists(hist_data);
4273
4274         kfree(hist_data);
4275 }
4276
4277 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4278 {
4279         struct tracing_map *map = hist_data->map;
4280         struct ftrace_event_field *field;
4281         struct hist_field *hist_field;
4282         int i, idx = 0;
4283
4284         for_each_hist_field(i, hist_data) {
4285                 hist_field = hist_data->fields[i];
4286                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4287                         tracing_map_cmp_fn_t cmp_fn;
4288
4289                         field = hist_field->field;
4290
4291                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4292                                 cmp_fn = tracing_map_cmp_none;
4293                         else if (!field)
4294                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4295                                                              hist_field->is_signed);
4296                         else if (is_string_field(field))
4297                                 cmp_fn = tracing_map_cmp_string;
4298                         else
4299                                 cmp_fn = tracing_map_cmp_num(field->size,
4300                                                              field->is_signed);
4301                         idx = tracing_map_add_key_field(map,
4302                                                         hist_field->offset,
4303                                                         cmp_fn);
4304                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4305                         idx = tracing_map_add_sum_field(map);
4306
4307                 if (idx < 0)
4308                         return idx;
4309
4310                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4311                         idx = tracing_map_add_var(map);
4312                         if (idx < 0)
4313                                 return idx;
4314                         hist_field->var.idx = idx;
4315                         hist_field->var.hist_data = hist_data;
4316                 }
4317         }
4318
4319         return 0;
4320 }
4321
4322 static struct hist_trigger_data *
4323 create_hist_data(unsigned int map_bits,
4324                  struct hist_trigger_attrs *attrs,
4325                  struct trace_event_file *file,
4326                  bool remove)
4327 {
4328         const struct tracing_map_ops *map_ops = NULL;
4329         struct hist_trigger_data *hist_data;
4330         int ret = 0;
4331
4332         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4333         if (!hist_data)
4334                 return ERR_PTR(-ENOMEM);
4335
4336         hist_data->attrs = attrs;
4337         hist_data->remove = remove;
4338         hist_data->event_file = file;
4339
4340         ret = parse_actions(hist_data);
4341         if (ret)
4342                 goto free;
4343
4344         ret = create_hist_fields(hist_data, file);
4345         if (ret)
4346                 goto free;
4347
4348         ret = create_sort_keys(hist_data);
4349         if (ret)
4350                 goto free;
4351
4352         map_ops = &hist_trigger_elt_data_ops;
4353
4354         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4355                                             map_ops, hist_data);
4356         if (IS_ERR(hist_data->map)) {
4357                 ret = PTR_ERR(hist_data->map);
4358                 hist_data->map = NULL;
4359                 goto free;
4360         }
4361
4362         ret = create_tracing_map_fields(hist_data);
4363         if (ret)
4364                 goto free;
4365  out:
4366         return hist_data;
4367  free:
4368         hist_data->attrs = NULL;
4369
4370         destroy_hist_data(hist_data);
4371
4372         hist_data = ERR_PTR(ret);
4373
4374         goto out;
4375 }
4376
4377 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4378                                     struct tracing_map_elt *elt, void *rec,
4379                                     struct ring_buffer_event *rbe,
4380                                     u64 *var_ref_vals)
4381 {
4382         struct hist_elt_data *elt_data;
4383         struct hist_field *hist_field;
4384         unsigned int i, var_idx;
4385         u64 hist_val;
4386
4387         elt_data = elt->private_data;
4388         elt_data->var_ref_vals = var_ref_vals;
4389
4390         for_each_hist_val_field(i, hist_data) {
4391                 hist_field = hist_data->fields[i];
4392                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4393                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4394                         var_idx = hist_field->var.idx;
4395                         tracing_map_set_var(elt, var_idx, hist_val);
4396                         continue;
4397                 }
4398                 tracing_map_update_sum(elt, i, hist_val);
4399         }
4400
4401         for_each_hist_key_field(i, hist_data) {
4402                 hist_field = hist_data->fields[i];
4403                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4404                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4405                         var_idx = hist_field->var.idx;
4406                         tracing_map_set_var(elt, var_idx, hist_val);
4407                 }
4408         }
4409
4410         update_field_vars(hist_data, elt, rbe, rec);
4411 }
4412
4413 static inline void add_to_key(char *compound_key, void *key,
4414                               struct hist_field *key_field, void *rec)
4415 {
4416         size_t size = key_field->size;
4417
4418         if (key_field->flags & HIST_FIELD_FL_STRING) {
4419                 struct ftrace_event_field *field;
4420
4421                 field = key_field->field;
4422                 if (field->filter_type == FILTER_DYN_STRING)
4423                         size = *(u32 *)(rec + field->offset) >> 16;
4424                 else if (field->filter_type == FILTER_PTR_STRING)
4425                         size = strlen(key);
4426                 else if (field->filter_type == FILTER_STATIC_STRING)
4427                         size = field->size;
4428
4429                 /* ensure NULL-termination */
4430                 if (size > key_field->size - 1)
4431                         size = key_field->size - 1;
4432
4433                 strncpy(compound_key + key_field->offset, (char *)key, size);
4434         } else
4435                 memcpy(compound_key + key_field->offset, key, size);
4436 }
4437
4438 static void
4439 hist_trigger_actions(struct hist_trigger_data *hist_data,
4440                      struct tracing_map_elt *elt, void *rec,
4441                      struct ring_buffer_event *rbe, void *key,
4442                      u64 *var_ref_vals)
4443 {
4444         struct action_data *data;
4445         unsigned int i;
4446
4447         for (i = 0; i < hist_data->n_actions; i++) {
4448                 data = hist_data->actions[i];
4449                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4450         }
4451 }
4452
4453 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4454                                struct ring_buffer_event *rbe)
4455 {
4456         struct hist_trigger_data *hist_data = data->private_data;
4457         bool use_compound_key = (hist_data->n_keys > 1);
4458         unsigned long entries[HIST_STACKTRACE_DEPTH];
4459         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4460         char compound_key[HIST_KEY_SIZE_MAX];
4461         struct tracing_map_elt *elt = NULL;
4462         struct hist_field *key_field;
4463         u64 field_contents;
4464         void *key = NULL;
4465         unsigned int i;
4466
4467         memset(compound_key, 0, hist_data->key_size);
4468
4469         for_each_hist_key_field(i, hist_data) {
4470                 key_field = hist_data->fields[i];
4471
4472                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4473                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4474                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4475                                          HIST_STACKTRACE_SKIP);
4476                         key = entries;
4477                 } else {
4478                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4479                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4480                                 key = (void *)(unsigned long)field_contents;
4481                                 use_compound_key = true;
4482                         } else
4483                                 key = (void *)&field_contents;
4484                 }
4485
4486                 if (use_compound_key)
4487                         add_to_key(compound_key, key, key_field, rec);
4488         }
4489
4490         if (use_compound_key)
4491                 key = compound_key;
4492
4493         if (hist_data->n_var_refs &&
4494             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4495                 return;
4496
4497         elt = tracing_map_insert(hist_data->map, key);
4498         if (!elt)
4499                 return;
4500
4501         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4502
4503         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4504                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
4505 }
4506
4507 static void hist_trigger_stacktrace_print(struct seq_file *m,
4508                                           unsigned long *stacktrace_entries,
4509                                           unsigned int max_entries)
4510 {
4511         char str[KSYM_SYMBOL_LEN];
4512         unsigned int spaces = 8;
4513         unsigned int i;
4514
4515         for (i = 0; i < max_entries; i++) {
4516                 if (!stacktrace_entries[i])
4517                         return;
4518
4519                 seq_printf(m, "%*c", 1 + spaces, ' ');
4520                 sprint_symbol(str, stacktrace_entries[i]);
4521                 seq_printf(m, "%s\n", str);
4522         }
4523 }
4524
4525 static void hist_trigger_print_key(struct seq_file *m,
4526                                    struct hist_trigger_data *hist_data,
4527                                    void *key,
4528                                    struct tracing_map_elt *elt)
4529 {
4530         struct hist_field *key_field;
4531         char str[KSYM_SYMBOL_LEN];
4532         bool multiline = false;
4533         const char *field_name;
4534         unsigned int i;
4535         u64 uval;
4536
4537         seq_puts(m, "{ ");
4538
4539         for_each_hist_key_field(i, hist_data) {
4540                 key_field = hist_data->fields[i];
4541
4542                 if (i > hist_data->n_vals)
4543                         seq_puts(m, ", ");
4544
4545                 field_name = hist_field_name(key_field, 0);
4546
4547                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4548                         uval = *(u64 *)(key + key_field->offset);
4549                         seq_printf(m, "%s: %llx", field_name, uval);
4550                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4551                         uval = *(u64 *)(key + key_field->offset);
4552                         sprint_symbol_no_offset(str, uval);
4553                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4554                                    uval, str);
4555                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4556                         uval = *(u64 *)(key + key_field->offset);
4557                         sprint_symbol(str, uval);
4558                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4559                                    uval, str);
4560                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4561                         struct hist_elt_data *elt_data = elt->private_data;
4562                         char *comm;
4563
4564                         if (WARN_ON_ONCE(!elt_data))
4565                                 return;
4566
4567                         comm = elt_data->comm;
4568
4569                         uval = *(u64 *)(key + key_field->offset);
4570                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4571                                    comm, uval);
4572                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4573                         const char *syscall_name;
4574
4575                         uval = *(u64 *)(key + key_field->offset);
4576                         syscall_name = get_syscall_name(uval);
4577                         if (!syscall_name)
4578                                 syscall_name = "unknown_syscall";
4579
4580                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4581                                    syscall_name, uval);
4582                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4583                         seq_puts(m, "stacktrace:\n");
4584                         hist_trigger_stacktrace_print(m,
4585                                                       key + key_field->offset,
4586                                                       HIST_STACKTRACE_DEPTH);
4587                         multiline = true;
4588                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4589                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4590                                    *(u64 *)(key + key_field->offset));
4591                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4592                         seq_printf(m, "%s: %-50s", field_name,
4593                                    (char *)(key + key_field->offset));
4594                 } else {
4595                         uval = *(u64 *)(key + key_field->offset);
4596                         seq_printf(m, "%s: %10llu", field_name, uval);
4597                 }
4598         }
4599
4600         if (!multiline)
4601                 seq_puts(m, " ");
4602
4603         seq_puts(m, "}");
4604 }
4605
4606 static void hist_trigger_entry_print(struct seq_file *m,
4607                                      struct hist_trigger_data *hist_data,
4608                                      void *key,
4609                                      struct tracing_map_elt *elt)
4610 {
4611         const char *field_name;
4612         unsigned int i;
4613
4614         hist_trigger_print_key(m, hist_data, key, elt);
4615
4616         seq_printf(m, " hitcount: %10llu",
4617                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4618
4619         for (i = 1; i < hist_data->n_vals; i++) {
4620                 field_name = hist_field_name(hist_data->fields[i], 0);
4621
4622                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4623                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4624                         continue;
4625
4626                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4627                         seq_printf(m, "  %s: %10llx", field_name,
4628                                    tracing_map_read_sum(elt, i));
4629                 } else {
4630                         seq_printf(m, "  %s: %10llu", field_name,
4631                                    tracing_map_read_sum(elt, i));
4632                 }
4633         }
4634
4635         print_actions(m, hist_data, elt);
4636
4637         seq_puts(m, "\n");
4638 }
4639
4640 static int print_entries(struct seq_file *m,
4641                          struct hist_trigger_data *hist_data)
4642 {
4643         struct tracing_map_sort_entry **sort_entries = NULL;
4644         struct tracing_map *map = hist_data->map;
4645         int i, n_entries;
4646
4647         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4648                                              hist_data->n_sort_keys,
4649                                              &sort_entries);
4650         if (n_entries < 0)
4651                 return n_entries;
4652
4653         for (i = 0; i < n_entries; i++)
4654                 hist_trigger_entry_print(m, hist_data,
4655                                          sort_entries[i]->key,
4656                                          sort_entries[i]->elt);
4657
4658         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4659
4660         return n_entries;
4661 }
4662
4663 static void hist_trigger_show(struct seq_file *m,
4664                               struct event_trigger_data *data, int n)
4665 {
4666         struct hist_trigger_data *hist_data;
4667         int n_entries;
4668
4669         if (n > 0)
4670                 seq_puts(m, "\n\n");
4671
4672         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4673         data->ops->print(m, data->ops, data);
4674         seq_puts(m, "#\n\n");
4675
4676         hist_data = data->private_data;
4677         n_entries = print_entries(m, hist_data);
4678         if (n_entries < 0)
4679                 n_entries = 0;
4680
4681         track_data_snapshot_print(m, hist_data);
4682
4683         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4684                    (u64)atomic64_read(&hist_data->map->hits),
4685                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4686 }
4687
4688 static int hist_show(struct seq_file *m, void *v)
4689 {
4690         struct event_trigger_data *data;
4691         struct trace_event_file *event_file;
4692         int n = 0, ret = 0;
4693
4694         mutex_lock(&event_mutex);
4695
4696         event_file = event_file_data(m->private);
4697         if (unlikely(!event_file)) {
4698                 ret = -ENODEV;
4699                 goto out_unlock;
4700         }
4701
4702         list_for_each_entry(data, &event_file->triggers, list) {
4703                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4704                         hist_trigger_show(m, data, n++);
4705         }
4706
4707  out_unlock:
4708         mutex_unlock(&event_mutex);
4709
4710         return ret;
4711 }
4712
4713 static int event_hist_open(struct inode *inode, struct file *file)
4714 {
4715         int ret;
4716
4717         ret = security_locked_down(LOCKDOWN_TRACEFS);
4718         if (ret)
4719                 return ret;
4720
4721         return single_open(file, hist_show, file);
4722 }
4723
4724 const struct file_operations event_hist_fops = {
4725         .open = event_hist_open,
4726         .read = seq_read,
4727         .llseek = seq_lseek,
4728         .release = single_release,
4729 };
4730
4731 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4732 static void hist_field_debug_show_flags(struct seq_file *m,
4733                                         unsigned long flags)
4734 {
4735         seq_puts(m, "      flags:\n");
4736
4737         if (flags & HIST_FIELD_FL_KEY)
4738                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4739         else if (flags & HIST_FIELD_FL_HITCOUNT)
4740                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4741         else if (flags & HIST_FIELD_FL_VAR)
4742                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4743         else if (flags & HIST_FIELD_FL_VAR_REF)
4744                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4745         else
4746                 seq_puts(m, "        VAL: normal u64 value\n");
4747
4748         if (flags & HIST_FIELD_FL_ALIAS)
4749                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4750 }
4751
4752 static int hist_field_debug_show(struct seq_file *m,
4753                                  struct hist_field *field, unsigned long flags)
4754 {
4755         if ((field->flags & flags) != flags) {
4756                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4757                 return -EINVAL;
4758         }
4759
4760         hist_field_debug_show_flags(m, field->flags);
4761         if (field->field)
4762                 seq_printf(m, "      ftrace_event_field name: %s\n",
4763                            field->field->name);
4764
4765         if (field->flags & HIST_FIELD_FL_VAR) {
4766                 seq_printf(m, "      var.name: %s\n", field->var.name);
4767                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4768                            field->var.idx);
4769         }
4770
4771         if (field->flags & HIST_FIELD_FL_ALIAS)
4772                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4773                            field->var_ref_idx);
4774
4775         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4776                 seq_printf(m, "      name: %s\n", field->name);
4777                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4778                            field->var.idx);
4779                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4780                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4781                            field->var_ref_idx);
4782                 if (field->system)
4783                         seq_printf(m, "      system: %s\n", field->system);
4784                 if (field->event_name)
4785                         seq_printf(m, "      event_name: %s\n", field->event_name);
4786         }
4787
4788         seq_printf(m, "      type: %s\n", field->type);
4789         seq_printf(m, "      size: %u\n", field->size);
4790         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4791
4792         return 0;
4793 }
4794
4795 static int field_var_debug_show(struct seq_file *m,
4796                                 struct field_var *field_var, unsigned int i,
4797                                 bool save_vars)
4798 {
4799         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4800         struct hist_field *field;
4801         int ret = 0;
4802
4803         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4804
4805         field = field_var->var;
4806
4807         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4808
4809         hist_field_debug_show_flags(m, field->flags);
4810         seq_printf(m, "      var.name: %s\n", field->var.name);
4811         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4812                    field->var.idx);
4813
4814         field = field_var->val;
4815
4816         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4817         if (field->field)
4818                 seq_printf(m, "      ftrace_event_field name: %s\n",
4819                            field->field->name);
4820         else {
4821                 ret = -EINVAL;
4822                 goto out;
4823         }
4824
4825         seq_printf(m, "      type: %s\n", field->type);
4826         seq_printf(m, "      size: %u\n", field->size);
4827         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4828 out:
4829         return ret;
4830 }
4831
4832 static int hist_action_debug_show(struct seq_file *m,
4833                                   struct action_data *data, int i)
4834 {
4835         int ret = 0;
4836
4837         if (data->handler == HANDLER_ONMAX ||
4838             data->handler == HANDLER_ONCHANGE) {
4839                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4840                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4841                                             HIST_FIELD_FL_VAR_REF);
4842                 if (ret)
4843                         goto out;
4844
4845                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4846                 ret = hist_field_debug_show(m, data->track_data.track_var,
4847                                             HIST_FIELD_FL_VAR);
4848                 if (ret)
4849                         goto out;
4850         }
4851
4852         if (data->handler == HANDLER_ONMATCH) {
4853                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4854                            i, data->match_data.event_system);
4855                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4856                            i, data->match_data.event);
4857         }
4858 out:
4859         return ret;
4860 }
4861
4862 static int hist_actions_debug_show(struct seq_file *m,
4863                                    struct hist_trigger_data *hist_data)
4864 {
4865         int i, ret = 0;
4866
4867         if (hist_data->n_actions)
4868                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4869
4870         for (i = 0; i < hist_data->n_actions; i++) {
4871                 struct action_data *action = hist_data->actions[i];
4872
4873                 ret = hist_action_debug_show(m, action, i);
4874                 if (ret)
4875                         goto out;
4876         }
4877
4878         if (hist_data->n_save_vars)
4879                 seq_puts(m, "\n  save action variables (save() params):\n");
4880
4881         for (i = 0; i < hist_data->n_save_vars; i++) {
4882                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4883                 if (ret)
4884                         goto out;
4885         }
4886 out:
4887         return ret;
4888 }
4889
4890 static void hist_trigger_debug_show(struct seq_file *m,
4891                                     struct event_trigger_data *data, int n)
4892 {
4893         struct hist_trigger_data *hist_data;
4894         int i, ret;
4895
4896         if (n > 0)
4897                 seq_puts(m, "\n\n");
4898
4899         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4900         data->ops->print(m, data->ops, data);
4901         seq_puts(m, "#\n\n");
4902
4903         hist_data = data->private_data;
4904
4905         seq_printf(m, "hist_data: %p\n\n", hist_data);
4906         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4907         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4908         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4909
4910         seq_puts(m, "\n  val fields:\n\n");
4911
4912         seq_puts(m, "    hist_data->fields[0]:\n");
4913         ret = hist_field_debug_show(m, hist_data->fields[0],
4914                                     HIST_FIELD_FL_HITCOUNT);
4915         if (ret)
4916                 return;
4917
4918         for (i = 1; i < hist_data->n_vals; i++) {
4919                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4920                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4921                 if (ret)
4922                         return;
4923         }
4924
4925         seq_puts(m, "\n  key fields:\n");
4926
4927         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
4928                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4929                 ret = hist_field_debug_show(m, hist_data->fields[i],
4930                                             HIST_FIELD_FL_KEY);
4931                 if (ret)
4932                         return;
4933         }
4934
4935         if (hist_data->n_var_refs)
4936                 seq_puts(m, "\n  variable reference fields:\n");
4937
4938         for (i = 0; i < hist_data->n_var_refs; i++) {
4939                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
4940                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
4941                                             HIST_FIELD_FL_VAR_REF);
4942                 if (ret)
4943                         return;
4944         }
4945
4946         if (hist_data->n_field_vars)
4947                 seq_puts(m, "\n  field variables:\n");
4948
4949         for (i = 0; i < hist_data->n_field_vars; i++) {
4950                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
4951                 if (ret)
4952                         return;
4953         }
4954
4955         ret = hist_actions_debug_show(m, hist_data);
4956         if (ret)
4957                 return;
4958 }
4959
4960 static int hist_debug_show(struct seq_file *m, void *v)
4961 {
4962         struct event_trigger_data *data;
4963         struct trace_event_file *event_file;
4964         int n = 0, ret = 0;
4965
4966         mutex_lock(&event_mutex);
4967
4968         event_file = event_file_data(m->private);
4969         if (unlikely(!event_file)) {
4970                 ret = -ENODEV;
4971                 goto out_unlock;
4972         }
4973
4974         list_for_each_entry(data, &event_file->triggers, list) {
4975                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4976                         hist_trigger_debug_show(m, data, n++);
4977         }
4978
4979  out_unlock:
4980         mutex_unlock(&event_mutex);
4981
4982         return ret;
4983 }
4984
4985 static int event_hist_debug_open(struct inode *inode, struct file *file)
4986 {
4987         int ret;
4988
4989         ret = security_locked_down(LOCKDOWN_TRACEFS);
4990         if (ret)
4991                 return ret;
4992
4993         return single_open(file, hist_debug_show, file);
4994 }
4995
4996 const struct file_operations event_hist_debug_fops = {
4997         .open = event_hist_debug_open,
4998         .read = seq_read,
4999         .llseek = seq_lseek,
5000         .release = single_release,
5001 };
5002 #endif
5003
5004 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5005 {
5006         const char *field_name = hist_field_name(hist_field, 0);
5007
5008         if (hist_field->var.name)
5009                 seq_printf(m, "%s=", hist_field->var.name);
5010
5011         if (hist_field->flags & HIST_FIELD_FL_CPU)
5012                 seq_puts(m, "cpu");
5013         else if (field_name) {
5014                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5015                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5016                         seq_putc(m, '$');
5017                 seq_printf(m, "%s", field_name);
5018         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5019                 seq_puts(m, "common_timestamp");
5020
5021         if (hist_field->flags) {
5022                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5023                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5024                         const char *flags = get_hist_field_flags(hist_field);
5025
5026                         if (flags)
5027                                 seq_printf(m, ".%s", flags);
5028                 }
5029         }
5030 }
5031
5032 static int event_hist_trigger_print(struct seq_file *m,
5033                                     struct event_trigger_ops *ops,
5034                                     struct event_trigger_data *data)
5035 {
5036         struct hist_trigger_data *hist_data = data->private_data;
5037         struct hist_field *field;
5038         bool have_var = false;
5039         unsigned int i;
5040
5041         seq_puts(m, "hist:");
5042
5043         if (data->name)
5044                 seq_printf(m, "%s:", data->name);
5045
5046         seq_puts(m, "keys=");
5047
5048         for_each_hist_key_field(i, hist_data) {
5049                 field = hist_data->fields[i];
5050
5051                 if (i > hist_data->n_vals)
5052                         seq_puts(m, ",");
5053
5054                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5055                         seq_puts(m, "stacktrace");
5056                 else
5057                         hist_field_print(m, field);
5058         }
5059
5060         seq_puts(m, ":vals=");
5061
5062         for_each_hist_val_field(i, hist_data) {
5063                 field = hist_data->fields[i];
5064                 if (field->flags & HIST_FIELD_FL_VAR) {
5065                         have_var = true;
5066                         continue;
5067                 }
5068
5069                 if (i == HITCOUNT_IDX)
5070                         seq_puts(m, "hitcount");
5071                 else {
5072                         seq_puts(m, ",");
5073                         hist_field_print(m, field);
5074                 }
5075         }
5076
5077         if (have_var) {
5078                 unsigned int n = 0;
5079
5080                 seq_puts(m, ":");
5081
5082                 for_each_hist_val_field(i, hist_data) {
5083                         field = hist_data->fields[i];
5084
5085                         if (field->flags & HIST_FIELD_FL_VAR) {
5086                                 if (n++)
5087                                         seq_puts(m, ",");
5088                                 hist_field_print(m, field);
5089                         }
5090                 }
5091         }
5092
5093         seq_puts(m, ":sort=");
5094
5095         for (i = 0; i < hist_data->n_sort_keys; i++) {
5096                 struct tracing_map_sort_key *sort_key;
5097                 unsigned int idx, first_key_idx;
5098
5099                 /* skip VAR vals */
5100                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5101
5102                 sort_key = &hist_data->sort_keys[i];
5103                 idx = sort_key->field_idx;
5104
5105                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5106                         return -EINVAL;
5107
5108                 if (i > 0)
5109                         seq_puts(m, ",");
5110
5111                 if (idx == HITCOUNT_IDX)
5112                         seq_puts(m, "hitcount");
5113                 else {
5114                         if (idx >= first_key_idx)
5115                                 idx += hist_data->n_vars;
5116                         hist_field_print(m, hist_data->fields[idx]);
5117                 }
5118
5119                 if (sort_key->descending)
5120                         seq_puts(m, ".descending");
5121         }
5122         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5123         if (hist_data->enable_timestamps)
5124                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5125
5126         print_actions_spec(m, hist_data);
5127
5128         if (data->filter_str)
5129                 seq_printf(m, " if %s", data->filter_str);
5130
5131         if (data->paused)
5132                 seq_puts(m, " [paused]");
5133         else
5134                 seq_puts(m, " [active]");
5135
5136         seq_putc(m, '\n');
5137
5138         return 0;
5139 }
5140
5141 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5142                                    struct event_trigger_data *data)
5143 {
5144         struct hist_trigger_data *hist_data = data->private_data;
5145
5146         if (!data->ref && hist_data->attrs->name)
5147                 save_named_trigger(hist_data->attrs->name, data);
5148
5149         data->ref++;
5150
5151         return 0;
5152 }
5153
5154 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5155 {
5156         struct trace_event_file *file;
5157         unsigned int i;
5158         char *cmd;
5159         int ret;
5160
5161         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5162                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5163                 cmd = hist_data->field_var_hists[i]->cmd;
5164                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5165                                               "!hist", "hist", cmd);
5166         }
5167 }
5168
5169 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5170                                     struct event_trigger_data *data)
5171 {
5172         struct hist_trigger_data *hist_data = data->private_data;
5173
5174         if (WARN_ON_ONCE(data->ref <= 0))
5175                 return;
5176
5177         data->ref--;
5178         if (!data->ref) {
5179                 if (data->name)
5180                         del_named_trigger(data);
5181
5182                 trigger_data_free(data);
5183
5184                 remove_hist_vars(hist_data);
5185
5186                 unregister_field_var_hists(hist_data);
5187
5188                 destroy_hist_data(hist_data);
5189         }
5190 }
5191
5192 static struct event_trigger_ops event_hist_trigger_ops = {
5193         .func                   = event_hist_trigger,
5194         .print                  = event_hist_trigger_print,
5195         .init                   = event_hist_trigger_init,
5196         .free                   = event_hist_trigger_free,
5197 };
5198
5199 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5200                                          struct event_trigger_data *data)
5201 {
5202         data->ref++;
5203
5204         save_named_trigger(data->named_data->name, data);
5205
5206         event_hist_trigger_init(ops, data->named_data);
5207
5208         return 0;
5209 }
5210
5211 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5212                                           struct event_trigger_data *data)
5213 {
5214         if (WARN_ON_ONCE(data->ref <= 0))
5215                 return;
5216
5217         event_hist_trigger_free(ops, data->named_data);
5218
5219         data->ref--;
5220         if (!data->ref) {
5221                 del_named_trigger(data);
5222                 trigger_data_free(data);
5223         }
5224 }
5225
5226 static struct event_trigger_ops event_hist_trigger_named_ops = {
5227         .func                   = event_hist_trigger,
5228         .print                  = event_hist_trigger_print,
5229         .init                   = event_hist_trigger_named_init,
5230         .free                   = event_hist_trigger_named_free,
5231 };
5232
5233 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5234                                                             char *param)
5235 {
5236         return &event_hist_trigger_ops;
5237 }
5238
5239 static void hist_clear(struct event_trigger_data *data)
5240 {
5241         struct hist_trigger_data *hist_data = data->private_data;
5242
5243         if (data->name)
5244                 pause_named_trigger(data);
5245
5246         tracepoint_synchronize_unregister();
5247
5248         tracing_map_clear(hist_data->map);
5249
5250         if (data->name)
5251                 unpause_named_trigger(data);
5252 }
5253
5254 static bool compatible_field(struct ftrace_event_field *field,
5255                              struct ftrace_event_field *test_field)
5256 {
5257         if (field == test_field)
5258                 return true;
5259         if (field == NULL || test_field == NULL)
5260                 return false;
5261         if (strcmp(field->name, test_field->name) != 0)
5262                 return false;
5263         if (strcmp(field->type, test_field->type) != 0)
5264                 return false;
5265         if (field->size != test_field->size)
5266                 return false;
5267         if (field->is_signed != test_field->is_signed)
5268                 return false;
5269
5270         return true;
5271 }
5272
5273 static bool hist_trigger_match(struct event_trigger_data *data,
5274                                struct event_trigger_data *data_test,
5275                                struct event_trigger_data *named_data,
5276                                bool ignore_filter)
5277 {
5278         struct tracing_map_sort_key *sort_key, *sort_key_test;
5279         struct hist_trigger_data *hist_data, *hist_data_test;
5280         struct hist_field *key_field, *key_field_test;
5281         unsigned int i;
5282
5283         if (named_data && (named_data != data_test) &&
5284             (named_data != data_test->named_data))
5285                 return false;
5286
5287         if (!named_data && is_named_trigger(data_test))
5288                 return false;
5289
5290         hist_data = data->private_data;
5291         hist_data_test = data_test->private_data;
5292
5293         if (hist_data->n_vals != hist_data_test->n_vals ||
5294             hist_data->n_fields != hist_data_test->n_fields ||
5295             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5296                 return false;
5297
5298         if (!ignore_filter) {
5299                 if ((data->filter_str && !data_test->filter_str) ||
5300                    (!data->filter_str && data_test->filter_str))
5301                         return false;
5302         }
5303
5304         for_each_hist_field(i, hist_data) {
5305                 key_field = hist_data->fields[i];
5306                 key_field_test = hist_data_test->fields[i];
5307
5308                 if (key_field->flags != key_field_test->flags)
5309                         return false;
5310                 if (!compatible_field(key_field->field, key_field_test->field))
5311                         return false;
5312                 if (key_field->offset != key_field_test->offset)
5313                         return false;
5314                 if (key_field->size != key_field_test->size)
5315                         return false;
5316                 if (key_field->is_signed != key_field_test->is_signed)
5317                         return false;
5318                 if (!!key_field->var.name != !!key_field_test->var.name)
5319                         return false;
5320                 if (key_field->var.name &&
5321                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5322                         return false;
5323         }
5324
5325         for (i = 0; i < hist_data->n_sort_keys; i++) {
5326                 sort_key = &hist_data->sort_keys[i];
5327                 sort_key_test = &hist_data_test->sort_keys[i];
5328
5329                 if (sort_key->field_idx != sort_key_test->field_idx ||
5330                     sort_key->descending != sort_key_test->descending)
5331                         return false;
5332         }
5333
5334         if (!ignore_filter && data->filter_str &&
5335             (strcmp(data->filter_str, data_test->filter_str) != 0))
5336                 return false;
5337
5338         if (!actions_match(hist_data, hist_data_test))
5339                 return false;
5340
5341         return true;
5342 }
5343
5344 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5345                                  struct event_trigger_data *data,
5346                                  struct trace_event_file *file)
5347 {
5348         struct hist_trigger_data *hist_data = data->private_data;
5349         struct event_trigger_data *test, *named_data = NULL;
5350         struct trace_array *tr = file->tr;
5351         int ret = 0;
5352
5353         if (hist_data->attrs->name) {
5354                 named_data = find_named_trigger(hist_data->attrs->name);
5355                 if (named_data) {
5356                         if (!hist_trigger_match(data, named_data, named_data,
5357                                                 true)) {
5358                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5359                                 ret = -EINVAL;
5360                                 goto out;
5361                         }
5362                 }
5363         }
5364
5365         if (hist_data->attrs->name && !named_data)
5366                 goto new;
5367
5368         lockdep_assert_held(&event_mutex);
5369
5370         list_for_each_entry(test, &file->triggers, list) {
5371                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5372                         if (!hist_trigger_match(data, test, named_data, false))
5373                                 continue;
5374                         if (hist_data->attrs->pause)
5375                                 test->paused = true;
5376                         else if (hist_data->attrs->cont)
5377                                 test->paused = false;
5378                         else if (hist_data->attrs->clear)
5379                                 hist_clear(test);
5380                         else {
5381                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5382                                 ret = -EEXIST;
5383                         }
5384                         goto out;
5385                 }
5386         }
5387  new:
5388         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5389                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5390                 ret = -ENOENT;
5391                 goto out;
5392         }
5393
5394         if (hist_data->attrs->pause)
5395                 data->paused = true;
5396
5397         if (named_data) {
5398                 data->private_data = named_data->private_data;
5399                 set_named_trigger_data(data, named_data);
5400                 data->ops = &event_hist_trigger_named_ops;
5401         }
5402
5403         if (data->ops->init) {
5404                 ret = data->ops->init(data->ops, data);
5405                 if (ret < 0)
5406                         goto out;
5407         }
5408
5409         if (hist_data->enable_timestamps) {
5410                 char *clock = hist_data->attrs->clock;
5411
5412                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5413                 if (ret) {
5414                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5415                         goto out;
5416                 }
5417
5418                 tracing_set_time_stamp_abs(file->tr, true);
5419         }
5420
5421         if (named_data)
5422                 destroy_hist_data(hist_data);
5423
5424         ret++;
5425  out:
5426         return ret;
5427 }
5428
5429 static int hist_trigger_enable(struct event_trigger_data *data,
5430                                struct trace_event_file *file)
5431 {
5432         int ret = 0;
5433
5434         list_add_tail_rcu(&data->list, &file->triggers);
5435
5436         update_cond_flag(file);
5437
5438         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5439                 list_del_rcu(&data->list);
5440                 update_cond_flag(file);
5441                 ret--;
5442         }
5443
5444         return ret;
5445 }
5446
5447 static bool have_hist_trigger_match(struct event_trigger_data *data,
5448                                     struct trace_event_file *file)
5449 {
5450         struct hist_trigger_data *hist_data = data->private_data;
5451         struct event_trigger_data *test, *named_data = NULL;
5452         bool match = false;
5453
5454         lockdep_assert_held(&event_mutex);
5455
5456         if (hist_data->attrs->name)
5457                 named_data = find_named_trigger(hist_data->attrs->name);
5458
5459         list_for_each_entry(test, &file->triggers, list) {
5460                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5461                         if (hist_trigger_match(data, test, named_data, false)) {
5462                                 match = true;
5463                                 break;
5464                         }
5465                 }
5466         }
5467
5468         return match;
5469 }
5470
5471 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5472                                     struct trace_event_file *file)
5473 {
5474         struct hist_trigger_data *hist_data = data->private_data;
5475         struct event_trigger_data *test, *named_data = NULL;
5476
5477         lockdep_assert_held(&event_mutex);
5478
5479         if (hist_data->attrs->name)
5480                 named_data = find_named_trigger(hist_data->attrs->name);
5481
5482         list_for_each_entry(test, &file->triggers, list) {
5483                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5484                         if (!hist_trigger_match(data, test, named_data, false))
5485                                 continue;
5486                         hist_data = test->private_data;
5487                         if (check_var_refs(hist_data))
5488                                 return true;
5489                         break;
5490                 }
5491         }
5492
5493         return false;
5494 }
5495
5496 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5497                                     struct event_trigger_data *data,
5498                                     struct trace_event_file *file)
5499 {
5500         struct hist_trigger_data *hist_data = data->private_data;
5501         struct event_trigger_data *test, *named_data = NULL;
5502         bool unregistered = false;
5503
5504         lockdep_assert_held(&event_mutex);
5505
5506         if (hist_data->attrs->name)
5507                 named_data = find_named_trigger(hist_data->attrs->name);
5508
5509         list_for_each_entry(test, &file->triggers, list) {
5510                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5511                         if (!hist_trigger_match(data, test, named_data, false))
5512                                 continue;
5513                         unregistered = true;
5514                         list_del_rcu(&test->list);
5515                         trace_event_trigger_enable_disable(file, 0);
5516                         update_cond_flag(file);
5517                         break;
5518                 }
5519         }
5520
5521         if (unregistered && test->ops->free)
5522                 test->ops->free(test->ops, test);
5523
5524         if (hist_data->enable_timestamps) {
5525                 if (!hist_data->remove || unregistered)
5526                         tracing_set_time_stamp_abs(file->tr, false);
5527         }
5528 }
5529
5530 static bool hist_file_check_refs(struct trace_event_file *file)
5531 {
5532         struct hist_trigger_data *hist_data;
5533         struct event_trigger_data *test;
5534
5535         lockdep_assert_held(&event_mutex);
5536
5537         list_for_each_entry(test, &file->triggers, list) {
5538                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5539                         hist_data = test->private_data;
5540                         if (check_var_refs(hist_data))
5541                                 return true;
5542                 }
5543         }
5544
5545         return false;
5546 }
5547
5548 static void hist_unreg_all(struct trace_event_file *file)
5549 {
5550         struct event_trigger_data *test, *n;
5551         struct hist_trigger_data *hist_data;
5552         struct synth_event *se;
5553         const char *se_name;
5554
5555         lockdep_assert_held(&event_mutex);
5556
5557         if (hist_file_check_refs(file))
5558                 return;
5559
5560         list_for_each_entry_safe(test, n, &file->triggers, list) {
5561                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5562                         hist_data = test->private_data;
5563                         list_del_rcu(&test->list);
5564                         trace_event_trigger_enable_disable(file, 0);
5565
5566                         se_name = trace_event_name(file->event_call);
5567                         se = find_synth_event(se_name);
5568                         if (se)
5569                                 se->ref--;
5570
5571                         update_cond_flag(file);
5572                         if (hist_data->enable_timestamps)
5573                                 tracing_set_time_stamp_abs(file->tr, false);
5574                         if (test->ops->free)
5575                                 test->ops->free(test->ops, test);
5576                 }
5577         }
5578 }
5579
5580 static int event_hist_trigger_func(struct event_command *cmd_ops,
5581                                    struct trace_event_file *file,
5582                                    char *glob, char *cmd, char *param)
5583 {
5584         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5585         struct event_trigger_data *trigger_data;
5586         struct hist_trigger_attrs *attrs;
5587         struct event_trigger_ops *trigger_ops;
5588         struct hist_trigger_data *hist_data;
5589         struct synth_event *se;
5590         const char *se_name;
5591         bool remove = false;
5592         char *trigger, *p;
5593         int ret = 0;
5594
5595         lockdep_assert_held(&event_mutex);
5596
5597         if (glob && strlen(glob)) {
5598                 hist_err_clear();
5599                 last_cmd_set(file, param);
5600         }
5601
5602         if (!param)
5603                 return -EINVAL;
5604
5605         if (glob[0] == '!')
5606                 remove = true;
5607
5608         /*
5609          * separate the trigger from the filter (k:v [if filter])
5610          * allowing for whitespace in the trigger
5611          */
5612         p = trigger = param;
5613         do {
5614                 p = strstr(p, "if");
5615                 if (!p)
5616                         break;
5617                 if (p == param)
5618                         return -EINVAL;
5619                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5620                         p++;
5621                         continue;
5622                 }
5623                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5624                         return -EINVAL;
5625                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5626                         p++;
5627                         continue;
5628                 }
5629                 break;
5630         } while (p);
5631
5632         if (!p)
5633                 param = NULL;
5634         else {
5635                 *(p - 1) = '\0';
5636                 param = strstrip(p);
5637                 trigger = strstrip(trigger);
5638         }
5639
5640         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5641         if (IS_ERR(attrs))
5642                 return PTR_ERR(attrs);
5643
5644         if (attrs->map_bits)
5645                 hist_trigger_bits = attrs->map_bits;
5646
5647         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5648         if (IS_ERR(hist_data)) {
5649                 destroy_hist_trigger_attrs(attrs);
5650                 return PTR_ERR(hist_data);
5651         }
5652
5653         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5654
5655         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5656         if (!trigger_data) {
5657                 ret = -ENOMEM;
5658                 goto out_free;
5659         }
5660
5661         trigger_data->count = -1;
5662         trigger_data->ops = trigger_ops;
5663         trigger_data->cmd_ops = cmd_ops;
5664
5665         INIT_LIST_HEAD(&trigger_data->list);
5666         RCU_INIT_POINTER(trigger_data->filter, NULL);
5667
5668         trigger_data->private_data = hist_data;
5669
5670         /* if param is non-empty, it's supposed to be a filter */
5671         if (param && cmd_ops->set_filter) {
5672                 ret = cmd_ops->set_filter(param, trigger_data, file);
5673                 if (ret < 0)
5674                         goto out_free;
5675         }
5676
5677         if (remove) {
5678                 if (!have_hist_trigger_match(trigger_data, file))
5679                         goto out_free;
5680
5681                 if (hist_trigger_check_refs(trigger_data, file)) {
5682                         ret = -EBUSY;
5683                         goto out_free;
5684                 }
5685
5686                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5687                 se_name = trace_event_name(file->event_call);
5688                 se = find_synth_event(se_name);
5689                 if (se)
5690                         se->ref--;
5691                 ret = 0;
5692                 goto out_free;
5693         }
5694
5695         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5696         /*
5697          * The above returns on success the # of triggers registered,
5698          * but if it didn't register any it returns zero.  Consider no
5699          * triggers registered a failure too.
5700          */
5701         if (!ret) {
5702                 if (!(attrs->pause || attrs->cont || attrs->clear))
5703                         ret = -ENOENT;
5704                 goto out_free;
5705         } else if (ret < 0)
5706                 goto out_free;
5707
5708         if (get_named_trigger_data(trigger_data))
5709                 goto enable;
5710
5711         if (has_hist_vars(hist_data))
5712                 save_hist_vars(hist_data);
5713
5714         ret = create_actions(hist_data);
5715         if (ret)
5716                 goto out_unreg;
5717
5718         ret = tracing_map_init(hist_data->map);
5719         if (ret)
5720                 goto out_unreg;
5721 enable:
5722         ret = hist_trigger_enable(trigger_data, file);
5723         if (ret)
5724                 goto out_unreg;
5725
5726         se_name = trace_event_name(file->event_call);
5727         se = find_synth_event(se_name);
5728         if (se)
5729                 se->ref++;
5730         /* Just return zero, not the number of registered triggers */
5731         ret = 0;
5732  out:
5733         if (ret == 0)
5734                 hist_err_clear();
5735
5736         return ret;
5737  out_unreg:
5738         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5739  out_free:
5740         if (cmd_ops->set_filter)
5741                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5742
5743         remove_hist_vars(hist_data);
5744
5745         kfree(trigger_data);
5746
5747         destroy_hist_data(hist_data);
5748         goto out;
5749 }
5750
5751 static struct event_command trigger_hist_cmd = {
5752         .name                   = "hist",
5753         .trigger_type           = ETT_EVENT_HIST,
5754         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5755         .func                   = event_hist_trigger_func,
5756         .reg                    = hist_register_trigger,
5757         .unreg                  = hist_unregister_trigger,
5758         .unreg_all              = hist_unreg_all,
5759         .get_trigger_ops        = event_hist_get_trigger_ops,
5760         .set_filter             = set_trigger_filter,
5761 };
5762
5763 __init int register_trigger_hist_cmd(void)
5764 {
5765         int ret;
5766
5767         ret = register_event_command(&trigger_hist_cmd);
5768         WARN_ON(ret < 0);
5769
5770         return ret;
5771 }
5772
5773 static void
5774 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5775                     struct ring_buffer_event *event)
5776 {
5777         struct enable_trigger_data *enable_data = data->private_data;
5778         struct event_trigger_data *test;
5779
5780         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5781                                 lockdep_is_held(&event_mutex)) {
5782                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5783                         if (enable_data->enable)
5784                                 test->paused = false;
5785                         else
5786                                 test->paused = true;
5787                 }
5788         }
5789 }
5790
5791 static void
5792 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5793                           struct ring_buffer_event *event)
5794 {
5795         if (!data->count)
5796                 return;
5797
5798         if (data->count != -1)
5799                 (data->count)--;
5800
5801         hist_enable_trigger(data, rec, event);
5802 }
5803
5804 static struct event_trigger_ops hist_enable_trigger_ops = {
5805         .func                   = hist_enable_trigger,
5806         .print                  = event_enable_trigger_print,
5807         .init                   = event_trigger_init,
5808         .free                   = event_enable_trigger_free,
5809 };
5810
5811 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5812         .func                   = hist_enable_count_trigger,
5813         .print                  = event_enable_trigger_print,
5814         .init                   = event_trigger_init,
5815         .free                   = event_enable_trigger_free,
5816 };
5817
5818 static struct event_trigger_ops hist_disable_trigger_ops = {
5819         .func                   = hist_enable_trigger,
5820         .print                  = event_enable_trigger_print,
5821         .init                   = event_trigger_init,
5822         .free                   = event_enable_trigger_free,
5823 };
5824
5825 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5826         .func                   = hist_enable_count_trigger,
5827         .print                  = event_enable_trigger_print,
5828         .init                   = event_trigger_init,
5829         .free                   = event_enable_trigger_free,
5830 };
5831
5832 static struct event_trigger_ops *
5833 hist_enable_get_trigger_ops(char *cmd, char *param)
5834 {
5835         struct event_trigger_ops *ops;
5836         bool enable;
5837
5838         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5839
5840         if (enable)
5841                 ops = param ? &hist_enable_count_trigger_ops :
5842                         &hist_enable_trigger_ops;
5843         else
5844                 ops = param ? &hist_disable_count_trigger_ops :
5845                         &hist_disable_trigger_ops;
5846
5847         return ops;
5848 }
5849
5850 static void hist_enable_unreg_all(struct trace_event_file *file)
5851 {
5852         struct event_trigger_data *test, *n;
5853
5854         list_for_each_entry_safe(test, n, &file->triggers, list) {
5855                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5856                         list_del_rcu(&test->list);
5857                         update_cond_flag(file);
5858                         trace_event_trigger_enable_disable(file, 0);
5859                         if (test->ops->free)
5860                                 test->ops->free(test->ops, test);
5861                 }
5862         }
5863 }
5864
5865 static struct event_command trigger_hist_enable_cmd = {
5866         .name                   = ENABLE_HIST_STR,
5867         .trigger_type           = ETT_HIST_ENABLE,
5868         .func                   = event_enable_trigger_func,
5869         .reg                    = event_enable_register_trigger,
5870         .unreg                  = event_enable_unregister_trigger,
5871         .unreg_all              = hist_enable_unreg_all,
5872         .get_trigger_ops        = hist_enable_get_trigger_ops,
5873         .set_filter             = set_trigger_filter,
5874 };
5875
5876 static struct event_command trigger_hist_disable_cmd = {
5877         .name                   = DISABLE_HIST_STR,
5878         .trigger_type           = ETT_HIST_ENABLE,
5879         .func                   = event_enable_trigger_func,
5880         .reg                    = event_enable_register_trigger,
5881         .unreg                  = event_enable_unregister_trigger,
5882         .unreg_all              = hist_enable_unreg_all,
5883         .get_trigger_ops        = hist_enable_get_trigger_ops,
5884         .set_filter             = set_trigger_filter,
5885 };
5886
5887 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5888 {
5889         unregister_event_command(&trigger_hist_enable_cmd);
5890         unregister_event_command(&trigger_hist_disable_cmd);
5891 }
5892
5893 __init int register_trigger_hist_enable_disable_cmds(void)
5894 {
5895         int ret;
5896
5897         ret = register_event_command(&trigger_hist_enable_cmd);
5898         if (WARN_ON(ret < 0))
5899                 return ret;
5900         ret = register_event_command(&trigger_hist_disable_cmd);
5901         if (WARN_ON(ret < 0))
5902                 unregister_trigger_hist_enable_disable_cmds();
5903
5904         return ret;
5905 }