kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  69
  70 #undef C
  71 #define C(a, b)         HIST_ERR_##a
  72
  73 enum { ERRORS };
  74
  75 #undef C
  76 #define C(a, b)         b
  77
  78 static const char *err_text[] = { ERRORS };
  79
  80 struct hist_field;
  81
  82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  83                                 struct tracing_map_elt *elt,
  84                                 struct ring_buffer_event *rbe,
  85                                 void *event);
  86
  87 #define HIST_FIELD_OPERANDS_MAX 2
  88 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  89 #define HIST_ACTIONS_MAX        8
  90
  91 enum field_op_id {
  92         FIELD_OP_NONE,
  93         FIELD_OP_PLUS,
  94         FIELD_OP_MINUS,
  95         FIELD_OP_UNARY_MINUS,
  96 };
  97
  98 /*
  99  * A hist_var (histogram variable) contains variable information for
 100  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 101  * flag set.  A hist_var has a variable name e.g. ts0, and is
 102  * associated with a given histogram trigger, as specified by
 103  * hist_data.  The hist_var idx is the unique index assigned to the
 104  * variable by the hist trigger's tracing_map.  The idx is what is
 105  * used to set a variable's value and, by a variable reference, to
 106  * retrieve it.
 107  */
 108 struct hist_var {
 109         char                            *name;
 110         struct hist_trigger_data        *hist_data;
 111         unsigned int                    idx;
 112 };
 113
 114 struct hist_field {
 115         struct ftrace_event_field       *field;
 116         unsigned long                   flags;
 117         hist_field_fn_t                 fn;
 118         unsigned int                    ref;
 119         unsigned int                    size;
 120         unsigned int                    offset;
 121         unsigned int                    is_signed;
 122         const char                      *type;
 123         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 124         struct hist_trigger_data        *hist_data;
 125
 126         /*
 127          * Variable fields contain variable-specific info in var.
 128          */
 129         struct hist_var                 var;
 130         enum field_op_id                operator;
 131         char                            *system;
 132         char                            *event_name;
 133
 134         /*
 135          * The name field is used for EXPR and VAR_REF fields.  VAR
 136          * fields contain the variable name in var.name.
 137          */
 138         char                            *name;
 139
 140         /*
 141          * When a histogram trigger is hit, if it has any references
 142          * to variables, the values of those variables are collected
 143          * into a var_ref_vals array by resolve_var_refs().  The
 144          * current value of each variable is read from the tracing_map
 145          * using the hist field's hist_var.idx and entered into the
 146          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 147          */
 148         unsigned int                    var_ref_idx;
 149         bool                            read_once;
 150 };
 151
 152 static u64 hist_field_none(struct hist_field *field,
 153                            struct tracing_map_elt *elt,
 154                            struct ring_buffer_event *rbe,
 155                            void *event)
 156 {
 157         return 0;
 158 }
 159
 160 static u64 hist_field_counter(struct hist_field *field,
 161                               struct tracing_map_elt *elt,
 162                               struct ring_buffer_event *rbe,
 163                               void *event)
 164 {
 165         return 1;
 166 }
 167
 168 static u64 hist_field_string(struct hist_field *hist_field,
 169                              struct tracing_map_elt *elt,
 170                              struct ring_buffer_event *rbe,
 171                              void *event)
 172 {
 173         char *addr = (char *)(event + hist_field->field->offset);
 174
 175         return (u64)(unsigned long)addr;
 176 }
 177
 178 static u64 hist_field_dynstring(struct hist_field *hist_field,
 179                                 struct tracing_map_elt *elt,
 180                                 struct ring_buffer_event *rbe,
 181                                 void *event)
 182 {
 183         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 184         int str_loc = str_item & 0xffff;
 185         char *addr = (char *)(event + str_loc);
 186
 187         return (u64)(unsigned long)addr;
 188 }
 189
 190 static u64 hist_field_pstring(struct hist_field *hist_field,
 191                               struct tracing_map_elt *elt,
 192                               struct ring_buffer_event *rbe,
 193                               void *event)
 194 {
 195         char **addr = (char **)(event + hist_field->field->offset);
 196
 197         return (u64)(unsigned long)*addr;
 198 }
 199
 200 static u64 hist_field_log2(struct hist_field *hist_field,
 201                            struct tracing_map_elt *elt,
 202                            struct ring_buffer_event *rbe,
 203                            void *event)
 204 {
 205         struct hist_field *operand = hist_field->operands[0];
 206
 207         u64 val = operand->fn(operand, elt, rbe, event);
 208
 209         return (u64) ilog2(roundup_pow_of_two(val));
 210 }
 211
 212 static u64 hist_field_plus(struct hist_field *hist_field,
 213                            struct tracing_map_elt *elt,
 214                            struct ring_buffer_event *rbe,
 215                            void *event)
 216 {
 217         struct hist_field *operand1 = hist_field->operands[0];
 218         struct hist_field *operand2 = hist_field->operands[1];
 219
 220         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 221         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 222
 223         return val1 + val2;
 224 }
 225
 226 static u64 hist_field_minus(struct hist_field *hist_field,
 227                             struct tracing_map_elt *elt,
 228                             struct ring_buffer_event *rbe,
 229                             void *event)
 230 {
 231         struct hist_field *operand1 = hist_field->operands[0];
 232         struct hist_field *operand2 = hist_field->operands[1];
 233
 234         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 235         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 236
 237         return val1 - val2;
 238 }
 239
 240 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 241                                   struct tracing_map_elt *elt,
 242                                   struct ring_buffer_event *rbe,
 243                                   void *event)
 244 {
 245         struct hist_field *operand = hist_field->operands[0];
 246
 247         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 248         u64 val = (u64)-sval;
 249
 250         return val;
 251 }
 252
 253 #define DEFINE_HIST_FIELD_FN(type)                                      \
 254         static u64 hist_field_##type(struct hist_field *hist_field,     \
 255                                      struct tracing_map_elt *elt,       \
 256                                      struct ring_buffer_event *rbe,     \
 257                                      void *event)                       \
 258 {                                                                       \
 259         type *addr = (type *)(event + hist_field->field->offset);       \
 260                                                                         \
 261         return (u64)(unsigned long)*addr;                               \
 262 }
 263
 264 DEFINE_HIST_FIELD_FN(s64);
 265 DEFINE_HIST_FIELD_FN(u64);
 266 DEFINE_HIST_FIELD_FN(s32);
 267 DEFINE_HIST_FIELD_FN(u32);
 268 DEFINE_HIST_FIELD_FN(s16);
 269 DEFINE_HIST_FIELD_FN(u16);
 270 DEFINE_HIST_FIELD_FN(s8);
 271 DEFINE_HIST_FIELD_FN(u8);
 272
 273 #define for_each_hist_field(i, hist_data)       \
 274         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 275
 276 #define for_each_hist_val_field(i, hist_data)   \
 277         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 278
 279 #define for_each_hist_key_field(i, hist_data)   \
 280         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 281
 282 #define HIST_STACKTRACE_DEPTH   16
 283 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 284 #define HIST_STACKTRACE_SKIP    5
 285
 286 #define HITCOUNT_IDX            0
 287 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 288
 289 enum hist_field_flags {
 290         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 291         HIST_FIELD_FL_KEY               = 1 << 1,
 292         HIST_FIELD_FL_STRING            = 1 << 2,
 293         HIST_FIELD_FL_HEX               = 1 << 3,
 294         HIST_FIELD_FL_SYM               = 1 << 4,
 295         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 296         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 297         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 298         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 299         HIST_FIELD_FL_LOG2              = 1 << 9,
 300         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 301         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 302         HIST_FIELD_FL_VAR               = 1 << 12,
 303         HIST_FIELD_FL_EXPR              = 1 << 13,
 304         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 305         HIST_FIELD_FL_CPU               = 1 << 15,
 306         HIST_FIELD_FL_ALIAS             = 1 << 16,
 307 };
 308
 309 struct var_defs {
 310         unsigned int    n_vars;
 311         char            *name[TRACING_MAP_VARS_MAX];
 312         char            *expr[TRACING_MAP_VARS_MAX];
 313 };
 314
 315 struct hist_trigger_attrs {
 316         char            *keys_str;
 317         char            *vals_str;
 318         char            *sort_key_str;
 319         char            *name;
 320         char            *clock;
 321         bool            pause;
 322         bool            cont;
 323         bool            clear;
 324         bool            ts_in_usecs;
 325         unsigned int    map_bits;
 326
 327         char            *assignment_str[TRACING_MAP_VARS_MAX];
 328         unsigned int    n_assignments;
 329
 330         char            *action_str[HIST_ACTIONS_MAX];
 331         unsigned int    n_actions;
 332
 333         struct var_defs var_defs;
 334 };
 335
 336 struct field_var {
 337         struct hist_field       *var;
 338         struct hist_field       *val;
 339 };
 340
 341 struct field_var_hist {
 342         struct hist_trigger_data        *hist_data;
 343         char                            *cmd;
 344 };
 345
 346 struct hist_trigger_data {
 347         struct hist_field               *fields[HIST_FIELDS_MAX];
 348         unsigned int                    n_vals;
 349         unsigned int                    n_keys;
 350         unsigned int                    n_fields;
 351         unsigned int                    n_vars;
 352         unsigned int                    key_size;
 353         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 354         unsigned int                    n_sort_keys;
 355         struct trace_event_file         *event_file;
 356         struct hist_trigger_attrs       *attrs;
 357         struct tracing_map              *map;
 358         bool                            enable_timestamps;
 359         bool                            remove;
 360         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 361         unsigned int                    n_var_refs;
 362
 363         struct action_data              *actions[HIST_ACTIONS_MAX];
 364         unsigned int                    n_actions;
 365
 366         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 367         unsigned int                    n_field_vars;
 368         unsigned int                    n_field_var_str;
 369         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 370         unsigned int                    n_field_var_hists;
 371
 372         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 373         unsigned int                    n_save_vars;
 374         unsigned int                    n_save_var_str;
 375 };
 376
 377 struct action_data;
 378
 379 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 380                              struct tracing_map_elt *elt, void *rec,
 381                              struct ring_buffer_event *rbe, void *key,
 382                              struct action_data *data, u64 *var_ref_vals);
 383
 384 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 385
 386 enum handler_id {
 387         HANDLER_ONMATCH = 1,
 388         HANDLER_ONMAX,
 389         HANDLER_ONCHANGE,
 390 };
 391
 392 enum action_id {
 393         ACTION_SAVE = 1,
 394         ACTION_TRACE,
 395         ACTION_SNAPSHOT,
 396 };
 397
 398 struct action_data {
 399         enum handler_id         handler;
 400         enum action_id          action;
 401         char                    *action_name;
 402         action_fn_t             fn;
 403
 404         unsigned int            n_params;
 405         char                    *params[SYNTH_FIELDS_MAX];
 406
 407         /*
 408          * When a histogram trigger is hit, the values of any
 409          * references to variables, including variables being passed
 410          * as parameters to synthetic events, are collected into a
 411          * var_ref_vals array.  This var_ref_idx array is an array of
 412          * indices into the var_ref_vals array, one for each synthetic
 413          * event param, and is passed to the synthetic event
 414          * invocation.
 415          */
 416         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 417         struct synth_event      *synth_event;
 418         bool                    use_trace_keyword;
 419         char                    *synth_event_name;
 420
 421         union {
 422                 struct {
 423                         char                    *event;
 424                         char                    *event_system;
 425                 } match_data;
 426
 427                 struct {
 428                         /*
 429                          * var_str contains the $-unstripped variable
 430                          * name referenced by var_ref, and used when
 431                          * printing the action.  Because var_ref
 432                          * creation is deferred to create_actions(),
 433                          * we need a per-action way to save it until
 434                          * then, thus var_str.
 435                          */
 436                         char                    *var_str;
 437
 438                         /*
 439                          * var_ref refers to the variable being
 440                          * tracked e.g onmax($var).
 441                          */
 442                         struct hist_field       *var_ref;
 443
 444                         /*
 445                          * track_var contains the 'invisible' tracking
 446                          * variable created to keep the current
 447                          * e.g. max value.
 448                          */
 449                         struct hist_field       *track_var;
 450
 451                         check_track_val_fn_t    check_val;
 452                         action_fn_t             save_data;
 453                 } track_data;
 454         };
 455 };
 456
 457 struct track_data {
 458         u64                             track_val;
 459         bool                            updated;
 460
 461         unsigned int                    key_len;
 462         void                            *key;
 463         struct tracing_map_elt          elt;
 464
 465         struct action_data              *action_data;
 466         struct hist_trigger_data        *hist_data;
 467 };
 468
 469 struct hist_elt_data {
 470         char *comm;
 471         u64 *var_ref_vals;
 472         char *field_var_str[SYNTH_FIELDS_MAX];
 473 };
 474
 475 struct snapshot_context {
 476         struct tracing_map_elt  *elt;
 477         void                    *key;
 478 };
 479
 480 static void track_data_free(struct track_data *track_data)
 481 {
 482         struct hist_elt_data *elt_data;
 483
 484         if (!track_data)
 485                 return;
 486
 487         kfree(track_data->key);
 488
 489         elt_data = track_data->elt.private_data;
 490         if (elt_data) {
 491                 kfree(elt_data->comm);
 492                 kfree(elt_data);
 493         }
 494
 495         kfree(track_data);
 496 }
 497
 498 static struct track_data *track_data_alloc(unsigned int key_len,
 499                                            struct action_data *action_data,
 500                                            struct hist_trigger_data *hist_data)
 501 {
 502         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 503         struct hist_elt_data *elt_data;
 504
 505         if (!data)
 506                 return ERR_PTR(-ENOMEM);
 507
 508         data->key = kzalloc(key_len, GFP_KERNEL);
 509         if (!data->key) {
 510                 track_data_free(data);
 511                 return ERR_PTR(-ENOMEM);
 512         }
 513
 514         data->key_len = key_len;
 515         data->action_data = action_data;
 516         data->hist_data = hist_data;
 517
 518         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 519         if (!elt_data) {
 520                 track_data_free(data);
 521                 return ERR_PTR(-ENOMEM);
 522         }
 523
 524         data->elt.private_data = elt_data;
 525
 526         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 527         if (!elt_data->comm) {
 528                 track_data_free(data);
 529                 return ERR_PTR(-ENOMEM);
 530         }
 531
 532         return data;
 533 }
 534
 535 static char last_cmd[MAX_FILTER_STR_VAL];
 536 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 537
 538 static int errpos(char *str)
 539 {
 540         return err_pos(last_cmd, str);
 541 }
 542
 543 static void last_cmd_set(struct trace_event_file *file, char *str)
 544 {
 545         const char *system = NULL, *name = NULL;
 546         struct trace_event_call *call;
 547
 548         if (!str)
 549                 return;
 550
 551         strcpy(last_cmd, "hist:");
 552         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 553
 554         if (file) {
 555                 call = file->event_call;
 556                 system = call->class->system;
 557                 if (system) {
 558                         name = trace_event_name(call);
 559                         if (!name)
 560                                 system = NULL;
 561                 }
 562         }
 563
 564         if (system)
 565                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 566 }
 567
 568 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 569 {
 570         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 571                         err_type, err_pos);
 572 }
 573
 574 static void hist_err_clear(void)
 575 {
 576         last_cmd[0] = '\0';
 577         last_cmd_loc[0] = '\0';
 578 }
 579
 580 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 581                                     unsigned int *var_ref_idx);
 582
 583 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 584                                unsigned int *var_ref_idx)
 585 {
 586         struct tracepoint *tp = event->tp;
 587
 588         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 589                 struct tracepoint_func *probe_func_ptr;
 590                 synth_probe_func_t probe_func;
 591                 void *__data;
 592
 593                 if (!(cpu_online(raw_smp_processor_id())))
 594                         return;
 595
 596                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 597                 if (probe_func_ptr) {
 598                         do {
 599                                 probe_func = probe_func_ptr->func;
 600                                 __data = probe_func_ptr->data;
 601                                 probe_func(__data, var_ref_vals, var_ref_idx);
 602                         } while ((++probe_func_ptr)->func);
 603                 }
 604         }
 605 }
 606
 607 static void action_trace(struct hist_trigger_data *hist_data,
 608                          struct tracing_map_elt *elt, void *rec,
 609                          struct ring_buffer_event *rbe, void *key,
 610                          struct action_data *data, u64 *var_ref_vals)
 611 {
 612         struct synth_event *event = data->synth_event;
 613
 614         trace_synth(event, var_ref_vals, data->var_ref_idx);
 615 }
 616
 617 struct hist_var_data {
 618         struct list_head list;
 619         struct hist_trigger_data *hist_data;
 620 };
 621
 622 static u64 hist_field_timestamp(struct hist_field *hist_field,
 623                                 struct tracing_map_elt *elt,
 624                                 struct ring_buffer_event *rbe,
 625                                 void *event)
 626 {
 627         struct hist_trigger_data *hist_data = hist_field->hist_data;
 628         struct trace_array *tr = hist_data->event_file->tr;
 629
 630         u64 ts = ring_buffer_event_time_stamp(rbe);
 631
 632         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 633                 ts = ns2usecs(ts);
 634
 635         return ts;
 636 }
 637
 638 static u64 hist_field_cpu(struct hist_field *hist_field,
 639                           struct tracing_map_elt *elt,
 640                           struct ring_buffer_event *rbe,
 641                           void *event)
 642 {
 643         int cpu = smp_processor_id();
 644
 645         return cpu;
 646 }
 647
 648 /**
 649  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 650  * @hist_field: The VAR_REF field to check
 651  * @var_data: The hist trigger that owns the variable
 652  * @var_idx: The trigger variable identifier
 653  *
 654  * Check the given VAR_REF field to see whether or not it references
 655  * the given variable associated with the given trigger.
 656  *
 657  * Return: The VAR_REF field if it does reference the variable, NULL if not
 658  */
 659 static struct hist_field *
 660 check_field_for_var_ref(struct hist_field *hist_field,
 661                         struct hist_trigger_data *var_data,
 662                         unsigned int var_idx)
 663 {
 664         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 665
 666         if (hist_field && hist_field->var.idx == var_idx &&
 667             hist_field->var.hist_data == var_data)
 668                 return hist_field;
 669
 670         return NULL;
 671 }
 672
 673 /**
 674  * find_var_ref - Check if a trigger has a reference to a trigger variable
 675  * @hist_data: The hist trigger that might have a reference to the variable
 676  * @var_data: The hist trigger that owns the variable
 677  * @var_idx: The trigger variable identifier
 678  *
 679  * Check the list of var_refs[] on the first hist trigger to see
 680  * whether any of them are references to the variable on the second
 681  * trigger.
 682  *
 683  * Return: The VAR_REF field referencing the variable if so, NULL if not
 684  */
 685 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 686                                        struct hist_trigger_data *var_data,
 687                                        unsigned int var_idx)
 688 {
 689         struct hist_field *hist_field;
 690         unsigned int i;
 691
 692         for (i = 0; i < hist_data->n_var_refs; i++) {
 693                 hist_field = hist_data->var_refs[i];
 694                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 695                         return hist_field;
 696         }
 697
 698         return NULL;
 699 }
 700
 701 /**
 702  * find_any_var_ref - Check if there is a reference to a given trigger variable
 703  * @hist_data: The hist trigger
 704  * @var_idx: The trigger variable identifier
 705  *
 706  * Check to see whether the given variable is currently referenced by
 707  * any other trigger.
 708  *
 709  * The trigger the variable is defined on is explicitly excluded - the
 710  * assumption being that a self-reference doesn't prevent a trigger
 711  * from being removed.
 712  *
 713  * Return: The VAR_REF field referencing the variable if so, NULL if not
 714  */
 715 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 716                                            unsigned int var_idx)
 717 {
 718         struct trace_array *tr = hist_data->event_file->tr;
 719         struct hist_field *found = NULL;
 720         struct hist_var_data *var_data;
 721
 722         list_for_each_entry(var_data, &tr->hist_vars, list) {
 723                 if (var_data->hist_data == hist_data)
 724                         continue;
 725                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 726                 if (found)
 727                         break;
 728         }
 729
 730         return found;
 731 }
 732
 733 /**
 734  * check_var_refs - Check if there is a reference to any of trigger's variables
 735  * @hist_data: The hist trigger
 736  *
 737  * A trigger can define one or more variables.  If any one of them is
 738  * currently referenced by any other trigger, this function will
 739  * determine that.
 740
 741  * Typically used to determine whether or not a trigger can be removed
 742  * - if there are any references to a trigger's variables, it cannot.
 743  *
 744  * Return: True if there is a reference to any of trigger's variables
 745  */
 746 static bool check_var_refs(struct hist_trigger_data *hist_data)
 747 {
 748         struct hist_field *field;
 749         bool found = false;
 750         int i;
 751
 752         for_each_hist_field(i, hist_data) {
 753                 field = hist_data->fields[i];
 754                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 755                         if (find_any_var_ref(hist_data, field->var.idx)) {
 756                                 found = true;
 757                                 break;
 758                         }
 759                 }
 760         }
 761
 762         return found;
 763 }
 764
 765 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 766 {
 767         struct trace_array *tr = hist_data->event_file->tr;
 768         struct hist_var_data *var_data, *found = NULL;
 769
 770         list_for_each_entry(var_data, &tr->hist_vars, list) {
 771                 if (var_data->hist_data == hist_data) {
 772                         found = var_data;
 773                         break;
 774                 }
 775         }
 776
 777         return found;
 778 }
 779
 780 static bool field_has_hist_vars(struct hist_field *hist_field,
 781                                 unsigned int level)
 782 {
 783         int i;
 784
 785         if (level > 3)
 786                 return false;
 787
 788         if (!hist_field)
 789                 return false;
 790
 791         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 792             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 793                 return true;
 794
 795         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 796                 struct hist_field *operand;
 797
 798                 operand = hist_field->operands[i];
 799                 if (field_has_hist_vars(operand, level + 1))
 800                         return true;
 801         }
 802
 803         return false;
 804 }
 805
 806 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 807 {
 808         struct hist_field *hist_field;
 809         int i;
 810
 811         for_each_hist_field(i, hist_data) {
 812                 hist_field = hist_data->fields[i];
 813                 if (field_has_hist_vars(hist_field, 0))
 814                         return true;
 815         }
 816
 817         return false;
 818 }
 819
 820 static int save_hist_vars(struct hist_trigger_data *hist_data)
 821 {
 822         struct trace_array *tr = hist_data->event_file->tr;
 823         struct hist_var_data *var_data;
 824
 825         var_data = find_hist_vars(hist_data);
 826         if (var_data)
 827                 return 0;
 828
 829         if (tracing_check_open_get_tr(tr))
 830                 return -ENODEV;
 831
 832         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 833         if (!var_data) {
 834                 trace_array_put(tr);
 835                 return -ENOMEM;
 836         }
 837
 838         var_data->hist_data = hist_data;
 839         list_add(&var_data->list, &tr->hist_vars);
 840
 841         return 0;
 842 }
 843
 844 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 845 {
 846         struct trace_array *tr = hist_data->event_file->tr;
 847         struct hist_var_data *var_data;
 848
 849         var_data = find_hist_vars(hist_data);
 850         if (!var_data)
 851                 return;
 852
 853         if (WARN_ON(check_var_refs(hist_data)))
 854                 return;
 855
 856         list_del(&var_data->list);
 857
 858         kfree(var_data);
 859
 860         trace_array_put(tr);
 861 }
 862
 863 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 864                                          const char *var_name)
 865 {
 866         struct hist_field *hist_field, *found = NULL;
 867         int i;
 868
 869         for_each_hist_field(i, hist_data) {
 870                 hist_field = hist_data->fields[i];
 871                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 872                     strcmp(hist_field->var.name, var_name) == 0) {
 873                         found = hist_field;
 874                         break;
 875                 }
 876         }
 877
 878         return found;
 879 }
 880
 881 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 882                                    struct trace_event_file *file,
 883                                    const char *var_name)
 884 {
 885         struct hist_trigger_data *test_data;
 886         struct event_trigger_data *test;
 887         struct hist_field *hist_field;
 888
 889         lockdep_assert_held(&event_mutex);
 890
 891         hist_field = find_var_field(hist_data, var_name);
 892         if (hist_field)
 893                 return hist_field;
 894
 895         list_for_each_entry(test, &file->triggers, list) {
 896                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 897                         test_data = test->private_data;
 898                         hist_field = find_var_field(test_data, var_name);
 899                         if (hist_field)
 900                                 return hist_field;
 901                 }
 902         }
 903
 904         return NULL;
 905 }
 906
 907 static struct trace_event_file *find_var_file(struct trace_array *tr,
 908                                               char *system,
 909                                               char *event_name,
 910                                               char *var_name)
 911 {
 912         struct hist_trigger_data *var_hist_data;
 913         struct hist_var_data *var_data;
 914         struct trace_event_file *file, *found = NULL;
 915
 916         if (system)
 917                 return find_event_file(tr, system, event_name);
 918
 919         list_for_each_entry(var_data, &tr->hist_vars, list) {
 920                 var_hist_data = var_data->hist_data;
 921                 file = var_hist_data->event_file;
 922                 if (file == found)
 923                         continue;
 924
 925                 if (find_var_field(var_hist_data, var_name)) {
 926                         if (found) {
 927                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 928                                 return NULL;
 929                         }
 930
 931                         found = file;
 932                 }
 933         }
 934
 935         return found;
 936 }
 937
 938 static struct hist_field *find_file_var(struct trace_event_file *file,
 939                                         const char *var_name)
 940 {
 941         struct hist_trigger_data *test_data;
 942         struct event_trigger_data *test;
 943         struct hist_field *hist_field;
 944
 945         lockdep_assert_held(&event_mutex);
 946
 947         list_for_each_entry(test, &file->triggers, list) {
 948                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 949                         test_data = test->private_data;
 950                         hist_field = find_var_field(test_data, var_name);
 951                         if (hist_field)
 952                                 return hist_field;
 953                 }
 954         }
 955
 956         return NULL;
 957 }
 958
 959 static struct hist_field *
 960 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 961 {
 962         struct trace_array *tr = hist_data->event_file->tr;
 963         struct hist_field *hist_field, *found = NULL;
 964         struct trace_event_file *file;
 965         unsigned int i;
 966
 967         for (i = 0; i < hist_data->n_actions; i++) {
 968                 struct action_data *data = hist_data->actions[i];
 969
 970                 if (data->handler == HANDLER_ONMATCH) {
 971                         char *system = data->match_data.event_system;
 972                         char *event_name = data->match_data.event;
 973
 974                         file = find_var_file(tr, system, event_name, var_name);
 975                         if (!file)
 976                                 continue;
 977                         hist_field = find_file_var(file, var_name);
 978                         if (hist_field) {
 979                                 if (found) {
 980                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
 981                                                  errpos(var_name));
 982                                         return ERR_PTR(-EINVAL);
 983                                 }
 984
 985                                 found = hist_field;
 986                         }
 987                 }
 988         }
 989         return found;
 990 }
 991
 992 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
 993                                          char *system,
 994                                          char *event_name,
 995                                          char *var_name)
 996 {
 997         struct trace_array *tr = hist_data->event_file->tr;
 998         struct hist_field *hist_field = NULL;
 999         struct trace_event_file *file;
1000
1001         if (!system || !event_name) {
1002                 hist_field = find_match_var(hist_data, var_name);
1003                 if (IS_ERR(hist_field))
1004                         return NULL;
1005                 if (hist_field)
1006                         return hist_field;
1007         }
1008
1009         file = find_var_file(tr, system, event_name, var_name);
1010         if (!file)
1011                 return NULL;
1012
1013         hist_field = find_file_var(file, var_name);
1014
1015         return hist_field;
1016 }
1017
1018 static u64 hist_field_var_ref(struct hist_field *hist_field,
1019                               struct tracing_map_elt *elt,
1020                               struct ring_buffer_event *rbe,
1021                               void *event)
1022 {
1023         struct hist_elt_data *elt_data;
1024         u64 var_val = 0;
1025
1026         if (WARN_ON_ONCE(!elt))
1027                 return var_val;
1028
1029         elt_data = elt->private_data;
1030         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1031
1032         return var_val;
1033 }
1034
1035 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1036                              u64 *var_ref_vals, bool self)
1037 {
1038         struct hist_trigger_data *var_data;
1039         struct tracing_map_elt *var_elt;
1040         struct hist_field *hist_field;
1041         unsigned int i, var_idx;
1042         bool resolved = true;
1043         u64 var_val = 0;
1044
1045         for (i = 0; i < hist_data->n_var_refs; i++) {
1046                 hist_field = hist_data->var_refs[i];
1047                 var_idx = hist_field->var.idx;
1048                 var_data = hist_field->var.hist_data;
1049
1050                 if (var_data == NULL) {
1051                         resolved = false;
1052                         break;
1053                 }
1054
1055                 if ((self && var_data != hist_data) ||
1056                     (!self && var_data == hist_data))
1057                         continue;
1058
1059                 var_elt = tracing_map_lookup(var_data->map, key);
1060                 if (!var_elt) {
1061                         resolved = false;
1062                         break;
1063                 }
1064
1065                 if (!tracing_map_var_set(var_elt, var_idx)) {
1066                         resolved = false;
1067                         break;
1068                 }
1069
1070                 if (self || !hist_field->read_once)
1071                         var_val = tracing_map_read_var(var_elt, var_idx);
1072                 else
1073                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1074
1075                 var_ref_vals[i] = var_val;
1076         }
1077
1078         return resolved;
1079 }
1080
1081 static const char *hist_field_name(struct hist_field *field,
1082                                    unsigned int level)
1083 {
1084         const char *field_name = "";
1085
1086         if (level > 1)
1087                 return field_name;
1088
1089         if (field->field)
1090                 field_name = field->field->name;
1091         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1092                  field->flags & HIST_FIELD_FL_ALIAS)
1093                 field_name = hist_field_name(field->operands[0], ++level);
1094         else if (field->flags & HIST_FIELD_FL_CPU)
1095                 field_name = "cpu";
1096         else if (field->flags & HIST_FIELD_FL_EXPR ||
1097                  field->flags & HIST_FIELD_FL_VAR_REF) {
1098                 if (field->system) {
1099                         static char full_name[MAX_FILTER_STR_VAL];
1100
1101                         strcat(full_name, field->system);
1102                         strcat(full_name, ".");
1103                         strcat(full_name, field->event_name);
1104                         strcat(full_name, ".");
1105                         strcat(full_name, field->name);
1106                         field_name = full_name;
1107                 } else
1108                         field_name = field->name;
1109         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1110                 field_name = "common_timestamp";
1111
1112         if (field_name == NULL)
1113                 field_name = "";
1114
1115         return field_name;
1116 }
1117
1118 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1119 {
1120         hist_field_fn_t fn = NULL;
1121
1122         switch (field_size) {
1123         case 8:
1124                 if (field_is_signed)
1125                         fn = hist_field_s64;
1126                 else
1127                         fn = hist_field_u64;
1128                 break;
1129         case 4:
1130                 if (field_is_signed)
1131                         fn = hist_field_s32;
1132                 else
1133                         fn = hist_field_u32;
1134                 break;
1135         case 2:
1136                 if (field_is_signed)
1137                         fn = hist_field_s16;
1138                 else
1139                         fn = hist_field_u16;
1140                 break;
1141         case 1:
1142                 if (field_is_signed)
1143                         fn = hist_field_s8;
1144                 else
1145                         fn = hist_field_u8;
1146                 break;
1147         }
1148
1149         return fn;
1150 }
1151
1152 static int parse_map_size(char *str)
1153 {
1154         unsigned long size, map_bits;
1155         int ret;
1156
1157         ret = kstrtoul(str, 0, &size);
1158         if (ret)
1159                 goto out;
1160
1161         map_bits = ilog2(roundup_pow_of_two(size));
1162         if (map_bits < TRACING_MAP_BITS_MIN ||
1163             map_bits > TRACING_MAP_BITS_MAX)
1164                 ret = -EINVAL;
1165         else
1166                 ret = map_bits;
1167  out:
1168         return ret;
1169 }
1170
1171 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1172 {
1173         unsigned int i;
1174
1175         if (!attrs)
1176                 return;
1177
1178         for (i = 0; i < attrs->n_assignments; i++)
1179                 kfree(attrs->assignment_str[i]);
1180
1181         for (i = 0; i < attrs->n_actions; i++)
1182                 kfree(attrs->action_str[i]);
1183
1184         kfree(attrs->name);
1185         kfree(attrs->sort_key_str);
1186         kfree(attrs->keys_str);
1187         kfree(attrs->vals_str);
1188         kfree(attrs->clock);
1189         kfree(attrs);
1190 }
1191
1192 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1193 {
1194         int ret = -EINVAL;
1195
1196         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1197                 return ret;
1198
1199         if ((str_has_prefix(str, "onmatch(")) ||
1200             (str_has_prefix(str, "onmax(")) ||
1201             (str_has_prefix(str, "onchange("))) {
1202                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1203                 if (!attrs->action_str[attrs->n_actions]) {
1204                         ret = -ENOMEM;
1205                         return ret;
1206                 }
1207                 attrs->n_actions++;
1208                 ret = 0;
1209         }
1210         return ret;
1211 }
1212
1213 static int parse_assignment(struct trace_array *tr,
1214                             char *str, struct hist_trigger_attrs *attrs)
1215 {
1216         int len, ret = 0;
1217
1218         if ((len = str_has_prefix(str, "key=")) ||
1219             (len = str_has_prefix(str, "keys="))) {
1220                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1221                 if (!attrs->keys_str) {
1222                         ret = -ENOMEM;
1223                         goto out;
1224                 }
1225         } else if ((len = str_has_prefix(str, "val=")) ||
1226                    (len = str_has_prefix(str, "vals=")) ||
1227                    (len = str_has_prefix(str, "values="))) {
1228                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1229                 if (!attrs->vals_str) {
1230                         ret = -ENOMEM;
1231                         goto out;
1232                 }
1233         } else if ((len = str_has_prefix(str, "sort="))) {
1234                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1235                 if (!attrs->sort_key_str) {
1236                         ret = -ENOMEM;
1237                         goto out;
1238                 }
1239         } else if (str_has_prefix(str, "name=")) {
1240                 attrs->name = kstrdup(str, GFP_KERNEL);
1241                 if (!attrs->name) {
1242                         ret = -ENOMEM;
1243                         goto out;
1244                 }
1245         } else if ((len = str_has_prefix(str, "clock="))) {
1246                 str += len;
1247
1248                 str = strstrip(str);
1249                 attrs->clock = kstrdup(str, GFP_KERNEL);
1250                 if (!attrs->clock) {
1251                         ret = -ENOMEM;
1252                         goto out;
1253                 }
1254         } else if ((len = str_has_prefix(str, "size="))) {
1255                 int map_bits = parse_map_size(str + len);
1256
1257                 if (map_bits < 0) {
1258                         ret = map_bits;
1259                         goto out;
1260                 }
1261                 attrs->map_bits = map_bits;
1262         } else {
1263                 char *assignment;
1264
1265                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1266                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1267                         ret = -EINVAL;
1268                         goto out;
1269                 }
1270
1271                 assignment = kstrdup(str, GFP_KERNEL);
1272                 if (!assignment) {
1273                         ret = -ENOMEM;
1274                         goto out;
1275                 }
1276
1277                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1278         }
1279  out:
1280         return ret;
1281 }
1282
1283 static struct hist_trigger_attrs *
1284 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1285 {
1286         struct hist_trigger_attrs *attrs;
1287         int ret = 0;
1288
1289         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1290         if (!attrs)
1291                 return ERR_PTR(-ENOMEM);
1292
1293         while (trigger_str) {
1294                 char *str = strsep(&trigger_str, ":");
1295                 char *rhs;
1296
1297                 rhs = strchr(str, '=');
1298                 if (rhs) {
1299                         if (!strlen(++rhs)) {
1300                                 ret = -EINVAL;
1301                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1302                                 goto free;
1303                         }
1304                         ret = parse_assignment(tr, str, attrs);
1305                         if (ret)
1306                                 goto free;
1307                 } else if (strcmp(str, "pause") == 0)
1308                         attrs->pause = true;
1309                 else if ((strcmp(str, "cont") == 0) ||
1310                          (strcmp(str, "continue") == 0))
1311                         attrs->cont = true;
1312                 else if (strcmp(str, "clear") == 0)
1313                         attrs->clear = true;
1314                 else {
1315                         ret = parse_action(str, attrs);
1316                         if (ret)
1317                                 goto free;
1318                 }
1319         }
1320
1321         if (!attrs->keys_str) {
1322                 ret = -EINVAL;
1323                 goto free;
1324         }
1325
1326         if (!attrs->clock) {
1327                 attrs->clock = kstrdup("global", GFP_KERNEL);
1328                 if (!attrs->clock) {
1329                         ret = -ENOMEM;
1330                         goto free;
1331                 }
1332         }
1333
1334         return attrs;
1335  free:
1336         destroy_hist_trigger_attrs(attrs);
1337
1338         return ERR_PTR(ret);
1339 }
1340
1341 static inline void save_comm(char *comm, struct task_struct *task)
1342 {
1343         if (!task->pid) {
1344                 strcpy(comm, "<idle>");
1345                 return;
1346         }
1347
1348         if (WARN_ON_ONCE(task->pid < 0)) {
1349                 strcpy(comm, "<XXX>");
1350                 return;
1351         }
1352
1353         strncpy(comm, task->comm, TASK_COMM_LEN);
1354 }
1355
1356 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1357 {
1358         unsigned int i;
1359
1360         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1361                 kfree(elt_data->field_var_str[i]);
1362
1363         kfree(elt_data->comm);
1364         kfree(elt_data);
1365 }
1366
1367 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1368 {
1369         struct hist_elt_data *elt_data = elt->private_data;
1370
1371         hist_elt_data_free(elt_data);
1372 }
1373
1374 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1375 {
1376         struct hist_trigger_data *hist_data = elt->map->private_data;
1377         unsigned int size = TASK_COMM_LEN;
1378         struct hist_elt_data *elt_data;
1379         struct hist_field *key_field;
1380         unsigned int i, n_str;
1381
1382         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1383         if (!elt_data)
1384                 return -ENOMEM;
1385
1386         for_each_hist_key_field(i, hist_data) {
1387                 key_field = hist_data->fields[i];
1388
1389                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1390                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1391                         if (!elt_data->comm) {
1392                                 kfree(elt_data);
1393                                 return -ENOMEM;
1394                         }
1395                         break;
1396                 }
1397         }
1398
1399         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
1400
1401         size = STR_VAR_LEN_MAX;
1402
1403         for (i = 0; i < n_str; i++) {
1404                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1405                 if (!elt_data->field_var_str[i]) {
1406                         hist_elt_data_free(elt_data);
1407                         return -ENOMEM;
1408                 }
1409         }
1410
1411         elt->private_data = elt_data;
1412
1413         return 0;
1414 }
1415
1416 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1417 {
1418         struct hist_elt_data *elt_data = elt->private_data;
1419
1420         if (elt_data->comm)
1421                 save_comm(elt_data->comm, current);
1422 }
1423
1424 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1425         .elt_alloc      = hist_trigger_elt_data_alloc,
1426         .elt_free       = hist_trigger_elt_data_free,
1427         .elt_init       = hist_trigger_elt_data_init,
1428 };
1429
1430 static const char *get_hist_field_flags(struct hist_field *hist_field)
1431 {
1432         const char *flags_str = NULL;
1433
1434         if (hist_field->flags & HIST_FIELD_FL_HEX)
1435                 flags_str = "hex";
1436         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1437                 flags_str = "sym";
1438         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1439                 flags_str = "sym-offset";
1440         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1441                 flags_str = "execname";
1442         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1443                 flags_str = "syscall";
1444         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1445                 flags_str = "log2";
1446         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1447                 flags_str = "usecs";
1448
1449         return flags_str;
1450 }
1451
1452 static void expr_field_str(struct hist_field *field, char *expr)
1453 {
1454         if (field->flags & HIST_FIELD_FL_VAR_REF)
1455                 strcat(expr, "$");
1456
1457         strcat(expr, hist_field_name(field, 0));
1458
1459         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1460                 const char *flags_str = get_hist_field_flags(field);
1461
1462                 if (flags_str) {
1463                         strcat(expr, ".");
1464                         strcat(expr, flags_str);
1465                 }
1466         }
1467 }
1468
1469 static char *expr_str(struct hist_field *field, unsigned int level)
1470 {
1471         char *expr;
1472
1473         if (level > 1)
1474                 return NULL;
1475
1476         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1477         if (!expr)
1478                 return NULL;
1479
1480         if (!field->operands[0]) {
1481                 expr_field_str(field, expr);
1482                 return expr;
1483         }
1484
1485         if (field->operator == FIELD_OP_UNARY_MINUS) {
1486                 char *subexpr;
1487
1488                 strcat(expr, "-(");
1489                 subexpr = expr_str(field->operands[0], ++level);
1490                 if (!subexpr) {
1491                         kfree(expr);
1492                         return NULL;
1493                 }
1494                 strcat(expr, subexpr);
1495                 strcat(expr, ")");
1496
1497                 kfree(subexpr);
1498
1499                 return expr;
1500         }
1501
1502         expr_field_str(field->operands[0], expr);
1503
1504         switch (field->operator) {
1505         case FIELD_OP_MINUS:
1506                 strcat(expr, "-");
1507                 break;
1508         case FIELD_OP_PLUS:
1509                 strcat(expr, "+");
1510                 break;
1511         default:
1512                 kfree(expr);
1513                 return NULL;
1514         }
1515
1516         expr_field_str(field->operands[1], expr);
1517
1518         return expr;
1519 }
1520
1521 static int contains_operator(char *str)
1522 {
1523         enum field_op_id field_op = FIELD_OP_NONE;
1524         char *op;
1525
1526         op = strpbrk(str, "+-");
1527         if (!op)
1528                 return FIELD_OP_NONE;
1529
1530         switch (*op) {
1531         case '-':
1532                 if (*str == '-')
1533                         field_op = FIELD_OP_UNARY_MINUS;
1534                 else
1535                         field_op = FIELD_OP_MINUS;
1536                 break;
1537         case '+':
1538                 field_op = FIELD_OP_PLUS;
1539                 break;
1540         default:
1541                 break;
1542         }
1543
1544         return field_op;
1545 }
1546
1547 static void get_hist_field(struct hist_field *hist_field)
1548 {
1549         hist_field->ref++;
1550 }
1551
1552 static void __destroy_hist_field(struct hist_field *hist_field)
1553 {
1554         if (--hist_field->ref > 1)
1555                 return;
1556
1557         kfree(hist_field->var.name);
1558         kfree(hist_field->name);
1559         kfree(hist_field->type);
1560
1561         kfree(hist_field->system);
1562         kfree(hist_field->event_name);
1563
1564         kfree(hist_field);
1565 }
1566
1567 static void destroy_hist_field(struct hist_field *hist_field,
1568                                unsigned int level)
1569 {
1570         unsigned int i;
1571
1572         if (level > 3)
1573                 return;
1574
1575         if (!hist_field)
1576                 return;
1577
1578         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1579                 return; /* var refs will be destroyed separately */
1580
1581         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1582                 destroy_hist_field(hist_field->operands[i], level + 1);
1583
1584         __destroy_hist_field(hist_field);
1585 }
1586
1587 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1588                                             struct ftrace_event_field *field,
1589                                             unsigned long flags,
1590                                             char *var_name)
1591 {
1592         struct hist_field *hist_field;
1593
1594         if (field && is_function_field(field))
1595                 return NULL;
1596
1597         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1598         if (!hist_field)
1599                 return NULL;
1600
1601         hist_field->ref = 1;
1602
1603         hist_field->hist_data = hist_data;
1604
1605         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1606                 goto out; /* caller will populate */
1607
1608         if (flags & HIST_FIELD_FL_VAR_REF) {
1609                 hist_field->fn = hist_field_var_ref;
1610                 goto out;
1611         }
1612
1613         if (flags & HIST_FIELD_FL_HITCOUNT) {
1614                 hist_field->fn = hist_field_counter;
1615                 hist_field->size = sizeof(u64);
1616                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1617                 if (!hist_field->type)
1618                         goto free;
1619                 goto out;
1620         }
1621
1622         if (flags & HIST_FIELD_FL_STACKTRACE) {
1623                 hist_field->fn = hist_field_none;
1624                 goto out;
1625         }
1626
1627         if (flags & HIST_FIELD_FL_LOG2) {
1628                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1629                 hist_field->fn = hist_field_log2;
1630                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1631                 hist_field->size = hist_field->operands[0]->size;
1632                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1633                 if (!hist_field->type)
1634                         goto free;
1635                 goto out;
1636         }
1637
1638         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1639                 hist_field->fn = hist_field_timestamp;
1640                 hist_field->size = sizeof(u64);
1641                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1642                 if (!hist_field->type)
1643                         goto free;
1644                 goto out;
1645         }
1646
1647         if (flags & HIST_FIELD_FL_CPU) {
1648                 hist_field->fn = hist_field_cpu;
1649                 hist_field->size = sizeof(int);
1650                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1651                 if (!hist_field->type)
1652                         goto free;
1653                 goto out;
1654         }
1655
1656         if (WARN_ON_ONCE(!field))
1657                 goto out;
1658
1659         if (is_string_field(field)) {
1660                 flags |= HIST_FIELD_FL_STRING;
1661
1662                 hist_field->size = MAX_FILTER_STR_VAL;
1663                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1664                 if (!hist_field->type)
1665                         goto free;
1666
1667                 if (field->filter_type == FILTER_STATIC_STRING)
1668                         hist_field->fn = hist_field_string;
1669                 else if (field->filter_type == FILTER_DYN_STRING)
1670                         hist_field->fn = hist_field_dynstring;
1671                 else
1672                         hist_field->fn = hist_field_pstring;
1673         } else {
1674                 hist_field->size = field->size;
1675                 hist_field->is_signed = field->is_signed;
1676                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1677                 if (!hist_field->type)
1678                         goto free;
1679
1680                 hist_field->fn = select_value_fn(field->size,
1681                                                  field->is_signed);
1682                 if (!hist_field->fn) {
1683                         destroy_hist_field(hist_field, 0);
1684                         return NULL;
1685                 }
1686         }
1687  out:
1688         hist_field->field = field;
1689         hist_field->flags = flags;
1690
1691         if (var_name) {
1692                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1693                 if (!hist_field->var.name)
1694                         goto free;
1695         }
1696
1697         return hist_field;
1698  free:
1699         destroy_hist_field(hist_field, 0);
1700         return NULL;
1701 }
1702
1703 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1704 {
1705         unsigned int i;
1706
1707         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1708                 if (hist_data->fields[i]) {
1709                         destroy_hist_field(hist_data->fields[i], 0);
1710                         hist_data->fields[i] = NULL;
1711                 }
1712         }
1713
1714         for (i = 0; i < hist_data->n_var_refs; i++) {
1715                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1716                 __destroy_hist_field(hist_data->var_refs[i]);
1717                 hist_data->var_refs[i] = NULL;
1718         }
1719 }
1720
1721 static int init_var_ref(struct hist_field *ref_field,
1722                         struct hist_field *var_field,
1723                         char *system, char *event_name)
1724 {
1725         int err = 0;
1726
1727         ref_field->var.idx = var_field->var.idx;
1728         ref_field->var.hist_data = var_field->hist_data;
1729         ref_field->size = var_field->size;
1730         ref_field->is_signed = var_field->is_signed;
1731         ref_field->flags |= var_field->flags &
1732                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1733
1734         if (system) {
1735                 ref_field->system = kstrdup(system, GFP_KERNEL);
1736                 if (!ref_field->system)
1737                         return -ENOMEM;
1738         }
1739
1740         if (event_name) {
1741                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1742                 if (!ref_field->event_name) {
1743                         err = -ENOMEM;
1744                         goto free;
1745                 }
1746         }
1747
1748         if (var_field->var.name) {
1749                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1750                 if (!ref_field->name) {
1751                         err = -ENOMEM;
1752                         goto free;
1753                 }
1754         } else if (var_field->name) {
1755                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1756                 if (!ref_field->name) {
1757                         err = -ENOMEM;
1758                         goto free;
1759                 }
1760         }
1761
1762         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1763         if (!ref_field->type) {
1764                 err = -ENOMEM;
1765                 goto free;
1766         }
1767  out:
1768         return err;
1769  free:
1770         kfree(ref_field->system);
1771         kfree(ref_field->event_name);
1772         kfree(ref_field->name);
1773
1774         goto out;
1775 }
1776
1777 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1778                             struct hist_field *var_field)
1779 {
1780         struct hist_field *ref_field;
1781         int i;
1782
1783         for (i = 0; i < hist_data->n_var_refs; i++) {
1784                 ref_field = hist_data->var_refs[i];
1785                 if (ref_field->var.idx == var_field->var.idx &&
1786                     ref_field->var.hist_data == var_field->hist_data)
1787                         return i;
1788         }
1789
1790         return -ENOENT;
1791 }
1792
1793 /**
1794  * create_var_ref - Create a variable reference and attach it to trigger
1795  * @hist_data: The trigger that will be referencing the variable
1796  * @var_field: The VAR field to create a reference to
1797  * @system: The optional system string
1798  * @event_name: The optional event_name string
1799  *
1800  * Given a variable hist_field, create a VAR_REF hist_field that
1801  * represents a reference to it.
1802  *
1803  * This function also adds the reference to the trigger that
1804  * now references the variable.
1805  *
1806  * Return: The VAR_REF field if successful, NULL if not
1807  */
1808 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1809                                          struct hist_field *var_field,
1810                                          char *system, char *event_name)
1811 {
1812         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1813         struct hist_field *ref_field;
1814         int i;
1815
1816         /* Check if the variable already exists */
1817         for (i = 0; i < hist_data->n_var_refs; i++) {
1818                 ref_field = hist_data->var_refs[i];
1819                 if (ref_field->var.idx == var_field->var.idx &&
1820                     ref_field->var.hist_data == var_field->hist_data) {
1821                         get_hist_field(ref_field);
1822                         return ref_field;
1823                 }
1824         }
1825
1826         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1827         if (ref_field) {
1828                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1829                         destroy_hist_field(ref_field, 0);
1830                         return NULL;
1831                 }
1832
1833                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1834                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1835         }
1836
1837         return ref_field;
1838 }
1839
1840 static bool is_var_ref(char *var_name)
1841 {
1842         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1843                 return false;
1844
1845         return true;
1846 }
1847
1848 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1849                                  char *var_name)
1850 {
1851         char *name, *field;
1852         unsigned int i;
1853
1854         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1855                 name = hist_data->attrs->var_defs.name[i];
1856
1857                 if (strcmp(var_name, name) == 0) {
1858                         field = hist_data->attrs->var_defs.expr[i];
1859                         if (contains_operator(field) || is_var_ref(field))
1860                                 continue;
1861                         return field;
1862                 }
1863         }
1864
1865         return NULL;
1866 }
1867
1868 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1869                                  char *system, char *event_name,
1870                                  char *var_name)
1871 {
1872         struct trace_event_call *call;
1873
1874         if (system && event_name) {
1875                 call = hist_data->event_file->event_call;
1876
1877                 if (strcmp(system, call->class->system) != 0)
1878                         return NULL;
1879
1880                 if (strcmp(event_name, trace_event_name(call)) != 0)
1881                         return NULL;
1882         }
1883
1884         if (!!system != !!event_name)
1885                 return NULL;
1886
1887         if (!is_var_ref(var_name))
1888                 return NULL;
1889
1890         var_name++;
1891
1892         return field_name_from_var(hist_data, var_name);
1893 }
1894
1895 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1896                                         char *system, char *event_name,
1897                                         char *var_name)
1898 {
1899         struct hist_field *var_field = NULL, *ref_field = NULL;
1900         struct trace_array *tr = hist_data->event_file->tr;
1901
1902         if (!is_var_ref(var_name))
1903                 return NULL;
1904
1905         var_name++;
1906
1907         var_field = find_event_var(hist_data, system, event_name, var_name);
1908         if (var_field)
1909                 ref_field = create_var_ref(hist_data, var_field,
1910                                            system, event_name);
1911
1912         if (!ref_field)
1913                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1914
1915         return ref_field;
1916 }
1917
1918 static struct ftrace_event_field *
1919 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1920             char *field_str, unsigned long *flags)
1921 {
1922         struct ftrace_event_field *field = NULL;
1923         char *field_name, *modifier, *str;
1924         struct trace_array *tr = file->tr;
1925
1926         modifier = str = kstrdup(field_str, GFP_KERNEL);
1927         if (!modifier)
1928                 return ERR_PTR(-ENOMEM);
1929
1930         field_name = strsep(&modifier, ".");
1931         if (modifier) {
1932                 if (strcmp(modifier, "hex") == 0)
1933                         *flags |= HIST_FIELD_FL_HEX;
1934                 else if (strcmp(modifier, "sym") == 0)
1935                         *flags |= HIST_FIELD_FL_SYM;
1936                 else if (strcmp(modifier, "sym-offset") == 0)
1937                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1938                 else if ((strcmp(modifier, "execname") == 0) &&
1939                          (strcmp(field_name, "common_pid") == 0))
1940                         *flags |= HIST_FIELD_FL_EXECNAME;
1941                 else if (strcmp(modifier, "syscall") == 0)
1942                         *flags |= HIST_FIELD_FL_SYSCALL;
1943                 else if (strcmp(modifier, "log2") == 0)
1944                         *flags |= HIST_FIELD_FL_LOG2;
1945                 else if (strcmp(modifier, "usecs") == 0)
1946                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1947                 else {
1948                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1949                         field = ERR_PTR(-EINVAL);
1950                         goto out;
1951                 }
1952         }
1953
1954         if (strcmp(field_name, "common_timestamp") == 0) {
1955                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1956                 hist_data->enable_timestamps = true;
1957                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1958                         hist_data->attrs->ts_in_usecs = true;
1959         } else if (strcmp(field_name, "cpu") == 0)
1960                 *flags |= HIST_FIELD_FL_CPU;
1961         else {
1962                 field = trace_find_event_field(file->event_call, field_name);
1963                 if (!field || !field->size) {
1964                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1965                         field = ERR_PTR(-EINVAL);
1966                         goto out;
1967                 }
1968         }
1969  out:
1970         kfree(str);
1971
1972         return field;
1973 }
1974
1975 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
1976                                        struct hist_field *var_ref,
1977                                        char *var_name)
1978 {
1979         struct hist_field *alias = NULL;
1980         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
1981
1982         alias = create_hist_field(hist_data, NULL, flags, var_name);
1983         if (!alias)
1984                 return NULL;
1985
1986         alias->fn = var_ref->fn;
1987         alias->operands[0] = var_ref;
1988
1989         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
1990                 destroy_hist_field(alias, 0);
1991                 return NULL;
1992         }
1993
1994         alias->var_ref_idx = var_ref->var_ref_idx;
1995
1996         return alias;
1997 }
1998
1999 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2000                                      struct trace_event_file *file, char *str,
2001                                      unsigned long *flags, char *var_name)
2002 {
2003         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2004         struct ftrace_event_field *field = NULL;
2005         struct hist_field *hist_field = NULL;
2006         int ret = 0;
2007
2008         s = strchr(str, '.');
2009         if (s) {
2010                 s = strchr(++s, '.');
2011                 if (s) {
2012                         ref_system = strsep(&str, ".");
2013                         if (!str) {
2014                                 ret = -EINVAL;
2015                                 goto out;
2016                         }
2017                         ref_event = strsep(&str, ".");
2018                         if (!str) {
2019                                 ret = -EINVAL;
2020                                 goto out;
2021                         }
2022                         ref_var = str;
2023                 }
2024         }
2025
2026         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2027         if (!s) {
2028                 hist_field = parse_var_ref(hist_data, ref_system,
2029                                            ref_event, ref_var);
2030                 if (hist_field) {
2031                         if (var_name) {
2032                                 hist_field = create_alias(hist_data, hist_field, var_name);
2033                                 if (!hist_field) {
2034                                         ret = -ENOMEM;
2035                                         goto out;
2036                                 }
2037                         }
2038                         return hist_field;
2039                 }
2040         } else
2041                 str = s;
2042
2043         field = parse_field(hist_data, file, str, flags);
2044         if (IS_ERR(field)) {
2045                 ret = PTR_ERR(field);
2046                 goto out;
2047         }
2048
2049         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2050         if (!hist_field) {
2051                 ret = -ENOMEM;
2052                 goto out;
2053         }
2054
2055         return hist_field;
2056  out:
2057         return ERR_PTR(ret);
2058 }
2059
2060 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2061                                      struct trace_event_file *file,
2062                                      char *str, unsigned long flags,
2063                                      char *var_name, unsigned int level);
2064
2065 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2066                                       struct trace_event_file *file,
2067                                       char *str, unsigned long flags,
2068                                       char *var_name, unsigned int level)
2069 {
2070         struct hist_field *operand1, *expr = NULL;
2071         unsigned long operand_flags;
2072         int ret = 0;
2073         char *s;
2074
2075         /* we support only -(xxx) i.e. explicit parens required */
2076
2077         if (level > 3) {
2078                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2079                 ret = -EINVAL;
2080                 goto free;
2081         }
2082
2083         str++; /* skip leading '-' */
2084
2085         s = strchr(str, '(');
2086         if (s)
2087                 str++;
2088         else {
2089                 ret = -EINVAL;
2090                 goto free;
2091         }
2092
2093         s = strrchr(str, ')');
2094         if (s)
2095                 *s = '\0';
2096         else {
2097                 ret = -EINVAL; /* no closing ')' */
2098                 goto free;
2099         }
2100
2101         flags |= HIST_FIELD_FL_EXPR;
2102         expr = create_hist_field(hist_data, NULL, flags, var_name);
2103         if (!expr) {
2104                 ret = -ENOMEM;
2105                 goto free;
2106         }
2107
2108         operand_flags = 0;
2109         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2110         if (IS_ERR(operand1)) {
2111                 ret = PTR_ERR(operand1);
2112                 goto free;
2113         }
2114
2115         expr->flags |= operand1->flags &
2116                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2117         expr->fn = hist_field_unary_minus;
2118         expr->operands[0] = operand1;
2119         expr->operator = FIELD_OP_UNARY_MINUS;
2120         expr->name = expr_str(expr, 0);
2121         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2122         if (!expr->type) {
2123                 ret = -ENOMEM;
2124                 goto free;
2125         }
2126
2127         return expr;
2128  free:
2129         destroy_hist_field(expr, 0);
2130         return ERR_PTR(ret);
2131 }
2132
2133 static int check_expr_operands(struct trace_array *tr,
2134                                struct hist_field *operand1,
2135                                struct hist_field *operand2)
2136 {
2137         unsigned long operand1_flags = operand1->flags;
2138         unsigned long operand2_flags = operand2->flags;
2139
2140         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2141             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2142                 struct hist_field *var;
2143
2144                 var = find_var_field(operand1->var.hist_data, operand1->name);
2145                 if (!var)
2146                         return -EINVAL;
2147                 operand1_flags = var->flags;
2148         }
2149
2150         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2151             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2152                 struct hist_field *var;
2153
2154                 var = find_var_field(operand2->var.hist_data, operand2->name);
2155                 if (!var)
2156                         return -EINVAL;
2157                 operand2_flags = var->flags;
2158         }
2159
2160         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2161             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2162                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2163                 return -EINVAL;
2164         }
2165
2166         return 0;
2167 }
2168
2169 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2170                                      struct trace_event_file *file,
2171                                      char *str, unsigned long flags,
2172                                      char *var_name, unsigned int level)
2173 {
2174         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2175         unsigned long operand_flags;
2176         int field_op, ret = -EINVAL;
2177         char *sep, *operand1_str;
2178
2179         if (level > 3) {
2180                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2181                 return ERR_PTR(-EINVAL);
2182         }
2183
2184         field_op = contains_operator(str);
2185
2186         if (field_op == FIELD_OP_NONE)
2187                 return parse_atom(hist_data, file, str, &flags, var_name);
2188
2189         if (field_op == FIELD_OP_UNARY_MINUS)
2190                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2191
2192         switch (field_op) {
2193         case FIELD_OP_MINUS:
2194                 sep = "-";
2195                 break;
2196         case FIELD_OP_PLUS:
2197                 sep = "+";
2198                 break;
2199         default:
2200                 goto free;
2201         }
2202
2203         operand1_str = strsep(&str, sep);
2204         if (!operand1_str || !str)
2205                 goto free;
2206
2207         operand_flags = 0;
2208         operand1 = parse_atom(hist_data, file, operand1_str,
2209                               &operand_flags, NULL);
2210         if (IS_ERR(operand1)) {
2211                 ret = PTR_ERR(operand1);
2212                 operand1 = NULL;
2213                 goto free;
2214         }
2215
2216         /* rest of string could be another expression e.g. b+c in a+b+c */
2217         operand_flags = 0;
2218         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2219         if (IS_ERR(operand2)) {
2220                 ret = PTR_ERR(operand2);
2221                 operand2 = NULL;
2222                 goto free;
2223         }
2224
2225         ret = check_expr_operands(file->tr, operand1, operand2);
2226         if (ret)
2227                 goto free;
2228
2229         flags |= HIST_FIELD_FL_EXPR;
2230
2231         flags |= operand1->flags &
2232                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2233
2234         expr = create_hist_field(hist_data, NULL, flags, var_name);
2235         if (!expr) {
2236                 ret = -ENOMEM;
2237                 goto free;
2238         }
2239
2240         operand1->read_once = true;
2241         operand2->read_once = true;
2242
2243         expr->operands[0] = operand1;
2244         expr->operands[1] = operand2;
2245         expr->operator = field_op;
2246         expr->name = expr_str(expr, 0);
2247         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2248         if (!expr->type) {
2249                 ret = -ENOMEM;
2250                 goto free;
2251         }
2252
2253         switch (field_op) {
2254         case FIELD_OP_MINUS:
2255                 expr->fn = hist_field_minus;
2256                 break;
2257         case FIELD_OP_PLUS:
2258                 expr->fn = hist_field_plus;
2259                 break;
2260         default:
2261                 ret = -EINVAL;
2262                 goto free;
2263         }
2264
2265         return expr;
2266  free:
2267         destroy_hist_field(operand1, 0);
2268         destroy_hist_field(operand2, 0);
2269         destroy_hist_field(expr, 0);
2270
2271         return ERR_PTR(ret);
2272 }
2273
2274 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2275                                  struct trace_event_file *file)
2276 {
2277         struct event_trigger_data *test;
2278
2279         lockdep_assert_held(&event_mutex);
2280
2281         list_for_each_entry(test, &file->triggers, list) {
2282                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2283                         if (test->private_data == hist_data)
2284                                 return test->filter_str;
2285                 }
2286         }
2287
2288         return NULL;
2289 }
2290
2291 static struct event_command trigger_hist_cmd;
2292 static int event_hist_trigger_func(struct event_command *cmd_ops,
2293                                    struct trace_event_file *file,
2294                                    char *glob, char *cmd, char *param);
2295
2296 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2297                             struct hist_trigger_data *hist_data,
2298                             unsigned int n_keys)
2299 {
2300         struct hist_field *target_hist_field, *hist_field;
2301         unsigned int n, i, j;
2302
2303         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2304                 return false;
2305
2306         i = hist_data->n_vals;
2307         j = target_hist_data->n_vals;
2308
2309         for (n = 0; n < n_keys; n++) {
2310                 hist_field = hist_data->fields[i + n];
2311                 target_hist_field = target_hist_data->fields[j + n];
2312
2313                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2314                         return false;
2315                 if (hist_field->size != target_hist_field->size)
2316                         return false;
2317                 if (hist_field->is_signed != target_hist_field->is_signed)
2318                         return false;
2319         }
2320
2321         return true;
2322 }
2323
2324 static struct hist_trigger_data *
2325 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2326                      struct trace_event_file *file)
2327 {
2328         struct hist_trigger_data *hist_data;
2329         struct event_trigger_data *test;
2330         unsigned int n_keys;
2331
2332         lockdep_assert_held(&event_mutex);
2333
2334         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2335
2336         list_for_each_entry(test, &file->triggers, list) {
2337                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2338                         hist_data = test->private_data;
2339
2340                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2341                                 return hist_data;
2342                 }
2343         }
2344
2345         return NULL;
2346 }
2347
2348 static struct trace_event_file *event_file(struct trace_array *tr,
2349                                            char *system, char *event_name)
2350 {
2351         struct trace_event_file *file;
2352
2353         file = __find_event_file(tr, system, event_name);
2354         if (!file)
2355                 return ERR_PTR(-EINVAL);
2356
2357         return file;
2358 }
2359
2360 static struct hist_field *
2361 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2362                          char *system, char *event_name, char *field_name)
2363 {
2364         struct hist_field *event_var;
2365         char *synthetic_name;
2366
2367         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2368         if (!synthetic_name)
2369                 return ERR_PTR(-ENOMEM);
2370
2371         strcpy(synthetic_name, "synthetic_");
2372         strcat(synthetic_name, field_name);
2373
2374         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2375
2376         kfree(synthetic_name);
2377
2378         return event_var;
2379 }
2380
2381 /**
2382  * create_field_var_hist - Automatically create a histogram and var for a field
2383  * @target_hist_data: The target hist trigger
2384  * @subsys_name: Optional subsystem name
2385  * @event_name: Optional event name
2386  * @field_name: The name of the field (and the resulting variable)
2387  *
2388  * Hist trigger actions fetch data from variables, not directly from
2389  * events.  However, for convenience, users are allowed to directly
2390  * specify an event field in an action, which will be automatically
2391  * converted into a variable on their behalf.
2392
2393  * If a user specifies a field on an event that isn't the event the
2394  * histogram currently being defined (the target event histogram), the
2395  * only way that can be accomplished is if a new hist trigger is
2396  * created and the field variable defined on that.
2397  *
2398  * This function creates a new histogram compatible with the target
2399  * event (meaning a histogram with the same key as the target
2400  * histogram), and creates a variable for the specified field, but
2401  * with 'synthetic_' prepended to the variable name in order to avoid
2402  * collision with normal field variables.
2403  *
2404  * Return: The variable created for the field.
2405  */
2406 static struct hist_field *
2407 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2408                       char *subsys_name, char *event_name, char *field_name)
2409 {
2410         struct trace_array *tr = target_hist_data->event_file->tr;
2411         struct hist_field *event_var = ERR_PTR(-EINVAL);
2412         struct hist_trigger_data *hist_data;
2413         unsigned int i, n, first = true;
2414         struct field_var_hist *var_hist;
2415         struct trace_event_file *file;
2416         struct hist_field *key_field;
2417         char *saved_filter;
2418         char *cmd;
2419         int ret;
2420
2421         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2422                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2423                 return ERR_PTR(-EINVAL);
2424         }
2425
2426         file = event_file(tr, subsys_name, event_name);
2427
2428         if (IS_ERR(file)) {
2429                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2430                 ret = PTR_ERR(file);
2431                 return ERR_PTR(ret);
2432         }
2433
2434         /*
2435          * Look for a histogram compatible with target.  We'll use the
2436          * found histogram specification to create a new matching
2437          * histogram with our variable on it.  target_hist_data is not
2438          * yet a registered histogram so we can't use that.
2439          */
2440         hist_data = find_compatible_hist(target_hist_data, file);
2441         if (!hist_data) {
2442                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2443                 return ERR_PTR(-EINVAL);
2444         }
2445
2446         /* See if a synthetic field variable has already been created */
2447         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2448                                              event_name, field_name);
2449         if (!IS_ERR_OR_NULL(event_var))
2450                 return event_var;
2451
2452         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2453         if (!var_hist)
2454                 return ERR_PTR(-ENOMEM);
2455
2456         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2457         if (!cmd) {
2458                 kfree(var_hist);
2459                 return ERR_PTR(-ENOMEM);
2460         }
2461
2462         /* Use the same keys as the compatible histogram */
2463         strcat(cmd, "keys=");
2464
2465         for_each_hist_key_field(i, hist_data) {
2466                 key_field = hist_data->fields[i];
2467                 if (!first)
2468                         strcat(cmd, ",");
2469                 strcat(cmd, key_field->field->name);
2470                 first = false;
2471         }
2472
2473         /* Create the synthetic field variable specification */
2474         strcat(cmd, ":synthetic_");
2475         strcat(cmd, field_name);
2476         strcat(cmd, "=");
2477         strcat(cmd, field_name);
2478
2479         /* Use the same filter as the compatible histogram */
2480         saved_filter = find_trigger_filter(hist_data, file);
2481         if (saved_filter) {
2482                 strcat(cmd, " if ");
2483                 strcat(cmd, saved_filter);
2484         }
2485
2486         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2487         if (!var_hist->cmd) {
2488                 kfree(cmd);
2489                 kfree(var_hist);
2490                 return ERR_PTR(-ENOMEM);
2491         }
2492
2493         /* Save the compatible histogram information */
2494         var_hist->hist_data = hist_data;
2495
2496         /* Create the new histogram with our variable */
2497         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2498                                       "", "hist", cmd);
2499         if (ret) {
2500                 kfree(cmd);
2501                 kfree(var_hist->cmd);
2502                 kfree(var_hist);
2503                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2504                 return ERR_PTR(ret);
2505         }
2506
2507         kfree(cmd);
2508
2509         /* If we can't find the variable, something went wrong */
2510         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2511                                              event_name, field_name);
2512         if (IS_ERR_OR_NULL(event_var)) {
2513                 kfree(var_hist->cmd);
2514                 kfree(var_hist);
2515                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2516                 return ERR_PTR(-EINVAL);
2517         }
2518
2519         n = target_hist_data->n_field_var_hists;
2520         target_hist_data->field_var_hists[n] = var_hist;
2521         target_hist_data->n_field_var_hists++;
2522
2523         return event_var;
2524 }
2525
2526 static struct hist_field *
2527 find_target_event_var(struct hist_trigger_data *hist_data,
2528                       char *subsys_name, char *event_name, char *var_name)
2529 {
2530         struct trace_event_file *file = hist_data->event_file;
2531         struct hist_field *hist_field = NULL;
2532
2533         if (subsys_name) {
2534                 struct trace_event_call *call;
2535
2536                 if (!event_name)
2537                         return NULL;
2538
2539                 call = file->event_call;
2540
2541                 if (strcmp(subsys_name, call->class->system) != 0)
2542                         return NULL;
2543
2544                 if (strcmp(event_name, trace_event_name(call)) != 0)
2545                         return NULL;
2546         }
2547
2548         hist_field = find_var_field(hist_data, var_name);
2549
2550         return hist_field;
2551 }
2552
2553 static inline void __update_field_vars(struct tracing_map_elt *elt,
2554                                        struct ring_buffer_event *rbe,
2555                                        void *rec,
2556                                        struct field_var **field_vars,
2557                                        unsigned int n_field_vars,
2558                                        unsigned int field_var_str_start)
2559 {
2560         struct hist_elt_data *elt_data = elt->private_data;
2561         unsigned int i, j, var_idx;
2562         u64 var_val;
2563
2564         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2565                 struct field_var *field_var = field_vars[i];
2566                 struct hist_field *var = field_var->var;
2567                 struct hist_field *val = field_var->val;
2568
2569                 var_val = val->fn(val, elt, rbe, rec);
2570                 var_idx = var->var.idx;
2571
2572                 if (val->flags & HIST_FIELD_FL_STRING) {
2573                         char *str = elt_data->field_var_str[j++];
2574                         char *val_str = (char *)(uintptr_t)var_val;
2575
2576                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2577                         var_val = (u64)(uintptr_t)str;
2578                 }
2579                 tracing_map_set_var(elt, var_idx, var_val);
2580         }
2581 }
2582
2583 static void update_field_vars(struct hist_trigger_data *hist_data,
2584                               struct tracing_map_elt *elt,
2585                               struct ring_buffer_event *rbe,
2586                               void *rec)
2587 {
2588         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
2589                             hist_data->n_field_vars, 0);
2590 }
2591
2592 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2593                                  struct tracing_map_elt *elt, void *rec,
2594                                  struct ring_buffer_event *rbe, void *key,
2595                                  struct action_data *data, u64 *var_ref_vals)
2596 {
2597         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
2598                             hist_data->n_save_vars, hist_data->n_field_var_str);
2599 }
2600
2601 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2602                                      struct trace_event_file *file,
2603                                      char *name, int size, const char *type)
2604 {
2605         struct hist_field *var;
2606         int idx;
2607
2608         if (find_var(hist_data, file, name) && !hist_data->remove) {
2609                 var = ERR_PTR(-EINVAL);
2610                 goto out;
2611         }
2612
2613         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2614         if (!var) {
2615                 var = ERR_PTR(-ENOMEM);
2616                 goto out;
2617         }
2618
2619         idx = tracing_map_add_var(hist_data->map);
2620         if (idx < 0) {
2621                 kfree(var);
2622                 var = ERR_PTR(-EINVAL);
2623                 goto out;
2624         }
2625
2626         var->ref = 1;
2627         var->flags = HIST_FIELD_FL_VAR;
2628         var->var.idx = idx;
2629         var->var.hist_data = var->hist_data = hist_data;
2630         var->size = size;
2631         var->var.name = kstrdup(name, GFP_KERNEL);
2632         var->type = kstrdup(type, GFP_KERNEL);
2633         if (!var->var.name || !var->type) {
2634                 kfree(var->var.name);
2635                 kfree(var->type);
2636                 kfree(var);
2637                 var = ERR_PTR(-ENOMEM);
2638         }
2639  out:
2640         return var;
2641 }
2642
2643 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2644                                           struct trace_event_file *file,
2645                                           char *field_name)
2646 {
2647         struct hist_field *val = NULL, *var = NULL;
2648         unsigned long flags = HIST_FIELD_FL_VAR;
2649         struct trace_array *tr = file->tr;
2650         struct field_var *field_var;
2651         int ret = 0;
2652
2653         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2654                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2655                 ret = -EINVAL;
2656                 goto err;
2657         }
2658
2659         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2660         if (IS_ERR(val)) {
2661                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2662                 ret = PTR_ERR(val);
2663                 goto err;
2664         }
2665
2666         var = create_var(hist_data, file, field_name, val->size, val->type);
2667         if (IS_ERR(var)) {
2668                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2669                 kfree(val);
2670                 ret = PTR_ERR(var);
2671                 goto err;
2672         }
2673
2674         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2675         if (!field_var) {
2676                 kfree(val);
2677                 kfree(var);
2678                 ret =  -ENOMEM;
2679                 goto err;
2680         }
2681
2682         field_var->var = var;
2683         field_var->val = val;
2684  out:
2685         return field_var;
2686  err:
2687         field_var = ERR_PTR(ret);
2688         goto out;
2689 }
2690
2691 /**
2692  * create_target_field_var - Automatically create a variable for a field
2693  * @target_hist_data: The target hist trigger
2694  * @subsys_name: Optional subsystem name
2695  * @event_name: Optional event name
2696  * @var_name: The name of the field (and the resulting variable)
2697  *
2698  * Hist trigger actions fetch data from variables, not directly from
2699  * events.  However, for convenience, users are allowed to directly
2700  * specify an event field in an action, which will be automatically
2701  * converted into a variable on their behalf.
2702
2703  * This function creates a field variable with the name var_name on
2704  * the hist trigger currently being defined on the target event.  If
2705  * subsys_name and event_name are specified, this function simply
2706  * verifies that they do in fact match the target event subsystem and
2707  * event name.
2708  *
2709  * Return: The variable created for the field.
2710  */
2711 static struct field_var *
2712 create_target_field_var(struct hist_trigger_data *target_hist_data,
2713                         char *subsys_name, char *event_name, char *var_name)
2714 {
2715         struct trace_event_file *file = target_hist_data->event_file;
2716
2717         if (subsys_name) {
2718                 struct trace_event_call *call;
2719
2720                 if (!event_name)
2721                         return NULL;
2722
2723                 call = file->event_call;
2724
2725                 if (strcmp(subsys_name, call->class->system) != 0)
2726                         return NULL;
2727
2728                 if (strcmp(event_name, trace_event_name(call)) != 0)
2729                         return NULL;
2730         }
2731
2732         return create_field_var(target_hist_data, file, var_name);
2733 }
2734
2735 static bool check_track_val_max(u64 track_val, u64 var_val)
2736 {
2737         if (var_val <= track_val)
2738                 return false;
2739
2740         return true;
2741 }
2742
2743 static bool check_track_val_changed(u64 track_val, u64 var_val)
2744 {
2745         if (var_val == track_val)
2746                 return false;
2747
2748         return true;
2749 }
2750
2751 static u64 get_track_val(struct hist_trigger_data *hist_data,
2752                          struct tracing_map_elt *elt,
2753                          struct action_data *data)
2754 {
2755         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2756         u64 track_val;
2757
2758         track_val = tracing_map_read_var(elt, track_var_idx);
2759
2760         return track_val;
2761 }
2762
2763 static void save_track_val(struct hist_trigger_data *hist_data,
2764                            struct tracing_map_elt *elt,
2765                            struct action_data *data, u64 var_val)
2766 {
2767         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2768
2769         tracing_map_set_var(elt, track_var_idx, var_val);
2770 }
2771
2772 static void save_track_data(struct hist_trigger_data *hist_data,
2773                             struct tracing_map_elt *elt, void *rec,
2774                             struct ring_buffer_event *rbe, void *key,
2775                             struct action_data *data, u64 *var_ref_vals)
2776 {
2777         if (data->track_data.save_data)
2778                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2779 }
2780
2781 static bool check_track_val(struct tracing_map_elt *elt,
2782                             struct action_data *data,
2783                             u64 var_val)
2784 {
2785         struct hist_trigger_data *hist_data;
2786         u64 track_val;
2787
2788         hist_data = data->track_data.track_var->hist_data;
2789         track_val = get_track_val(hist_data, elt, data);
2790
2791         return data->track_data.check_val(track_val, var_val);
2792 }
2793
2794 #ifdef CONFIG_TRACER_SNAPSHOT
2795 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2796 {
2797         /* called with tr->max_lock held */
2798         struct track_data *track_data = tr->cond_snapshot->cond_data;
2799         struct hist_elt_data *elt_data, *track_elt_data;
2800         struct snapshot_context *context = cond_data;
2801         struct action_data *action;
2802         u64 track_val;
2803
2804         if (!track_data)
2805                 return false;
2806
2807         action = track_data->action_data;
2808
2809         track_val = get_track_val(track_data->hist_data, context->elt,
2810                                   track_data->action_data);
2811
2812         if (!action->track_data.check_val(track_data->track_val, track_val))
2813                 return false;
2814
2815         track_data->track_val = track_val;
2816         memcpy(track_data->key, context->key, track_data->key_len);
2817
2818         elt_data = context->elt->private_data;
2819         track_elt_data = track_data->elt.private_data;
2820         if (elt_data->comm)
2821                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2822
2823         track_data->updated = true;
2824
2825         return true;
2826 }
2827
2828 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2829                                      struct tracing_map_elt *elt, void *rec,
2830                                      struct ring_buffer_event *rbe, void *key,
2831                                      struct action_data *data,
2832                                      u64 *var_ref_vals)
2833 {
2834         struct trace_event_file *file = hist_data->event_file;
2835         struct snapshot_context context;
2836
2837         context.elt = elt;
2838         context.key = key;
2839
2840         tracing_snapshot_cond(file->tr, &context);
2841 }
2842
2843 static void hist_trigger_print_key(struct seq_file *m,
2844                                    struct hist_trigger_data *hist_data,
2845                                    void *key,
2846                                    struct tracing_map_elt *elt);
2847
2848 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2849 {
2850         unsigned int i;
2851
2852         if (!hist_data->n_actions)
2853                 return NULL;
2854
2855         for (i = 0; i < hist_data->n_actions; i++) {
2856                 struct action_data *data = hist_data->actions[i];
2857
2858                 if (data->action == ACTION_SNAPSHOT)
2859                         return data;
2860         }
2861
2862         return NULL;
2863 }
2864
2865 static void track_data_snapshot_print(struct seq_file *m,
2866                                       struct hist_trigger_data *hist_data)
2867 {
2868         struct trace_event_file *file = hist_data->event_file;
2869         struct track_data *track_data;
2870         struct action_data *action;
2871
2872         track_data = tracing_cond_snapshot_data(file->tr);
2873         if (!track_data)
2874                 return;
2875
2876         if (!track_data->updated)
2877                 return;
2878
2879         action = snapshot_action(hist_data);
2880         if (!action)
2881                 return;
2882
2883         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2884         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2885                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2886                    action->track_data.var_str, track_data->track_val);
2887
2888         seq_puts(m, "\ttriggered by event with key: ");
2889         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2890         seq_putc(m, '\n');
2891 }
2892 #else
2893 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2894 {
2895         return false;
2896 }
2897 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2898                                      struct tracing_map_elt *elt, void *rec,
2899                                      struct ring_buffer_event *rbe, void *key,
2900                                      struct action_data *data,
2901                                      u64 *var_ref_vals) {}
2902 static void track_data_snapshot_print(struct seq_file *m,
2903                                       struct hist_trigger_data *hist_data) {}
2904 #endif /* CONFIG_TRACER_SNAPSHOT */
2905
2906 static void track_data_print(struct seq_file *m,
2907                              struct hist_trigger_data *hist_data,
2908                              struct tracing_map_elt *elt,
2909                              struct action_data *data)
2910 {
2911         u64 track_val = get_track_val(hist_data, elt, data);
2912         unsigned int i, save_var_idx;
2913
2914         if (data->handler == HANDLER_ONMAX)
2915                 seq_printf(m, "\n\tmax: %10llu", track_val);
2916         else if (data->handler == HANDLER_ONCHANGE)
2917                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2918
2919         if (data->action == ACTION_SNAPSHOT)
2920                 return;
2921
2922         for (i = 0; i < hist_data->n_save_vars; i++) {
2923                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2924                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2925                 u64 val;
2926
2927                 save_var_idx = save_var->var.idx;
2928
2929                 val = tracing_map_read_var(elt, save_var_idx);
2930
2931                 if (save_val->flags & HIST_FIELD_FL_STRING) {
2932                         seq_printf(m, "  %s: %-32s", save_var->var.name,
2933                                    (char *)(uintptr_t)(val));
2934                 } else
2935                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2936         }
2937 }
2938
2939 static void ontrack_action(struct hist_trigger_data *hist_data,
2940                            struct tracing_map_elt *elt, void *rec,
2941                            struct ring_buffer_event *rbe, void *key,
2942                            struct action_data *data, u64 *var_ref_vals)
2943 {
2944         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2945
2946         if (check_track_val(elt, data, var_val)) {
2947                 save_track_val(hist_data, elt, data, var_val);
2948                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2949         }
2950 }
2951
2952 static void action_data_destroy(struct action_data *data)
2953 {
2954         unsigned int i;
2955
2956         lockdep_assert_held(&event_mutex);
2957
2958         kfree(data->action_name);
2959
2960         for (i = 0; i < data->n_params; i++)
2961                 kfree(data->params[i]);
2962
2963         if (data->synth_event)
2964                 data->synth_event->ref--;
2965
2966         kfree(data->synth_event_name);
2967
2968         kfree(data);
2969 }
2970
2971 static void track_data_destroy(struct hist_trigger_data *hist_data,
2972                                struct action_data *data)
2973 {
2974         struct trace_event_file *file = hist_data->event_file;
2975
2976         destroy_hist_field(data->track_data.track_var, 0);
2977
2978         if (data->action == ACTION_SNAPSHOT) {
2979                 struct track_data *track_data;
2980
2981                 track_data = tracing_cond_snapshot_data(file->tr);
2982                 if (track_data && track_data->hist_data == hist_data) {
2983                         tracing_snapshot_cond_disable(file->tr);
2984                         track_data_free(track_data);
2985                 }
2986         }
2987
2988         kfree(data->track_data.var_str);
2989
2990         action_data_destroy(data);
2991 }
2992
2993 static int action_create(struct hist_trigger_data *hist_data,
2994                          struct action_data *data);
2995
2996 static int track_data_create(struct hist_trigger_data *hist_data,
2997                              struct action_data *data)
2998 {
2999         struct hist_field *var_field, *ref_field, *track_var = NULL;
3000         struct trace_event_file *file = hist_data->event_file;
3001         struct trace_array *tr = file->tr;
3002         char *track_data_var_str;
3003         int ret = 0;
3004
3005         track_data_var_str = data->track_data.var_str;
3006         if (track_data_var_str[0] != '$') {
3007                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3008                 return -EINVAL;
3009         }
3010         track_data_var_str++;
3011
3012         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3013         if (!var_field) {
3014                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3015                 return -EINVAL;
3016         }
3017
3018         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3019         if (!ref_field)
3020                 return -ENOMEM;
3021
3022         data->track_data.var_ref = ref_field;
3023
3024         if (data->handler == HANDLER_ONMAX)
3025                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3026         if (IS_ERR(track_var)) {
3027                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3028                 ret = PTR_ERR(track_var);
3029                 goto out;
3030         }
3031
3032         if (data->handler == HANDLER_ONCHANGE)
3033                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3034         if (IS_ERR(track_var)) {
3035                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3036                 ret = PTR_ERR(track_var);
3037                 goto out;
3038         }
3039         data->track_data.track_var = track_var;
3040
3041         ret = action_create(hist_data, data);
3042  out:
3043         return ret;
3044 }
3045
3046 static int parse_action_params(struct trace_array *tr, char *params,
3047                                struct action_data *data)
3048 {
3049         char *param, *saved_param;
3050         bool first_param = true;
3051         int ret = 0;
3052
3053         while (params) {
3054                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3055                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3056                         goto out;
3057                 }
3058
3059                 param = strsep(&params, ",");
3060                 if (!param) {
3061                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3062                         ret = -EINVAL;
3063                         goto out;
3064                 }
3065
3066                 param = strstrip(param);
3067                 if (strlen(param) < 2) {
3068                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3069                         ret = -EINVAL;
3070                         goto out;
3071                 }
3072
3073                 saved_param = kstrdup(param, GFP_KERNEL);
3074                 if (!saved_param) {
3075                         ret = -ENOMEM;
3076                         goto out;
3077                 }
3078
3079                 if (first_param && data->use_trace_keyword) {
3080                         data->synth_event_name = saved_param;
3081                         first_param = false;
3082                         continue;
3083                 }
3084                 first_param = false;
3085
3086                 data->params[data->n_params++] = saved_param;
3087         }
3088  out:
3089         return ret;
3090 }
3091
3092 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3093                         enum handler_id handler)
3094 {
3095         char *action_name;
3096         int ret = 0;
3097
3098         strsep(&str, ".");
3099         if (!str) {
3100                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3101                 ret = -EINVAL;
3102                 goto out;
3103         }
3104
3105         action_name = strsep(&str, "(");
3106         if (!action_name || !str) {
3107                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3108                 ret = -EINVAL;
3109                 goto out;
3110         }
3111
3112         if (str_has_prefix(action_name, "save")) {
3113                 char *params = strsep(&str, ")");
3114
3115                 if (!params) {
3116                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3117                         ret = -EINVAL;
3118                         goto out;
3119                 }
3120
3121                 ret = parse_action_params(tr, params, data);
3122                 if (ret)
3123                         goto out;
3124
3125                 if (handler == HANDLER_ONMAX)
3126                         data->track_data.check_val = check_track_val_max;
3127                 else if (handler == HANDLER_ONCHANGE)
3128                         data->track_data.check_val = check_track_val_changed;
3129                 else {
3130                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3131                         ret = -EINVAL;
3132                         goto out;
3133                 }
3134
3135                 data->track_data.save_data = save_track_data_vars;
3136                 data->fn = ontrack_action;
3137                 data->action = ACTION_SAVE;
3138         } else if (str_has_prefix(action_name, "snapshot")) {
3139                 char *params = strsep(&str, ")");
3140
3141                 if (!str) {
3142                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3143                         ret = -EINVAL;
3144                         goto out;
3145                 }
3146
3147                 if (handler == HANDLER_ONMAX)
3148                         data->track_data.check_val = check_track_val_max;
3149                 else if (handler == HANDLER_ONCHANGE)
3150                         data->track_data.check_val = check_track_val_changed;
3151                 else {
3152                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3153                         ret = -EINVAL;
3154                         goto out;
3155                 }
3156
3157                 data->track_data.save_data = save_track_data_snapshot;
3158                 data->fn = ontrack_action;
3159                 data->action = ACTION_SNAPSHOT;
3160         } else {
3161                 char *params = strsep(&str, ")");
3162
3163                 if (str_has_prefix(action_name, "trace"))
3164                         data->use_trace_keyword = true;
3165
3166                 if (params) {
3167                         ret = parse_action_params(tr, params, data);
3168                         if (ret)
3169                                 goto out;
3170                 }
3171
3172                 if (handler == HANDLER_ONMAX)
3173                         data->track_data.check_val = check_track_val_max;
3174                 else if (handler == HANDLER_ONCHANGE)
3175                         data->track_data.check_val = check_track_val_changed;
3176
3177                 if (handler != HANDLER_ONMATCH) {
3178                         data->track_data.save_data = action_trace;
3179                         data->fn = ontrack_action;
3180                 } else
3181                         data->fn = action_trace;
3182
3183                 data->action = ACTION_TRACE;
3184         }
3185
3186         data->action_name = kstrdup(action_name, GFP_KERNEL);
3187         if (!data->action_name) {
3188                 ret = -ENOMEM;
3189                 goto out;
3190         }
3191
3192         data->handler = handler;
3193  out:
3194         return ret;
3195 }
3196
3197 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3198                                             char *str, enum handler_id handler)
3199 {
3200         struct action_data *data;
3201         int ret = -EINVAL;
3202         char *var_str;
3203
3204         data = kzalloc(sizeof(*data), GFP_KERNEL);
3205         if (!data)
3206                 return ERR_PTR(-ENOMEM);
3207
3208         var_str = strsep(&str, ")");
3209         if (!var_str || !str) {
3210                 ret = -EINVAL;
3211                 goto free;
3212         }
3213
3214         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3215         if (!data->track_data.var_str) {
3216                 ret = -ENOMEM;
3217                 goto free;
3218         }
3219
3220         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3221         if (ret)
3222                 goto free;
3223  out:
3224         return data;
3225  free:
3226         track_data_destroy(hist_data, data);
3227         data = ERR_PTR(ret);
3228         goto out;
3229 }
3230
3231 static void onmatch_destroy(struct action_data *data)
3232 {
3233         kfree(data->match_data.event);
3234         kfree(data->match_data.event_system);
3235
3236         action_data_destroy(data);
3237 }
3238
3239 static void destroy_field_var(struct field_var *field_var)
3240 {
3241         if (!field_var)
3242                 return;
3243
3244         destroy_hist_field(field_var->var, 0);
3245         destroy_hist_field(field_var->val, 0);
3246
3247         kfree(field_var);
3248 }
3249
3250 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3251 {
3252         unsigned int i;
3253
3254         for (i = 0; i < hist_data->n_field_vars; i++)
3255                 destroy_field_var(hist_data->field_vars[i]);
3256
3257         for (i = 0; i < hist_data->n_save_vars; i++)
3258                 destroy_field_var(hist_data->save_vars[i]);
3259 }
3260
3261 static void save_field_var(struct hist_trigger_data *hist_data,
3262                            struct field_var *field_var)
3263 {
3264         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3265
3266         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3267                 hist_data->n_field_var_str++;
3268 }
3269
3270
3271 static int check_synth_field(struct synth_event *event,
3272                              struct hist_field *hist_field,
3273                              unsigned int field_pos)
3274 {
3275         struct synth_field *field;
3276
3277         if (field_pos >= event->n_fields)
3278                 return -EINVAL;
3279
3280         field = event->fields[field_pos];
3281
3282         if (strcmp(field->type, hist_field->type) != 0) {
3283                 if (field->size != hist_field->size ||
3284                     field->is_signed != hist_field->is_signed)
3285                         return -EINVAL;
3286         }
3287
3288         return 0;
3289 }
3290
3291 static struct hist_field *
3292 trace_action_find_var(struct hist_trigger_data *hist_data,
3293                       struct action_data *data,
3294                       char *system, char *event, char *var)
3295 {
3296         struct trace_array *tr = hist_data->event_file->tr;
3297         struct hist_field *hist_field;
3298
3299         var++; /* skip '$' */
3300
3301         hist_field = find_target_event_var(hist_data, system, event, var);
3302         if (!hist_field) {
3303                 if (!system && data->handler == HANDLER_ONMATCH) {
3304                         system = data->match_data.event_system;
3305                         event = data->match_data.event;
3306                 }
3307
3308                 hist_field = find_event_var(hist_data, system, event, var);
3309         }
3310
3311         if (!hist_field)
3312                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3313
3314         return hist_field;
3315 }
3316
3317 static struct hist_field *
3318 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3319                               struct action_data *data, char *system,
3320                               char *event, char *var)
3321 {
3322         struct hist_field *hist_field = NULL;
3323         struct field_var *field_var;
3324
3325         /*
3326          * First try to create a field var on the target event (the
3327          * currently being defined).  This will create a variable for
3328          * unqualified fields on the target event, or if qualified,
3329          * target fields that have qualified names matching the target.
3330          */
3331         field_var = create_target_field_var(hist_data, system, event, var);
3332
3333         if (field_var && !IS_ERR(field_var)) {
3334                 save_field_var(hist_data, field_var);
3335                 hist_field = field_var->var;
3336         } else {
3337                 field_var = NULL;
3338                 /*
3339                  * If no explicit system.event is specfied, default to
3340                  * looking for fields on the onmatch(system.event.xxx)
3341                  * event.
3342                  */
3343                 if (!system && data->handler == HANDLER_ONMATCH) {
3344                         system = data->match_data.event_system;
3345                         event = data->match_data.event;
3346                 }
3347
3348                 /*
3349                  * At this point, we're looking at a field on another
3350                  * event.  Because we can't modify a hist trigger on
3351                  * another event to add a variable for a field, we need
3352                  * to create a new trigger on that event and create the
3353                  * variable at the same time.
3354                  */
3355                 hist_field = create_field_var_hist(hist_data, system, event, var);
3356                 if (IS_ERR(hist_field))
3357                         goto free;
3358         }
3359  out:
3360         return hist_field;
3361  free:
3362         destroy_field_var(field_var);
3363         hist_field = NULL;
3364         goto out;
3365 }
3366
3367 static int trace_action_create(struct hist_trigger_data *hist_data,
3368                                struct action_data *data)
3369 {
3370         struct trace_array *tr = hist_data->event_file->tr;
3371         char *event_name, *param, *system = NULL;
3372         struct hist_field *hist_field, *var_ref;
3373         unsigned int i;
3374         unsigned int field_pos = 0;
3375         struct synth_event *event;
3376         char *synth_event_name;
3377         int var_ref_idx, ret = 0;
3378
3379         lockdep_assert_held(&event_mutex);
3380
3381         if (data->use_trace_keyword)
3382                 synth_event_name = data->synth_event_name;
3383         else
3384                 synth_event_name = data->action_name;
3385
3386         event = find_synth_event(synth_event_name);
3387         if (!event) {
3388                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3389                 return -EINVAL;
3390         }
3391
3392         event->ref++;
3393
3394         for (i = 0; i < data->n_params; i++) {
3395                 char *p;
3396
3397                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3398                 if (!param) {
3399                         ret = -ENOMEM;
3400                         goto err;
3401                 }
3402
3403                 system = strsep(&param, ".");
3404                 if (!param) {
3405                         param = (char *)system;
3406                         system = event_name = NULL;
3407                 } else {
3408                         event_name = strsep(&param, ".");
3409                         if (!param) {
3410                                 kfree(p);
3411                                 ret = -EINVAL;
3412                                 goto err;
3413                         }
3414                 }
3415
3416                 if (param[0] == '$')
3417                         hist_field = trace_action_find_var(hist_data, data,
3418                                                            system, event_name,
3419                                                            param);
3420                 else
3421                         hist_field = trace_action_create_field_var(hist_data,
3422                                                                    data,
3423                                                                    system,
3424                                                                    event_name,
3425                                                                    param);
3426
3427                 if (!hist_field) {
3428                         kfree(p);
3429                         ret = -EINVAL;
3430                         goto err;
3431                 }
3432
3433                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3434                         var_ref = create_var_ref(hist_data, hist_field,
3435                                                  system, event_name);
3436                         if (!var_ref) {
3437                                 kfree(p);
3438                                 ret = -ENOMEM;
3439                                 goto err;
3440                         }
3441
3442                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3443                         if (WARN_ON(var_ref_idx < 0)) {
3444                                 ret = var_ref_idx;
3445                                 goto err;
3446                         }
3447
3448                         data->var_ref_idx[i] = var_ref_idx;
3449
3450                         field_pos++;
3451                         kfree(p);
3452                         continue;
3453                 }
3454
3455                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3456                 kfree(p);
3457                 ret = -EINVAL;
3458                 goto err;
3459         }
3460
3461         if (field_pos != event->n_fields) {
3462                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3463                 ret = -EINVAL;
3464                 goto err;
3465         }
3466
3467         data->synth_event = event;
3468  out:
3469         return ret;
3470  err:
3471         event->ref--;
3472
3473         goto out;
3474 }
3475
3476 static int action_create(struct hist_trigger_data *hist_data,
3477                          struct action_data *data)
3478 {
3479         struct trace_event_file *file = hist_data->event_file;
3480         struct trace_array *tr = file->tr;
3481         struct track_data *track_data;
3482         struct field_var *field_var;
3483         unsigned int i;
3484         char *param;
3485         int ret = 0;
3486
3487         if (data->action == ACTION_TRACE)
3488                 return trace_action_create(hist_data, data);
3489
3490         if (data->action == ACTION_SNAPSHOT) {
3491                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3492                 if (IS_ERR(track_data)) {
3493                         ret = PTR_ERR(track_data);
3494                         goto out;
3495                 }
3496
3497                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3498                                                    cond_snapshot_update);
3499                 if (ret)
3500                         track_data_free(track_data);
3501
3502                 goto out;
3503         }
3504
3505         if (data->action == ACTION_SAVE) {
3506                 if (hist_data->n_save_vars) {
3507                         ret = -EEXIST;
3508                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3509                         goto out;
3510                 }
3511
3512                 for (i = 0; i < data->n_params; i++) {
3513                         param = kstrdup(data->params[i], GFP_KERNEL);
3514                         if (!param) {
3515                                 ret = -ENOMEM;
3516                                 goto out;
3517                         }
3518
3519                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3520                         if (IS_ERR(field_var)) {
3521                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3522                                          errpos(param));
3523                                 ret = PTR_ERR(field_var);
3524                                 kfree(param);
3525                                 goto out;
3526                         }
3527
3528                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3529                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3530                                 hist_data->n_save_var_str++;
3531                         kfree(param);
3532                 }
3533         }
3534  out:
3535         return ret;
3536 }
3537
3538 static int onmatch_create(struct hist_trigger_data *hist_data,
3539                           struct action_data *data)
3540 {
3541         return action_create(hist_data, data);
3542 }
3543
3544 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3545 {
3546         char *match_event, *match_event_system;
3547         struct action_data *data;
3548         int ret = -EINVAL;
3549
3550         data = kzalloc(sizeof(*data), GFP_KERNEL);
3551         if (!data)
3552                 return ERR_PTR(-ENOMEM);
3553
3554         match_event = strsep(&str, ")");
3555         if (!match_event || !str) {
3556                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3557                 goto free;
3558         }
3559
3560         match_event_system = strsep(&match_event, ".");
3561         if (!match_event) {
3562                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3563                 goto free;
3564         }
3565
3566         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3567                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3568                 goto free;
3569         }
3570
3571         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3572         if (!data->match_data.event) {
3573                 ret = -ENOMEM;
3574                 goto free;
3575         }
3576
3577         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3578         if (!data->match_data.event_system) {
3579                 ret = -ENOMEM;
3580                 goto free;
3581         }
3582
3583         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3584         if (ret)
3585                 goto free;
3586  out:
3587         return data;
3588  free:
3589         onmatch_destroy(data);
3590         data = ERR_PTR(ret);
3591         goto out;
3592 }
3593
3594 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3595 {
3596         hist_data->fields[HITCOUNT_IDX] =
3597                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3598         if (!hist_data->fields[HITCOUNT_IDX])
3599                 return -ENOMEM;
3600
3601         hist_data->n_vals++;
3602         hist_data->n_fields++;
3603
3604         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3605                 return -EINVAL;
3606
3607         return 0;
3608 }
3609
3610 static int __create_val_field(struct hist_trigger_data *hist_data,
3611                               unsigned int val_idx,
3612                               struct trace_event_file *file,
3613                               char *var_name, char *field_str,
3614                               unsigned long flags)
3615 {
3616         struct hist_field *hist_field;
3617         int ret = 0;
3618
3619         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3620         if (IS_ERR(hist_field)) {
3621                 ret = PTR_ERR(hist_field);
3622                 goto out;
3623         }
3624
3625         hist_data->fields[val_idx] = hist_field;
3626
3627         ++hist_data->n_vals;
3628         ++hist_data->n_fields;
3629
3630         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3631                 ret = -EINVAL;
3632  out:
3633         return ret;
3634 }
3635
3636 static int create_val_field(struct hist_trigger_data *hist_data,
3637                             unsigned int val_idx,
3638                             struct trace_event_file *file,
3639                             char *field_str)
3640 {
3641         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3642                 return -EINVAL;
3643
3644         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3645 }
3646
3647 static int create_var_field(struct hist_trigger_data *hist_data,
3648                             unsigned int val_idx,
3649                             struct trace_event_file *file,
3650                             char *var_name, char *expr_str)
3651 {
3652         struct trace_array *tr = hist_data->event_file->tr;
3653         unsigned long flags = 0;
3654
3655         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3656                 return -EINVAL;
3657
3658         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3659                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3660                 return -EINVAL;
3661         }
3662
3663         flags |= HIST_FIELD_FL_VAR;
3664         hist_data->n_vars++;
3665         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3666                 return -EINVAL;
3667
3668         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3669 }
3670
3671 static int create_val_fields(struct hist_trigger_data *hist_data,
3672                              struct trace_event_file *file)
3673 {
3674         char *fields_str, *field_str;
3675         unsigned int i, j = 1;
3676         int ret;
3677
3678         ret = create_hitcount_val(hist_data);
3679         if (ret)
3680                 goto out;
3681
3682         fields_str = hist_data->attrs->vals_str;
3683         if (!fields_str)
3684                 goto out;
3685
3686         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3687                      j < TRACING_MAP_VALS_MAX; i++) {
3688                 field_str = strsep(&fields_str, ",");
3689                 if (!field_str)
3690                         break;
3691
3692                 if (strcmp(field_str, "hitcount") == 0)
3693                         continue;
3694
3695                 ret = create_val_field(hist_data, j++, file, field_str);
3696                 if (ret)
3697                         goto out;
3698         }
3699
3700         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3701                 ret = -EINVAL;
3702  out:
3703         return ret;
3704 }
3705
3706 static int create_key_field(struct hist_trigger_data *hist_data,
3707                             unsigned int key_idx,
3708                             unsigned int key_offset,
3709                             struct trace_event_file *file,
3710                             char *field_str)
3711 {
3712         struct trace_array *tr = hist_data->event_file->tr;
3713         struct hist_field *hist_field = NULL;
3714         unsigned long flags = 0;
3715         unsigned int key_size;
3716         int ret = 0;
3717
3718         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3719                 return -EINVAL;
3720
3721         flags |= HIST_FIELD_FL_KEY;
3722
3723         if (strcmp(field_str, "stacktrace") == 0) {
3724                 flags |= HIST_FIELD_FL_STACKTRACE;
3725                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3726                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3727         } else {
3728                 hist_field = parse_expr(hist_data, file, field_str, flags,
3729                                         NULL, 0);
3730                 if (IS_ERR(hist_field)) {
3731                         ret = PTR_ERR(hist_field);
3732                         goto out;
3733                 }
3734
3735                 if (field_has_hist_vars(hist_field, 0)) {
3736                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3737                         destroy_hist_field(hist_field, 0);
3738                         ret = -EINVAL;
3739                         goto out;
3740                 }
3741
3742                 key_size = hist_field->size;
3743         }
3744
3745         hist_data->fields[key_idx] = hist_field;
3746
3747         key_size = ALIGN(key_size, sizeof(u64));
3748         hist_data->fields[key_idx]->size = key_size;
3749         hist_data->fields[key_idx]->offset = key_offset;
3750
3751         hist_data->key_size += key_size;
3752
3753         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3754                 ret = -EINVAL;
3755                 goto out;
3756         }
3757
3758         hist_data->n_keys++;
3759         hist_data->n_fields++;
3760
3761         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3762                 return -EINVAL;
3763
3764         ret = key_size;
3765  out:
3766         return ret;
3767 }
3768
3769 static int create_key_fields(struct hist_trigger_data *hist_data,
3770                              struct trace_event_file *file)
3771 {
3772         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3773         char *fields_str, *field_str;
3774         int ret = -EINVAL;
3775
3776         fields_str = hist_data->attrs->keys_str;
3777         if (!fields_str)
3778                 goto out;
3779
3780         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3781                 field_str = strsep(&fields_str, ",");
3782                 if (!field_str)
3783                         break;
3784                 ret = create_key_field(hist_data, i, key_offset,
3785                                        file, field_str);
3786                 if (ret < 0)
3787                         goto out;
3788                 key_offset += ret;
3789         }
3790         if (fields_str) {
3791                 ret = -EINVAL;
3792                 goto out;
3793         }
3794         ret = 0;
3795  out:
3796         return ret;
3797 }
3798
3799 static int create_var_fields(struct hist_trigger_data *hist_data,
3800                              struct trace_event_file *file)
3801 {
3802         unsigned int i, j = hist_data->n_vals;
3803         int ret = 0;
3804
3805         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3806
3807         for (i = 0; i < n_vars; i++) {
3808                 char *var_name = hist_data->attrs->var_defs.name[i];
3809                 char *expr = hist_data->attrs->var_defs.expr[i];
3810
3811                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3812                 if (ret)
3813                         goto out;
3814         }
3815  out:
3816         return ret;
3817 }
3818
3819 static void free_var_defs(struct hist_trigger_data *hist_data)
3820 {
3821         unsigned int i;
3822
3823         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3824                 kfree(hist_data->attrs->var_defs.name[i]);
3825                 kfree(hist_data->attrs->var_defs.expr[i]);
3826         }
3827
3828         hist_data->attrs->var_defs.n_vars = 0;
3829 }
3830
3831 static int parse_var_defs(struct hist_trigger_data *hist_data)
3832 {
3833         struct trace_array *tr = hist_data->event_file->tr;
3834         char *s, *str, *var_name, *field_str;
3835         unsigned int i, j, n_vars = 0;
3836         int ret = 0;
3837
3838         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3839                 str = hist_data->attrs->assignment_str[i];
3840                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3841                         field_str = strsep(&str, ",");
3842                         if (!field_str)
3843                                 break;
3844
3845                         var_name = strsep(&field_str, "=");
3846                         if (!var_name || !field_str) {
3847                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3848                                          errpos(var_name));
3849                                 ret = -EINVAL;
3850                                 goto free;
3851                         }
3852
3853                         if (n_vars == TRACING_MAP_VARS_MAX) {
3854                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3855                                 ret = -EINVAL;
3856                                 goto free;
3857                         }
3858
3859                         s = kstrdup(var_name, GFP_KERNEL);
3860                         if (!s) {
3861                                 ret = -ENOMEM;
3862                                 goto free;
3863                         }
3864                         hist_data->attrs->var_defs.name[n_vars] = s;
3865
3866                         s = kstrdup(field_str, GFP_KERNEL);
3867                         if (!s) {
3868                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
3869                                 ret = -ENOMEM;
3870                                 goto free;
3871                         }
3872                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3873
3874                         hist_data->attrs->var_defs.n_vars = n_vars;
3875                 }
3876         }
3877
3878         return ret;
3879  free:
3880         free_var_defs(hist_data);
3881
3882         return ret;
3883 }
3884
3885 static int create_hist_fields(struct hist_trigger_data *hist_data,
3886                               struct trace_event_file *file)
3887 {
3888         int ret;
3889
3890         ret = parse_var_defs(hist_data);
3891         if (ret)
3892                 goto out;
3893
3894         ret = create_val_fields(hist_data, file);
3895         if (ret)
3896                 goto out;
3897
3898         ret = create_var_fields(hist_data, file);
3899         if (ret)
3900                 goto out;
3901
3902         ret = create_key_fields(hist_data, file);
3903         if (ret)
3904                 goto out;
3905  out:
3906         free_var_defs(hist_data);
3907
3908         return ret;
3909 }
3910
3911 static int is_descending(struct trace_array *tr, const char *str)
3912 {
3913         if (!str)
3914                 return 0;
3915
3916         if (strcmp(str, "descending") == 0)
3917                 return 1;
3918
3919         if (strcmp(str, "ascending") == 0)
3920                 return 0;
3921
3922         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3923
3924         return -EINVAL;
3925 }
3926
3927 static int create_sort_keys(struct hist_trigger_data *hist_data)
3928 {
3929         struct trace_array *tr = hist_data->event_file->tr;
3930         char *fields_str = hist_data->attrs->sort_key_str;
3931         struct tracing_map_sort_key *sort_key;
3932         int descending, ret = 0;
3933         unsigned int i, j, k;
3934
3935         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3936
3937         if (!fields_str)
3938                 goto out;
3939
3940         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3941                 struct hist_field *hist_field;
3942                 char *field_str, *field_name;
3943                 const char *test_name;
3944
3945                 sort_key = &hist_data->sort_keys[i];
3946
3947                 field_str = strsep(&fields_str, ",");
3948                 if (!field_str)
3949                         break;
3950
3951                 if (!*field_str) {
3952                         ret = -EINVAL;
3953                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3954                         break;
3955                 }
3956
3957                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
3958                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
3959                         ret = -EINVAL;
3960                         break;
3961                 }
3962
3963                 field_name = strsep(&field_str, ".");
3964                 if (!field_name || !*field_name) {
3965                         ret = -EINVAL;
3966                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3967                         break;
3968                 }
3969
3970                 if (strcmp(field_name, "hitcount") == 0) {
3971                         descending = is_descending(tr, field_str);
3972                         if (descending < 0) {
3973                                 ret = descending;
3974                                 break;
3975                         }
3976                         sort_key->descending = descending;
3977                         continue;
3978                 }
3979
3980                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
3981                         unsigned int idx;
3982
3983                         hist_field = hist_data->fields[j];
3984                         if (hist_field->flags & HIST_FIELD_FL_VAR)
3985                                 continue;
3986
3987                         idx = k++;
3988
3989                         test_name = hist_field_name(hist_field, 0);
3990
3991                         if (strcmp(field_name, test_name) == 0) {
3992                                 sort_key->field_idx = idx;
3993                                 descending = is_descending(tr, field_str);
3994                                 if (descending < 0) {
3995                                         ret = descending;
3996                                         goto out;
3997                                 }
3998                                 sort_key->descending = descending;
3999                                 break;
4000                         }
4001                 }
4002                 if (j == hist_data->n_fields) {
4003                         ret = -EINVAL;
4004                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4005                         break;
4006                 }
4007         }
4008
4009         hist_data->n_sort_keys = i;
4010  out:
4011         return ret;
4012 }
4013
4014 static void destroy_actions(struct hist_trigger_data *hist_data)
4015 {
4016         unsigned int i;
4017
4018         for (i = 0; i < hist_data->n_actions; i++) {
4019                 struct action_data *data = hist_data->actions[i];
4020
4021                 if (data->handler == HANDLER_ONMATCH)
4022                         onmatch_destroy(data);
4023                 else if (data->handler == HANDLER_ONMAX ||
4024                          data->handler == HANDLER_ONCHANGE)
4025                         track_data_destroy(hist_data, data);
4026                 else
4027                         kfree(data);
4028         }
4029 }
4030
4031 static int parse_actions(struct hist_trigger_data *hist_data)
4032 {
4033         struct trace_array *tr = hist_data->event_file->tr;
4034         struct action_data *data;
4035         unsigned int i;
4036         int ret = 0;
4037         char *str;
4038         int len;
4039
4040         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4041                 str = hist_data->attrs->action_str[i];
4042
4043                 if ((len = str_has_prefix(str, "onmatch("))) {
4044                         char *action_str = str + len;
4045
4046                         data = onmatch_parse(tr, action_str);
4047                         if (IS_ERR(data)) {
4048                                 ret = PTR_ERR(data);
4049                                 break;
4050                         }
4051                 } else if ((len = str_has_prefix(str, "onmax("))) {
4052                         char *action_str = str + len;
4053
4054                         data = track_data_parse(hist_data, action_str,
4055                                                 HANDLER_ONMAX);
4056                         if (IS_ERR(data)) {
4057                                 ret = PTR_ERR(data);
4058                                 break;
4059                         }
4060                 } else if ((len = str_has_prefix(str, "onchange("))) {
4061                         char *action_str = str + len;
4062
4063                         data = track_data_parse(hist_data, action_str,
4064                                                 HANDLER_ONCHANGE);
4065                         if (IS_ERR(data)) {
4066                                 ret = PTR_ERR(data);
4067                                 break;
4068                         }
4069                 } else {
4070                         ret = -EINVAL;
4071                         break;
4072                 }
4073
4074                 hist_data->actions[hist_data->n_actions++] = data;
4075         }
4076
4077         return ret;
4078 }
4079
4080 static int create_actions(struct hist_trigger_data *hist_data)
4081 {
4082         struct action_data *data;
4083         unsigned int i;
4084         int ret = 0;
4085
4086         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4087                 data = hist_data->actions[i];
4088
4089                 if (data->handler == HANDLER_ONMATCH) {
4090                         ret = onmatch_create(hist_data, data);
4091                         if (ret)
4092                                 break;
4093                 } else if (data->handler == HANDLER_ONMAX ||
4094                            data->handler == HANDLER_ONCHANGE) {
4095                         ret = track_data_create(hist_data, data);
4096                         if (ret)
4097                                 break;
4098                 } else {
4099                         ret = -EINVAL;
4100                         break;
4101                 }
4102         }
4103
4104         return ret;
4105 }
4106
4107 static void print_actions(struct seq_file *m,
4108                           struct hist_trigger_data *hist_data,
4109                           struct tracing_map_elt *elt)
4110 {
4111         unsigned int i;
4112
4113         for (i = 0; i < hist_data->n_actions; i++) {
4114                 struct action_data *data = hist_data->actions[i];
4115
4116                 if (data->action == ACTION_SNAPSHOT)
4117                         continue;
4118
4119                 if (data->handler == HANDLER_ONMAX ||
4120                     data->handler == HANDLER_ONCHANGE)
4121                         track_data_print(m, hist_data, elt, data);
4122         }
4123 }
4124
4125 static void print_action_spec(struct seq_file *m,
4126                               struct hist_trigger_data *hist_data,
4127                               struct action_data *data)
4128 {
4129         unsigned int i;
4130
4131         if (data->action == ACTION_SAVE) {
4132                 for (i = 0; i < hist_data->n_save_vars; i++) {
4133                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4134                         if (i < hist_data->n_save_vars - 1)
4135                                 seq_puts(m, ",");
4136                 }
4137         } else if (data->action == ACTION_TRACE) {
4138                 if (data->use_trace_keyword)
4139                         seq_printf(m, "%s", data->synth_event_name);
4140                 for (i = 0; i < data->n_params; i++) {
4141                         if (i || data->use_trace_keyword)
4142                                 seq_puts(m, ",");
4143                         seq_printf(m, "%s", data->params[i]);
4144                 }
4145         }
4146 }
4147
4148 static void print_track_data_spec(struct seq_file *m,
4149                                   struct hist_trigger_data *hist_data,
4150                                   struct action_data *data)
4151 {
4152         if (data->handler == HANDLER_ONMAX)
4153                 seq_puts(m, ":onmax(");
4154         else if (data->handler == HANDLER_ONCHANGE)
4155                 seq_puts(m, ":onchange(");
4156         seq_printf(m, "%s", data->track_data.var_str);
4157         seq_printf(m, ").%s(", data->action_name);
4158
4159         print_action_spec(m, hist_data, data);
4160
4161         seq_puts(m, ")");
4162 }
4163
4164 static void print_onmatch_spec(struct seq_file *m,
4165                                struct hist_trigger_data *hist_data,
4166                                struct action_data *data)
4167 {
4168         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4169                    data->match_data.event);
4170
4171         seq_printf(m, "%s(", data->action_name);
4172
4173         print_action_spec(m, hist_data, data);
4174
4175         seq_puts(m, ")");
4176 }
4177
4178 static bool actions_match(struct hist_trigger_data *hist_data,
4179                           struct hist_trigger_data *hist_data_test)
4180 {
4181         unsigned int i, j;
4182
4183         if (hist_data->n_actions != hist_data_test->n_actions)
4184                 return false;
4185
4186         for (i = 0; i < hist_data->n_actions; i++) {
4187                 struct action_data *data = hist_data->actions[i];
4188                 struct action_data *data_test = hist_data_test->actions[i];
4189                 char *action_name, *action_name_test;
4190
4191                 if (data->handler != data_test->handler)
4192                         return false;
4193                 if (data->action != data_test->action)
4194                         return false;
4195
4196                 if (data->n_params != data_test->n_params)
4197                         return false;
4198
4199                 for (j = 0; j < data->n_params; j++) {
4200                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4201                                 return false;
4202                 }
4203
4204                 if (data->use_trace_keyword)
4205                         action_name = data->synth_event_name;
4206                 else
4207                         action_name = data->action_name;
4208
4209                 if (data_test->use_trace_keyword)
4210                         action_name_test = data_test->synth_event_name;
4211                 else
4212                         action_name_test = data_test->action_name;
4213
4214                 if (strcmp(action_name, action_name_test) != 0)
4215                         return false;
4216
4217                 if (data->handler == HANDLER_ONMATCH) {
4218                         if (strcmp(data->match_data.event_system,
4219                                    data_test->match_data.event_system) != 0)
4220                                 return false;
4221                         if (strcmp(data->match_data.event,
4222                                    data_test->match_data.event) != 0)
4223                                 return false;
4224                 } else if (data->handler == HANDLER_ONMAX ||
4225                            data->handler == HANDLER_ONCHANGE) {
4226                         if (strcmp(data->track_data.var_str,
4227                                    data_test->track_data.var_str) != 0)
4228                                 return false;
4229                 }
4230         }
4231
4232         return true;
4233 }
4234
4235
4236 static void print_actions_spec(struct seq_file *m,
4237                                struct hist_trigger_data *hist_data)
4238 {
4239         unsigned int i;
4240
4241         for (i = 0; i < hist_data->n_actions; i++) {
4242                 struct action_data *data = hist_data->actions[i];
4243
4244                 if (data->handler == HANDLER_ONMATCH)
4245                         print_onmatch_spec(m, hist_data, data);
4246                 else if (data->handler == HANDLER_ONMAX ||
4247                          data->handler == HANDLER_ONCHANGE)
4248                         print_track_data_spec(m, hist_data, data);
4249         }
4250 }
4251
4252 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4253 {
4254         unsigned int i;
4255
4256         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4257                 kfree(hist_data->field_var_hists[i]->cmd);
4258                 kfree(hist_data->field_var_hists[i]);
4259         }
4260 }
4261
4262 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4263 {
4264         if (!hist_data)
4265                 return;
4266
4267         destroy_hist_trigger_attrs(hist_data->attrs);
4268         destroy_hist_fields(hist_data);
4269         tracing_map_destroy(hist_data->map);
4270
4271         destroy_actions(hist_data);
4272         destroy_field_vars(hist_data);
4273         destroy_field_var_hists(hist_data);
4274
4275         kfree(hist_data);
4276 }
4277
4278 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4279 {
4280         struct tracing_map *map = hist_data->map;
4281         struct ftrace_event_field *field;
4282         struct hist_field *hist_field;
4283         int i, idx = 0;
4284
4285         for_each_hist_field(i, hist_data) {
4286                 hist_field = hist_data->fields[i];
4287                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4288                         tracing_map_cmp_fn_t cmp_fn;
4289
4290                         field = hist_field->field;
4291
4292                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4293                                 cmp_fn = tracing_map_cmp_none;
4294                         else if (!field)
4295                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4296                                                              hist_field->is_signed);
4297                         else if (is_string_field(field))
4298                                 cmp_fn = tracing_map_cmp_string;
4299                         else
4300                                 cmp_fn = tracing_map_cmp_num(field->size,
4301                                                              field->is_signed);
4302                         idx = tracing_map_add_key_field(map,
4303                                                         hist_field->offset,
4304                                                         cmp_fn);
4305                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4306                         idx = tracing_map_add_sum_field(map);
4307
4308                 if (idx < 0)
4309                         return idx;
4310
4311                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4312                         idx = tracing_map_add_var(map);
4313                         if (idx < 0)
4314                                 return idx;
4315                         hist_field->var.idx = idx;
4316                         hist_field->var.hist_data = hist_data;
4317                 }
4318         }
4319
4320         return 0;
4321 }
4322
4323 static struct hist_trigger_data *
4324 create_hist_data(unsigned int map_bits,
4325                  struct hist_trigger_attrs *attrs,
4326                  struct trace_event_file *file,
4327                  bool remove)
4328 {
4329         const struct tracing_map_ops *map_ops = NULL;
4330         struct hist_trigger_data *hist_data;
4331         int ret = 0;
4332
4333         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4334         if (!hist_data)
4335                 return ERR_PTR(-ENOMEM);
4336
4337         hist_data->attrs = attrs;
4338         hist_data->remove = remove;
4339         hist_data->event_file = file;
4340
4341         ret = parse_actions(hist_data);
4342         if (ret)
4343                 goto free;
4344
4345         ret = create_hist_fields(hist_data, file);
4346         if (ret)
4347                 goto free;
4348
4349         ret = create_sort_keys(hist_data);
4350         if (ret)
4351                 goto free;
4352
4353         map_ops = &hist_trigger_elt_data_ops;
4354
4355         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4356                                             map_ops, hist_data);
4357         if (IS_ERR(hist_data->map)) {
4358                 ret = PTR_ERR(hist_data->map);
4359                 hist_data->map = NULL;
4360                 goto free;
4361         }
4362
4363         ret = create_tracing_map_fields(hist_data);
4364         if (ret)
4365                 goto free;
4366  out:
4367         return hist_data;
4368  free:
4369         hist_data->attrs = NULL;
4370
4371         destroy_hist_data(hist_data);
4372
4373         hist_data = ERR_PTR(ret);
4374
4375         goto out;
4376 }
4377
4378 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4379                                     struct tracing_map_elt *elt, void *rec,
4380                                     struct ring_buffer_event *rbe,
4381                                     u64 *var_ref_vals)
4382 {
4383         struct hist_elt_data *elt_data;
4384         struct hist_field *hist_field;
4385         unsigned int i, var_idx;
4386         u64 hist_val;
4387
4388         elt_data = elt->private_data;
4389         elt_data->var_ref_vals = var_ref_vals;
4390
4391         for_each_hist_val_field(i, hist_data) {
4392                 hist_field = hist_data->fields[i];
4393                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4394                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4395                         var_idx = hist_field->var.idx;
4396                         tracing_map_set_var(elt, var_idx, hist_val);
4397                         continue;
4398                 }
4399                 tracing_map_update_sum(elt, i, hist_val);
4400         }
4401
4402         for_each_hist_key_field(i, hist_data) {
4403                 hist_field = hist_data->fields[i];
4404                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4405                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4406                         var_idx = hist_field->var.idx;
4407                         tracing_map_set_var(elt, var_idx, hist_val);
4408                 }
4409         }
4410
4411         update_field_vars(hist_data, elt, rbe, rec);
4412 }
4413
4414 static inline void add_to_key(char *compound_key, void *key,
4415                               struct hist_field *key_field, void *rec)
4416 {
4417         size_t size = key_field->size;
4418
4419         if (key_field->flags & HIST_FIELD_FL_STRING) {
4420                 struct ftrace_event_field *field;
4421
4422                 field = key_field->field;
4423                 if (field->filter_type == FILTER_DYN_STRING)
4424                         size = *(u32 *)(rec + field->offset) >> 16;
4425                 else if (field->filter_type == FILTER_PTR_STRING)
4426                         size = strlen(key);
4427                 else if (field->filter_type == FILTER_STATIC_STRING)
4428                         size = field->size;
4429
4430                 /* ensure NULL-termination */
4431                 if (size > key_field->size - 1)
4432                         size = key_field->size - 1;
4433
4434                 strncpy(compound_key + key_field->offset, (char *)key, size);
4435         } else
4436                 memcpy(compound_key + key_field->offset, key, size);
4437 }
4438
4439 static void
4440 hist_trigger_actions(struct hist_trigger_data *hist_data,
4441                      struct tracing_map_elt *elt, void *rec,
4442                      struct ring_buffer_event *rbe, void *key,
4443                      u64 *var_ref_vals)
4444 {
4445         struct action_data *data;
4446         unsigned int i;
4447
4448         for (i = 0; i < hist_data->n_actions; i++) {
4449                 data = hist_data->actions[i];
4450                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4451         }
4452 }
4453
4454 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4455                                struct ring_buffer_event *rbe)
4456 {
4457         struct hist_trigger_data *hist_data = data->private_data;
4458         bool use_compound_key = (hist_data->n_keys > 1);
4459         unsigned long entries[HIST_STACKTRACE_DEPTH];
4460         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4461         char compound_key[HIST_KEY_SIZE_MAX];
4462         struct tracing_map_elt *elt = NULL;
4463         struct hist_field *key_field;
4464         u64 field_contents;
4465         void *key = NULL;
4466         unsigned int i;
4467
4468         memset(compound_key, 0, hist_data->key_size);
4469
4470         for_each_hist_key_field(i, hist_data) {
4471                 key_field = hist_data->fields[i];
4472
4473                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4474                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4475                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4476                                          HIST_STACKTRACE_SKIP);
4477                         key = entries;
4478                 } else {
4479                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4480                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4481                                 key = (void *)(unsigned long)field_contents;
4482                                 use_compound_key = true;
4483                         } else
4484                                 key = (void *)&field_contents;
4485                 }
4486
4487                 if (use_compound_key)
4488                         add_to_key(compound_key, key, key_field, rec);
4489         }
4490
4491         if (use_compound_key)
4492                 key = compound_key;
4493
4494         if (hist_data->n_var_refs &&
4495             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4496                 return;
4497
4498         elt = tracing_map_insert(hist_data->map, key);
4499         if (!elt)
4500                 return;
4501
4502         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4503
4504         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4505                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
4506 }
4507
4508 static void hist_trigger_stacktrace_print(struct seq_file *m,
4509                                           unsigned long *stacktrace_entries,
4510                                           unsigned int max_entries)
4511 {
4512         char str[KSYM_SYMBOL_LEN];
4513         unsigned int spaces = 8;
4514         unsigned int i;
4515
4516         for (i = 0; i < max_entries; i++) {
4517                 if (!stacktrace_entries[i])
4518                         return;
4519
4520                 seq_printf(m, "%*c", 1 + spaces, ' ');
4521                 sprint_symbol(str, stacktrace_entries[i]);
4522                 seq_printf(m, "%s\n", str);
4523         }
4524 }
4525
4526 static void hist_trigger_print_key(struct seq_file *m,
4527                                    struct hist_trigger_data *hist_data,
4528                                    void *key,
4529                                    struct tracing_map_elt *elt)
4530 {
4531         struct hist_field *key_field;
4532         char str[KSYM_SYMBOL_LEN];
4533         bool multiline = false;
4534         const char *field_name;
4535         unsigned int i;
4536         u64 uval;
4537
4538         seq_puts(m, "{ ");
4539
4540         for_each_hist_key_field(i, hist_data) {
4541                 key_field = hist_data->fields[i];
4542
4543                 if (i > hist_data->n_vals)
4544                         seq_puts(m, ", ");
4545
4546                 field_name = hist_field_name(key_field, 0);
4547
4548                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4549                         uval = *(u64 *)(key + key_field->offset);
4550                         seq_printf(m, "%s: %llx", field_name, uval);
4551                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4552                         uval = *(u64 *)(key + key_field->offset);
4553                         sprint_symbol_no_offset(str, uval);
4554                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4555                                    uval, str);
4556                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4557                         uval = *(u64 *)(key + key_field->offset);
4558                         sprint_symbol(str, uval);
4559                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4560                                    uval, str);
4561                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4562                         struct hist_elt_data *elt_data = elt->private_data;
4563                         char *comm;
4564
4565                         if (WARN_ON_ONCE(!elt_data))
4566                                 return;
4567
4568                         comm = elt_data->comm;
4569
4570                         uval = *(u64 *)(key + key_field->offset);
4571                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4572                                    comm, uval);
4573                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4574                         const char *syscall_name;
4575
4576                         uval = *(u64 *)(key + key_field->offset);
4577                         syscall_name = get_syscall_name(uval);
4578                         if (!syscall_name)
4579                                 syscall_name = "unknown_syscall";
4580
4581                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4582                                    syscall_name, uval);
4583                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4584                         seq_puts(m, "stacktrace:\n");
4585                         hist_trigger_stacktrace_print(m,
4586                                                       key + key_field->offset,
4587                                                       HIST_STACKTRACE_DEPTH);
4588                         multiline = true;
4589                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4590                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4591                                    *(u64 *)(key + key_field->offset));
4592                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4593                         seq_printf(m, "%s: %-50s", field_name,
4594                                    (char *)(key + key_field->offset));
4595                 } else {
4596                         uval = *(u64 *)(key + key_field->offset);
4597                         seq_printf(m, "%s: %10llu", field_name, uval);
4598                 }
4599         }
4600
4601         if (!multiline)
4602                 seq_puts(m, " ");
4603
4604         seq_puts(m, "}");
4605 }
4606
4607 static void hist_trigger_entry_print(struct seq_file *m,
4608                                      struct hist_trigger_data *hist_data,
4609                                      void *key,
4610                                      struct tracing_map_elt *elt)
4611 {
4612         const char *field_name;
4613         unsigned int i;
4614
4615         hist_trigger_print_key(m, hist_data, key, elt);
4616
4617         seq_printf(m, " hitcount: %10llu",
4618                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4619
4620         for (i = 1; i < hist_data->n_vals; i++) {
4621                 field_name = hist_field_name(hist_data->fields[i], 0);
4622
4623                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4624                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4625                         continue;
4626
4627                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4628                         seq_printf(m, "  %s: %10llx", field_name,
4629                                    tracing_map_read_sum(elt, i));
4630                 } else {
4631                         seq_printf(m, "  %s: %10llu", field_name,
4632                                    tracing_map_read_sum(elt, i));
4633                 }
4634         }
4635
4636         print_actions(m, hist_data, elt);
4637
4638         seq_puts(m, "\n");
4639 }
4640
4641 static int print_entries(struct seq_file *m,
4642                          struct hist_trigger_data *hist_data)
4643 {
4644         struct tracing_map_sort_entry **sort_entries = NULL;
4645         struct tracing_map *map = hist_data->map;
4646         int i, n_entries;
4647
4648         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4649                                              hist_data->n_sort_keys,
4650                                              &sort_entries);
4651         if (n_entries < 0)
4652                 return n_entries;
4653
4654         for (i = 0; i < n_entries; i++)
4655                 hist_trigger_entry_print(m, hist_data,
4656                                          sort_entries[i]->key,
4657                                          sort_entries[i]->elt);
4658
4659         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4660
4661         return n_entries;
4662 }
4663
4664 static void hist_trigger_show(struct seq_file *m,
4665                               struct event_trigger_data *data, int n)
4666 {
4667         struct hist_trigger_data *hist_data;
4668         int n_entries;
4669
4670         if (n > 0)
4671                 seq_puts(m, "\n\n");
4672
4673         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4674         data->ops->print(m, data->ops, data);
4675         seq_puts(m, "#\n\n");
4676
4677         hist_data = data->private_data;
4678         n_entries = print_entries(m, hist_data);
4679         if (n_entries < 0)
4680                 n_entries = 0;
4681
4682         track_data_snapshot_print(m, hist_data);
4683
4684         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4685                    (u64)atomic64_read(&hist_data->map->hits),
4686                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4687 }
4688
4689 static int hist_show(struct seq_file *m, void *v)
4690 {
4691         struct event_trigger_data *data;
4692         struct trace_event_file *event_file;
4693         int n = 0, ret = 0;
4694
4695         mutex_lock(&event_mutex);
4696
4697         event_file = event_file_data(m->private);
4698         if (unlikely(!event_file)) {
4699                 ret = -ENODEV;
4700                 goto out_unlock;
4701         }
4702
4703         list_for_each_entry(data, &event_file->triggers, list) {
4704                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4705                         hist_trigger_show(m, data, n++);
4706         }
4707
4708  out_unlock:
4709         mutex_unlock(&event_mutex);
4710
4711         return ret;
4712 }
4713
4714 static int event_hist_open(struct inode *inode, struct file *file)
4715 {
4716         int ret;
4717
4718         ret = security_locked_down(LOCKDOWN_TRACEFS);
4719         if (ret)
4720                 return ret;
4721
4722         return single_open(file, hist_show, file);
4723 }
4724
4725 const struct file_operations event_hist_fops = {
4726         .open = event_hist_open,
4727         .read = seq_read,
4728         .llseek = seq_lseek,
4729         .release = single_release,
4730 };
4731
4732 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4733 static void hist_field_debug_show_flags(struct seq_file *m,
4734                                         unsigned long flags)
4735 {
4736         seq_puts(m, "      flags:\n");
4737
4738         if (flags & HIST_FIELD_FL_KEY)
4739                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4740         else if (flags & HIST_FIELD_FL_HITCOUNT)
4741                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4742         else if (flags & HIST_FIELD_FL_VAR)
4743                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4744         else if (flags & HIST_FIELD_FL_VAR_REF)
4745                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4746         else
4747                 seq_puts(m, "        VAL: normal u64 value\n");
4748
4749         if (flags & HIST_FIELD_FL_ALIAS)
4750                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4751 }
4752
4753 static int hist_field_debug_show(struct seq_file *m,
4754                                  struct hist_field *field, unsigned long flags)
4755 {
4756         if ((field->flags & flags) != flags) {
4757                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4758                 return -EINVAL;
4759         }
4760
4761         hist_field_debug_show_flags(m, field->flags);
4762         if (field->field)
4763                 seq_printf(m, "      ftrace_event_field name: %s\n",
4764                            field->field->name);
4765
4766         if (field->flags & HIST_FIELD_FL_VAR) {
4767                 seq_printf(m, "      var.name: %s\n", field->var.name);
4768                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4769                            field->var.idx);
4770         }
4771
4772         if (field->flags & HIST_FIELD_FL_ALIAS)
4773                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4774                            field->var_ref_idx);
4775
4776         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4777                 seq_printf(m, "      name: %s\n", field->name);
4778                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4779                            field->var.idx);
4780                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4781                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4782                            field->var_ref_idx);
4783                 if (field->system)
4784                         seq_printf(m, "      system: %s\n", field->system);
4785                 if (field->event_name)
4786                         seq_printf(m, "      event_name: %s\n", field->event_name);
4787         }
4788
4789         seq_printf(m, "      type: %s\n", field->type);
4790         seq_printf(m, "      size: %u\n", field->size);
4791         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4792
4793         return 0;
4794 }
4795
4796 static int field_var_debug_show(struct seq_file *m,
4797                                 struct field_var *field_var, unsigned int i,
4798                                 bool save_vars)
4799 {
4800         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4801         struct hist_field *field;
4802         int ret = 0;
4803
4804         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4805
4806         field = field_var->var;
4807
4808         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4809
4810         hist_field_debug_show_flags(m, field->flags);
4811         seq_printf(m, "      var.name: %s\n", field->var.name);
4812         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4813                    field->var.idx);
4814
4815         field = field_var->val;
4816
4817         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4818         if (field->field)
4819                 seq_printf(m, "      ftrace_event_field name: %s\n",
4820                            field->field->name);
4821         else {
4822                 ret = -EINVAL;
4823                 goto out;
4824         }
4825
4826         seq_printf(m, "      type: %s\n", field->type);
4827         seq_printf(m, "      size: %u\n", field->size);
4828         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4829 out:
4830         return ret;
4831 }
4832
4833 static int hist_action_debug_show(struct seq_file *m,
4834                                   struct action_data *data, int i)
4835 {
4836         int ret = 0;
4837
4838         if (data->handler == HANDLER_ONMAX ||
4839             data->handler == HANDLER_ONCHANGE) {
4840                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4841                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4842                                             HIST_FIELD_FL_VAR_REF);
4843                 if (ret)
4844                         goto out;
4845
4846                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4847                 ret = hist_field_debug_show(m, data->track_data.track_var,
4848                                             HIST_FIELD_FL_VAR);
4849                 if (ret)
4850                         goto out;
4851         }
4852
4853         if (data->handler == HANDLER_ONMATCH) {
4854                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4855                            i, data->match_data.event_system);
4856                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4857                            i, data->match_data.event);
4858         }
4859 out:
4860         return ret;
4861 }
4862
4863 static int hist_actions_debug_show(struct seq_file *m,
4864                                    struct hist_trigger_data *hist_data)
4865 {
4866         int i, ret = 0;
4867
4868         if (hist_data->n_actions)
4869                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4870
4871         for (i = 0; i < hist_data->n_actions; i++) {
4872                 struct action_data *action = hist_data->actions[i];
4873
4874                 ret = hist_action_debug_show(m, action, i);
4875                 if (ret)
4876                         goto out;
4877         }
4878
4879         if (hist_data->n_save_vars)
4880                 seq_puts(m, "\n  save action variables (save() params):\n");
4881
4882         for (i = 0; i < hist_data->n_save_vars; i++) {
4883                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4884                 if (ret)
4885                         goto out;
4886         }
4887 out:
4888         return ret;
4889 }
4890
4891 static void hist_trigger_debug_show(struct seq_file *m,
4892                                     struct event_trigger_data *data, int n)
4893 {
4894         struct hist_trigger_data *hist_data;
4895         int i, ret;
4896
4897         if (n > 0)
4898                 seq_puts(m, "\n\n");
4899
4900         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4901         data->ops->print(m, data->ops, data);
4902         seq_puts(m, "#\n\n");
4903
4904         hist_data = data->private_data;
4905
4906         seq_printf(m, "hist_data: %p\n\n", hist_data);
4907         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4908         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4909         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4910
4911         seq_puts(m, "\n  val fields:\n\n");
4912
4913         seq_puts(m, "    hist_data->fields[0]:\n");
4914         ret = hist_field_debug_show(m, hist_data->fields[0],
4915                                     HIST_FIELD_FL_HITCOUNT);
4916         if (ret)
4917                 return;
4918
4919         for (i = 1; i < hist_data->n_vals; i++) {
4920                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4921                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4922                 if (ret)
4923                         return;
4924         }
4925
4926         seq_puts(m, "\n  key fields:\n");
4927
4928         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
4929                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4930                 ret = hist_field_debug_show(m, hist_data->fields[i],
4931                                             HIST_FIELD_FL_KEY);
4932                 if (ret)
4933                         return;
4934         }
4935
4936         if (hist_data->n_var_refs)
4937                 seq_puts(m, "\n  variable reference fields:\n");
4938
4939         for (i = 0; i < hist_data->n_var_refs; i++) {
4940                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
4941                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
4942                                             HIST_FIELD_FL_VAR_REF);
4943                 if (ret)
4944                         return;
4945         }
4946
4947         if (hist_data->n_field_vars)
4948                 seq_puts(m, "\n  field variables:\n");
4949
4950         for (i = 0; i < hist_data->n_field_vars; i++) {
4951                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
4952                 if (ret)
4953                         return;
4954         }
4955
4956         ret = hist_actions_debug_show(m, hist_data);
4957         if (ret)
4958                 return;
4959 }
4960
4961 static int hist_debug_show(struct seq_file *m, void *v)
4962 {
4963         struct event_trigger_data *data;
4964         struct trace_event_file *event_file;
4965         int n = 0, ret = 0;
4966
4967         mutex_lock(&event_mutex);
4968
4969         event_file = event_file_data(m->private);
4970         if (unlikely(!event_file)) {
4971                 ret = -ENODEV;
4972                 goto out_unlock;
4973         }
4974
4975         list_for_each_entry(data, &event_file->triggers, list) {
4976                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4977                         hist_trigger_debug_show(m, data, n++);
4978         }
4979
4980  out_unlock:
4981         mutex_unlock(&event_mutex);
4982
4983         return ret;
4984 }
4985
4986 static int event_hist_debug_open(struct inode *inode, struct file *file)
4987 {
4988         int ret;
4989
4990         ret = security_locked_down(LOCKDOWN_TRACEFS);
4991         if (ret)
4992                 return ret;
4993
4994         return single_open(file, hist_debug_show, file);
4995 }
4996
4997 const struct file_operations event_hist_debug_fops = {
4998         .open = event_hist_debug_open,
4999         .read = seq_read,
5000         .llseek = seq_lseek,
5001         .release = single_release,
5002 };
5003 #endif
5004
5005 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5006 {
5007         const char *field_name = hist_field_name(hist_field, 0);
5008
5009         if (hist_field->var.name)
5010                 seq_printf(m, "%s=", hist_field->var.name);
5011
5012         if (hist_field->flags & HIST_FIELD_FL_CPU)
5013                 seq_puts(m, "cpu");
5014         else if (field_name) {
5015                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5016                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5017                         seq_putc(m, '$');
5018                 seq_printf(m, "%s", field_name);
5019         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5020                 seq_puts(m, "common_timestamp");
5021
5022         if (hist_field->flags) {
5023                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5024                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5025                         const char *flags = get_hist_field_flags(hist_field);
5026
5027                         if (flags)
5028                                 seq_printf(m, ".%s", flags);
5029                 }
5030         }
5031 }
5032
5033 static int event_hist_trigger_print(struct seq_file *m,
5034                                     struct event_trigger_ops *ops,
5035                                     struct event_trigger_data *data)
5036 {
5037         struct hist_trigger_data *hist_data = data->private_data;
5038         struct hist_field *field;
5039         bool have_var = false;
5040         unsigned int i;
5041
5042         seq_puts(m, "hist:");
5043
5044         if (data->name)
5045                 seq_printf(m, "%s:", data->name);
5046
5047         seq_puts(m, "keys=");
5048
5049         for_each_hist_key_field(i, hist_data) {
5050                 field = hist_data->fields[i];
5051
5052                 if (i > hist_data->n_vals)
5053                         seq_puts(m, ",");
5054
5055                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5056                         seq_puts(m, "stacktrace");
5057                 else
5058                         hist_field_print(m, field);
5059         }
5060
5061         seq_puts(m, ":vals=");
5062
5063         for_each_hist_val_field(i, hist_data) {
5064                 field = hist_data->fields[i];
5065                 if (field->flags & HIST_FIELD_FL_VAR) {
5066                         have_var = true;
5067                         continue;
5068                 }
5069
5070                 if (i == HITCOUNT_IDX)
5071                         seq_puts(m, "hitcount");
5072                 else {
5073                         seq_puts(m, ",");
5074                         hist_field_print(m, field);
5075                 }
5076         }
5077
5078         if (have_var) {
5079                 unsigned int n = 0;
5080
5081                 seq_puts(m, ":");
5082
5083                 for_each_hist_val_field(i, hist_data) {
5084                         field = hist_data->fields[i];
5085
5086                         if (field->flags & HIST_FIELD_FL_VAR) {
5087                                 if (n++)
5088                                         seq_puts(m, ",");
5089                                 hist_field_print(m, field);
5090                         }
5091                 }
5092         }
5093
5094         seq_puts(m, ":sort=");
5095
5096         for (i = 0; i < hist_data->n_sort_keys; i++) {
5097                 struct tracing_map_sort_key *sort_key;
5098                 unsigned int idx, first_key_idx;
5099
5100                 /* skip VAR vals */
5101                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5102
5103                 sort_key = &hist_data->sort_keys[i];
5104                 idx = sort_key->field_idx;
5105
5106                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5107                         return -EINVAL;
5108
5109                 if (i > 0)
5110                         seq_puts(m, ",");
5111
5112                 if (idx == HITCOUNT_IDX)
5113                         seq_puts(m, "hitcount");
5114                 else {
5115                         if (idx >= first_key_idx)
5116                                 idx += hist_data->n_vars;
5117                         hist_field_print(m, hist_data->fields[idx]);
5118                 }
5119
5120                 if (sort_key->descending)
5121                         seq_puts(m, ".descending");
5122         }
5123         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5124         if (hist_data->enable_timestamps)
5125                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5126
5127         print_actions_spec(m, hist_data);
5128
5129         if (data->filter_str)
5130                 seq_printf(m, " if %s", data->filter_str);
5131
5132         if (data->paused)
5133                 seq_puts(m, " [paused]");
5134         else
5135                 seq_puts(m, " [active]");
5136
5137         seq_putc(m, '\n');
5138
5139         return 0;
5140 }
5141
5142 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5143                                    struct event_trigger_data *data)
5144 {
5145         struct hist_trigger_data *hist_data = data->private_data;
5146
5147         if (!data->ref && hist_data->attrs->name)
5148                 save_named_trigger(hist_data->attrs->name, data);
5149
5150         data->ref++;
5151
5152         return 0;
5153 }
5154
5155 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5156 {
5157         struct trace_event_file *file;
5158         unsigned int i;
5159         char *cmd;
5160         int ret;
5161
5162         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5163                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5164                 cmd = hist_data->field_var_hists[i]->cmd;
5165                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5166                                               "!hist", "hist", cmd);
5167         }
5168 }
5169
5170 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5171                                     struct event_trigger_data *data)
5172 {
5173         struct hist_trigger_data *hist_data = data->private_data;
5174
5175         if (WARN_ON_ONCE(data->ref <= 0))
5176                 return;
5177
5178         data->ref--;
5179         if (!data->ref) {
5180                 if (data->name)
5181                         del_named_trigger(data);
5182
5183                 trigger_data_free(data);
5184
5185                 remove_hist_vars(hist_data);
5186
5187                 unregister_field_var_hists(hist_data);
5188
5189                 destroy_hist_data(hist_data);
5190         }
5191 }
5192
5193 static struct event_trigger_ops event_hist_trigger_ops = {
5194         .func                   = event_hist_trigger,
5195         .print                  = event_hist_trigger_print,
5196         .init                   = event_hist_trigger_init,
5197         .free                   = event_hist_trigger_free,
5198 };
5199
5200 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5201                                          struct event_trigger_data *data)
5202 {
5203         data->ref++;
5204
5205         save_named_trigger(data->named_data->name, data);
5206
5207         event_hist_trigger_init(ops, data->named_data);
5208
5209         return 0;
5210 }
5211
5212 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5213                                           struct event_trigger_data *data)
5214 {
5215         if (WARN_ON_ONCE(data->ref <= 0))
5216                 return;
5217
5218         event_hist_trigger_free(ops, data->named_data);
5219
5220         data->ref--;
5221         if (!data->ref) {
5222                 del_named_trigger(data);
5223                 trigger_data_free(data);
5224         }
5225 }
5226
5227 static struct event_trigger_ops event_hist_trigger_named_ops = {
5228         .func                   = event_hist_trigger,
5229         .print                  = event_hist_trigger_print,
5230         .init                   = event_hist_trigger_named_init,
5231         .free                   = event_hist_trigger_named_free,
5232 };
5233
5234 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5235                                                             char *param)
5236 {
5237         return &event_hist_trigger_ops;
5238 }
5239
5240 static void hist_clear(struct event_trigger_data *data)
5241 {
5242         struct hist_trigger_data *hist_data = data->private_data;
5243
5244         if (data->name)
5245                 pause_named_trigger(data);
5246
5247         tracepoint_synchronize_unregister();
5248
5249         tracing_map_clear(hist_data->map);
5250
5251         if (data->name)
5252                 unpause_named_trigger(data);
5253 }
5254
5255 static bool compatible_field(struct ftrace_event_field *field,
5256                              struct ftrace_event_field *test_field)
5257 {
5258         if (field == test_field)
5259                 return true;
5260         if (field == NULL || test_field == NULL)
5261                 return false;
5262         if (strcmp(field->name, test_field->name) != 0)
5263                 return false;
5264         if (strcmp(field->type, test_field->type) != 0)
5265                 return false;
5266         if (field->size != test_field->size)
5267                 return false;
5268         if (field->is_signed != test_field->is_signed)
5269                 return false;
5270
5271         return true;
5272 }
5273
5274 static bool hist_trigger_match(struct event_trigger_data *data,
5275                                struct event_trigger_data *data_test,
5276                                struct event_trigger_data *named_data,
5277                                bool ignore_filter)
5278 {
5279         struct tracing_map_sort_key *sort_key, *sort_key_test;
5280         struct hist_trigger_data *hist_data, *hist_data_test;
5281         struct hist_field *key_field, *key_field_test;
5282         unsigned int i;
5283
5284         if (named_data && (named_data != data_test) &&
5285             (named_data != data_test->named_data))
5286                 return false;
5287
5288         if (!named_data && is_named_trigger(data_test))
5289                 return false;
5290
5291         hist_data = data->private_data;
5292         hist_data_test = data_test->private_data;
5293
5294         if (hist_data->n_vals != hist_data_test->n_vals ||
5295             hist_data->n_fields != hist_data_test->n_fields ||
5296             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5297                 return false;
5298
5299         if (!ignore_filter) {
5300                 if ((data->filter_str && !data_test->filter_str) ||
5301                    (!data->filter_str && data_test->filter_str))
5302                         return false;
5303         }
5304
5305         for_each_hist_field(i, hist_data) {
5306                 key_field = hist_data->fields[i];
5307                 key_field_test = hist_data_test->fields[i];
5308
5309                 if (key_field->flags != key_field_test->flags)
5310                         return false;
5311                 if (!compatible_field(key_field->field, key_field_test->field))
5312                         return false;
5313                 if (key_field->offset != key_field_test->offset)
5314                         return false;
5315                 if (key_field->size != key_field_test->size)
5316                         return false;
5317                 if (key_field->is_signed != key_field_test->is_signed)
5318                         return false;
5319                 if (!!key_field->var.name != !!key_field_test->var.name)
5320                         return false;
5321                 if (key_field->var.name &&
5322                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5323                         return false;
5324         }
5325
5326         for (i = 0; i < hist_data->n_sort_keys; i++) {
5327                 sort_key = &hist_data->sort_keys[i];
5328                 sort_key_test = &hist_data_test->sort_keys[i];
5329
5330                 if (sort_key->field_idx != sort_key_test->field_idx ||
5331                     sort_key->descending != sort_key_test->descending)
5332                         return false;
5333         }
5334
5335         if (!ignore_filter && data->filter_str &&
5336             (strcmp(data->filter_str, data_test->filter_str) != 0))
5337                 return false;
5338
5339         if (!actions_match(hist_data, hist_data_test))
5340                 return false;
5341
5342         return true;
5343 }
5344
5345 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5346                                  struct event_trigger_data *data,
5347                                  struct trace_event_file *file)
5348 {
5349         struct hist_trigger_data *hist_data = data->private_data;
5350         struct event_trigger_data *test, *named_data = NULL;
5351         struct trace_array *tr = file->tr;
5352         int ret = 0;
5353
5354         if (hist_data->attrs->name) {
5355                 named_data = find_named_trigger(hist_data->attrs->name);
5356                 if (named_data) {
5357                         if (!hist_trigger_match(data, named_data, named_data,
5358                                                 true)) {
5359                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5360                                 ret = -EINVAL;
5361                                 goto out;
5362                         }
5363                 }
5364         }
5365
5366         if (hist_data->attrs->name && !named_data)
5367                 goto new;
5368
5369         lockdep_assert_held(&event_mutex);
5370
5371         list_for_each_entry(test, &file->triggers, list) {
5372                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5373                         if (!hist_trigger_match(data, test, named_data, false))
5374                                 continue;
5375                         if (hist_data->attrs->pause)
5376                                 test->paused = true;
5377                         else if (hist_data->attrs->cont)
5378                                 test->paused = false;
5379                         else if (hist_data->attrs->clear)
5380                                 hist_clear(test);
5381                         else {
5382                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5383                                 ret = -EEXIST;
5384                         }
5385                         goto out;
5386                 }
5387         }
5388  new:
5389         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5390                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5391                 ret = -ENOENT;
5392                 goto out;
5393         }
5394
5395         if (hist_data->attrs->pause)
5396                 data->paused = true;
5397
5398         if (named_data) {
5399                 data->private_data = named_data->private_data;
5400                 set_named_trigger_data(data, named_data);
5401                 data->ops = &event_hist_trigger_named_ops;
5402         }
5403
5404         if (data->ops->init) {
5405                 ret = data->ops->init(data->ops, data);
5406                 if (ret < 0)
5407                         goto out;
5408         }
5409
5410         if (hist_data->enable_timestamps) {
5411                 char *clock = hist_data->attrs->clock;
5412
5413                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5414                 if (ret) {
5415                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5416                         goto out;
5417                 }
5418
5419                 tracing_set_time_stamp_abs(file->tr, true);
5420         }
5421
5422         if (named_data)
5423                 destroy_hist_data(hist_data);
5424
5425         ret++;
5426  out:
5427         return ret;
5428 }
5429
5430 static int hist_trigger_enable(struct event_trigger_data *data,
5431                                struct trace_event_file *file)
5432 {
5433         int ret = 0;
5434
5435         list_add_tail_rcu(&data->list, &file->triggers);
5436
5437         update_cond_flag(file);
5438
5439         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5440                 list_del_rcu(&data->list);
5441                 update_cond_flag(file);
5442                 ret--;
5443         }
5444
5445         return ret;
5446 }
5447
5448 static bool have_hist_trigger_match(struct event_trigger_data *data,
5449                                     struct trace_event_file *file)
5450 {
5451         struct hist_trigger_data *hist_data = data->private_data;
5452         struct event_trigger_data *test, *named_data = NULL;
5453         bool match = false;
5454
5455         lockdep_assert_held(&event_mutex);
5456
5457         if (hist_data->attrs->name)
5458                 named_data = find_named_trigger(hist_data->attrs->name);
5459
5460         list_for_each_entry(test, &file->triggers, list) {
5461                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5462                         if (hist_trigger_match(data, test, named_data, false)) {
5463                                 match = true;
5464                                 break;
5465                         }
5466                 }
5467         }
5468
5469         return match;
5470 }
5471
5472 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5473                                     struct trace_event_file *file)
5474 {
5475         struct hist_trigger_data *hist_data = data->private_data;
5476         struct event_trigger_data *test, *named_data = NULL;
5477
5478         lockdep_assert_held(&event_mutex);
5479
5480         if (hist_data->attrs->name)
5481                 named_data = find_named_trigger(hist_data->attrs->name);
5482
5483         list_for_each_entry(test, &file->triggers, list) {
5484                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5485                         if (!hist_trigger_match(data, test, named_data, false))
5486                                 continue;
5487                         hist_data = test->private_data;
5488                         if (check_var_refs(hist_data))
5489                                 return true;
5490                         break;
5491                 }
5492         }
5493
5494         return false;
5495 }
5496
5497 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5498                                     struct event_trigger_data *data,
5499                                     struct trace_event_file *file)
5500 {
5501         struct hist_trigger_data *hist_data = data->private_data;
5502         struct event_trigger_data *test, *named_data = NULL;
5503         bool unregistered = false;
5504
5505         lockdep_assert_held(&event_mutex);
5506
5507         if (hist_data->attrs->name)
5508                 named_data = find_named_trigger(hist_data->attrs->name);
5509
5510         list_for_each_entry(test, &file->triggers, list) {
5511                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5512                         if (!hist_trigger_match(data, test, named_data, false))
5513                                 continue;
5514                         unregistered = true;
5515                         list_del_rcu(&test->list);
5516                         trace_event_trigger_enable_disable(file, 0);
5517                         update_cond_flag(file);
5518                         break;
5519                 }
5520         }
5521
5522         if (unregistered && test->ops->free)
5523                 test->ops->free(test->ops, test);
5524
5525         if (hist_data->enable_timestamps) {
5526                 if (!hist_data->remove || unregistered)
5527                         tracing_set_time_stamp_abs(file->tr, false);
5528         }
5529 }
5530
5531 static bool hist_file_check_refs(struct trace_event_file *file)
5532 {
5533         struct hist_trigger_data *hist_data;
5534         struct event_trigger_data *test;
5535
5536         lockdep_assert_held(&event_mutex);
5537
5538         list_for_each_entry(test, &file->triggers, list) {
5539                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5540                         hist_data = test->private_data;
5541                         if (check_var_refs(hist_data))
5542                                 return true;
5543                 }
5544         }
5545
5546         return false;
5547 }
5548
5549 static void hist_unreg_all(struct trace_event_file *file)
5550 {
5551         struct event_trigger_data *test, *n;
5552         struct hist_trigger_data *hist_data;
5553         struct synth_event *se;
5554         const char *se_name;
5555
5556         lockdep_assert_held(&event_mutex);
5557
5558         if (hist_file_check_refs(file))
5559                 return;
5560
5561         list_for_each_entry_safe(test, n, &file->triggers, list) {
5562                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5563                         hist_data = test->private_data;
5564                         list_del_rcu(&test->list);
5565                         trace_event_trigger_enable_disable(file, 0);
5566
5567                         se_name = trace_event_name(file->event_call);
5568                         se = find_synth_event(se_name);
5569                         if (se)
5570                                 se->ref--;
5571
5572                         update_cond_flag(file);
5573                         if (hist_data->enable_timestamps)
5574                                 tracing_set_time_stamp_abs(file->tr, false);
5575                         if (test->ops->free)
5576                                 test->ops->free(test->ops, test);
5577                 }
5578         }
5579 }
5580
5581 static int event_hist_trigger_func(struct event_command *cmd_ops,
5582                                    struct trace_event_file *file,
5583                                    char *glob, char *cmd, char *param)
5584 {
5585         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5586         struct event_trigger_data *trigger_data;
5587         struct hist_trigger_attrs *attrs;
5588         struct event_trigger_ops *trigger_ops;
5589         struct hist_trigger_data *hist_data;
5590         struct synth_event *se;
5591         const char *se_name;
5592         bool remove = false;
5593         char *trigger, *p;
5594         int ret = 0;
5595
5596         lockdep_assert_held(&event_mutex);
5597
5598         if (glob && strlen(glob)) {
5599                 hist_err_clear();
5600                 last_cmd_set(file, param);
5601         }
5602
5603         if (!param)
5604                 return -EINVAL;
5605
5606         if (glob[0] == '!')
5607                 remove = true;
5608
5609         /*
5610          * separate the trigger from the filter (k:v [if filter])
5611          * allowing for whitespace in the trigger
5612          */
5613         p = trigger = param;
5614         do {
5615                 p = strstr(p, "if");
5616                 if (!p)
5617                         break;
5618                 if (p == param)
5619                         return -EINVAL;
5620                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5621                         p++;
5622                         continue;
5623                 }
5624                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5625                         return -EINVAL;
5626                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5627                         p++;
5628                         continue;
5629                 }
5630                 break;
5631         } while (p);
5632
5633         if (!p)
5634                 param = NULL;
5635         else {
5636                 *(p - 1) = '\0';
5637                 param = strstrip(p);
5638                 trigger = strstrip(trigger);
5639         }
5640
5641         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5642         if (IS_ERR(attrs))
5643                 return PTR_ERR(attrs);
5644
5645         if (attrs->map_bits)
5646                 hist_trigger_bits = attrs->map_bits;
5647
5648         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5649         if (IS_ERR(hist_data)) {
5650                 destroy_hist_trigger_attrs(attrs);
5651                 return PTR_ERR(hist_data);
5652         }
5653
5654         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5655
5656         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5657         if (!trigger_data) {
5658                 ret = -ENOMEM;
5659                 goto out_free;
5660         }
5661
5662         trigger_data->count = -1;
5663         trigger_data->ops = trigger_ops;
5664         trigger_data->cmd_ops = cmd_ops;
5665
5666         INIT_LIST_HEAD(&trigger_data->list);
5667         RCU_INIT_POINTER(trigger_data->filter, NULL);
5668
5669         trigger_data->private_data = hist_data;
5670
5671         /* if param is non-empty, it's supposed to be a filter */
5672         if (param && cmd_ops->set_filter) {
5673                 ret = cmd_ops->set_filter(param, trigger_data, file);
5674                 if (ret < 0)
5675                         goto out_free;
5676         }
5677
5678         if (remove) {
5679                 if (!have_hist_trigger_match(trigger_data, file))
5680                         goto out_free;
5681
5682                 if (hist_trigger_check_refs(trigger_data, file)) {
5683                         ret = -EBUSY;
5684                         goto out_free;
5685                 }
5686
5687                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5688                 se_name = trace_event_name(file->event_call);
5689                 se = find_synth_event(se_name);
5690                 if (se)
5691                         se->ref--;
5692                 ret = 0;
5693                 goto out_free;
5694         }
5695
5696         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5697         /*
5698          * The above returns on success the # of triggers registered,
5699          * but if it didn't register any it returns zero.  Consider no
5700          * triggers registered a failure too.
5701          */
5702         if (!ret) {
5703                 if (!(attrs->pause || attrs->cont || attrs->clear))
5704                         ret = -ENOENT;
5705                 goto out_free;
5706         } else if (ret < 0)
5707                 goto out_free;
5708
5709         if (get_named_trigger_data(trigger_data))
5710                 goto enable;
5711
5712         if (has_hist_vars(hist_data))
5713                 save_hist_vars(hist_data);
5714
5715         ret = create_actions(hist_data);
5716         if (ret)
5717                 goto out_unreg;
5718
5719         ret = tracing_map_init(hist_data->map);
5720         if (ret)
5721                 goto out_unreg;
5722 enable:
5723         ret = hist_trigger_enable(trigger_data, file);
5724         if (ret)
5725                 goto out_unreg;
5726
5727         se_name = trace_event_name(file->event_call);
5728         se = find_synth_event(se_name);
5729         if (se)
5730                 se->ref++;
5731         /* Just return zero, not the number of registered triggers */
5732         ret = 0;
5733  out:
5734         if (ret == 0)
5735                 hist_err_clear();
5736
5737         return ret;
5738  out_unreg:
5739         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5740  out_free:
5741         if (cmd_ops->set_filter)
5742                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5743
5744         remove_hist_vars(hist_data);
5745
5746         kfree(trigger_data);
5747
5748         destroy_hist_data(hist_data);
5749         goto out;
5750 }
5751
5752 static struct event_command trigger_hist_cmd = {
5753         .name                   = "hist",
5754         .trigger_type           = ETT_EVENT_HIST,
5755         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5756         .func                   = event_hist_trigger_func,
5757         .reg                    = hist_register_trigger,
5758         .unreg                  = hist_unregister_trigger,
5759         .unreg_all              = hist_unreg_all,
5760         .get_trigger_ops        = event_hist_get_trigger_ops,
5761         .set_filter             = set_trigger_filter,
5762 };
5763
5764 __init int register_trigger_hist_cmd(void)
5765 {
5766         int ret;
5767
5768         ret = register_event_command(&trigger_hist_cmd);
5769         WARN_ON(ret < 0);
5770
5771         return ret;
5772 }
5773
5774 static void
5775 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5776                     struct ring_buffer_event *event)
5777 {
5778         struct enable_trigger_data *enable_data = data->private_data;
5779         struct event_trigger_data *test;
5780
5781         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5782                                 lockdep_is_held(&event_mutex)) {
5783                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5784                         if (enable_data->enable)
5785                                 test->paused = false;
5786                         else
5787                                 test->paused = true;
5788                 }
5789         }
5790 }
5791
5792 static void
5793 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5794                           struct ring_buffer_event *event)
5795 {
5796         if (!data->count)
5797                 return;
5798
5799         if (data->count != -1)
5800                 (data->count)--;
5801
5802         hist_enable_trigger(data, rec, event);
5803 }
5804
5805 static struct event_trigger_ops hist_enable_trigger_ops = {
5806         .func                   = hist_enable_trigger,
5807         .print                  = event_enable_trigger_print,
5808         .init                   = event_trigger_init,
5809         .free                   = event_enable_trigger_free,
5810 };
5811
5812 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5813         .func                   = hist_enable_count_trigger,
5814         .print                  = event_enable_trigger_print,
5815         .init                   = event_trigger_init,
5816         .free                   = event_enable_trigger_free,
5817 };
5818
5819 static struct event_trigger_ops hist_disable_trigger_ops = {
5820         .func                   = hist_enable_trigger,
5821         .print                  = event_enable_trigger_print,
5822         .init                   = event_trigger_init,
5823         .free                   = event_enable_trigger_free,
5824 };
5825
5826 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5827         .func                   = hist_enable_count_trigger,
5828         .print                  = event_enable_trigger_print,
5829         .init                   = event_trigger_init,
5830         .free                   = event_enable_trigger_free,
5831 };
5832
5833 static struct event_trigger_ops *
5834 hist_enable_get_trigger_ops(char *cmd, char *param)
5835 {
5836         struct event_trigger_ops *ops;
5837         bool enable;
5838
5839         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5840
5841         if (enable)
5842                 ops = param ? &hist_enable_count_trigger_ops :
5843                         &hist_enable_trigger_ops;
5844         else
5845                 ops = param ? &hist_disable_count_trigger_ops :
5846                         &hist_disable_trigger_ops;
5847
5848         return ops;
5849 }
5850
5851 static void hist_enable_unreg_all(struct trace_event_file *file)
5852 {
5853         struct event_trigger_data *test, *n;
5854
5855         list_for_each_entry_safe(test, n, &file->triggers, list) {
5856                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5857                         list_del_rcu(&test->list);
5858                         update_cond_flag(file);
5859                         trace_event_trigger_enable_disable(file, 0);
5860                         if (test->ops->free)
5861                                 test->ops->free(test->ops, test);
5862                 }
5863         }
5864 }
5865
5866 static struct event_command trigger_hist_enable_cmd = {
5867         .name                   = ENABLE_HIST_STR,
5868         .trigger_type           = ETT_HIST_ENABLE,
5869         .func                   = event_enable_trigger_func,
5870         .reg                    = event_enable_register_trigger,
5871         .unreg                  = event_enable_unregister_trigger,
5872         .unreg_all              = hist_enable_unreg_all,
5873         .get_trigger_ops        = hist_enable_get_trigger_ops,
5874         .set_filter             = set_trigger_filter,
5875 };
5876
5877 static struct event_command trigger_hist_disable_cmd = {
5878         .name                   = DISABLE_HIST_STR,
5879         .trigger_type           = ETT_HIST_ENABLE,
5880         .func                   = event_enable_trigger_func,
5881         .reg                    = event_enable_register_trigger,
5882         .unreg                  = event_enable_unregister_trigger,
5883         .unreg_all              = hist_enable_unreg_all,
5884         .get_trigger_ops        = hist_enable_get_trigger_ops,
5885         .set_filter             = set_trigger_filter,
5886 };
5887
5888 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5889 {
5890         unregister_event_command(&trigger_hist_enable_cmd);
5891         unregister_event_command(&trigger_hist_disable_cmd);
5892 }
5893
5894 __init int register_trigger_hist_enable_disable_cmds(void)
5895 {
5896         int ret;
5897
5898         ret = register_event_command(&trigger_hist_enable_cmd);
5899         if (WARN_ON(ret < 0))
5900                 return ret;
5901         ret = register_event_command(&trigger_hist_disable_cmd);
5902         if (WARN_ON(ret < 0))
5903                 unregister_trigger_hist_enable_disable_cmds();
5904
5905         return ret;
5906 }