kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  69
  70 #undef C
  71 #define C(a, b)         HIST_ERR_##a
  72
  73 enum { ERRORS };
  74
  75 #undef C
  76 #define C(a, b)         b
  77
  78 static const char *err_text[] = { ERRORS };
  79
  80 struct hist_field;
  81
  82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  83                                 struct tracing_map_elt *elt,
  84                                 struct trace_buffer *buffer,
  85                                 struct ring_buffer_event *rbe,
  86                                 void *event);
  87
  88 #define HIST_FIELD_OPERANDS_MAX 2
  89 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  90 #define HIST_ACTIONS_MAX        8
  91
  92 enum field_op_id {
  93         FIELD_OP_NONE,
  94         FIELD_OP_PLUS,
  95         FIELD_OP_MINUS,
  96         FIELD_OP_UNARY_MINUS,
  97 };
  98
  99 /*
 100  * A hist_var (histogram variable) contains variable information for
 101  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 102  * flag set.  A hist_var has a variable name e.g. ts0, and is
 103  * associated with a given histogram trigger, as specified by
 104  * hist_data.  The hist_var idx is the unique index assigned to the
 105  * variable by the hist trigger's tracing_map.  The idx is what is
 106  * used to set a variable's value and, by a variable reference, to
 107  * retrieve it.
 108  */
 109 struct hist_var {
 110         char                            *name;
 111         struct hist_trigger_data        *hist_data;
 112         unsigned int                    idx;
 113 };
 114
 115 struct hist_field {
 116         struct ftrace_event_field       *field;
 117         unsigned long                   flags;
 118         hist_field_fn_t                 fn;
 119         unsigned int                    ref;
 120         unsigned int                    size;
 121         unsigned int                    offset;
 122         unsigned int                    is_signed;
 123         const char                      *type;
 124         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 125         struct hist_trigger_data        *hist_data;
 126
 127         /*
 128          * Variable fields contain variable-specific info in var.
 129          */
 130         struct hist_var                 var;
 131         enum field_op_id                operator;
 132         char                            *system;
 133         char                            *event_name;
 134
 135         /*
 136          * The name field is used for EXPR and VAR_REF fields.  VAR
 137          * fields contain the variable name in var.name.
 138          */
 139         char                            *name;
 140
 141         /*
 142          * When a histogram trigger is hit, if it has any references
 143          * to variables, the values of those variables are collected
 144          * into a var_ref_vals array by resolve_var_refs().  The
 145          * current value of each variable is read from the tracing_map
 146          * using the hist field's hist_var.idx and entered into the
 147          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 148          */
 149         unsigned int                    var_ref_idx;
 150         bool                            read_once;
 151
 152         unsigned int                    var_str_idx;
 153 };
 154
 155 static u64 hist_field_none(struct hist_field *field,
 156                            struct tracing_map_elt *elt,
 157                            struct trace_buffer *buffer,
 158                            struct ring_buffer_event *rbe,
 159                            void *event)
 160 {
 161         return 0;
 162 }
 163
 164 static u64 hist_field_counter(struct hist_field *field,
 165                               struct tracing_map_elt *elt,
 166                               struct trace_buffer *buffer,
 167                               struct ring_buffer_event *rbe,
 168                               void *event)
 169 {
 170         return 1;
 171 }
 172
 173 static u64 hist_field_string(struct hist_field *hist_field,
 174                              struct tracing_map_elt *elt,
 175                              struct trace_buffer *buffer,
 176                              struct ring_buffer_event *rbe,
 177                              void *event)
 178 {
 179         char *addr = (char *)(event + hist_field->field->offset);
 180
 181         return (u64)(unsigned long)addr;
 182 }
 183
 184 static u64 hist_field_dynstring(struct hist_field *hist_field,
 185                                 struct tracing_map_elt *elt,
 186                                 struct trace_buffer *buffer,
 187                                 struct ring_buffer_event *rbe,
 188                                 void *event)
 189 {
 190         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 191         int str_loc = str_item & 0xffff;
 192         char *addr = (char *)(event + str_loc);
 193
 194         return (u64)(unsigned long)addr;
 195 }
 196
 197 static u64 hist_field_pstring(struct hist_field *hist_field,
 198                               struct tracing_map_elt *elt,
 199                               struct trace_buffer *buffer,
 200                               struct ring_buffer_event *rbe,
 201                               void *event)
 202 {
 203         char **addr = (char **)(event + hist_field->field->offset);
 204
 205         return (u64)(unsigned long)*addr;
 206 }
 207
 208 static u64 hist_field_log2(struct hist_field *hist_field,
 209                            struct tracing_map_elt *elt,
 210                            struct trace_buffer *buffer,
 211                            struct ring_buffer_event *rbe,
 212                            void *event)
 213 {
 214         struct hist_field *operand = hist_field->operands[0];
 215
 216         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 217
 218         return (u64) ilog2(roundup_pow_of_two(val));
 219 }
 220
 221 static u64 hist_field_plus(struct hist_field *hist_field,
 222                            struct tracing_map_elt *elt,
 223                            struct trace_buffer *buffer,
 224                            struct ring_buffer_event *rbe,
 225                            void *event)
 226 {
 227         struct hist_field *operand1 = hist_field->operands[0];
 228         struct hist_field *operand2 = hist_field->operands[1];
 229
 230         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 231         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 232
 233         return val1 + val2;
 234 }
 235
 236 static u64 hist_field_minus(struct hist_field *hist_field,
 237                             struct tracing_map_elt *elt,
 238                             struct trace_buffer *buffer,
 239                             struct ring_buffer_event *rbe,
 240                             void *event)
 241 {
 242         struct hist_field *operand1 = hist_field->operands[0];
 243         struct hist_field *operand2 = hist_field->operands[1];
 244
 245         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 246         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 247
 248         return val1 - val2;
 249 }
 250
 251 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 252                                   struct tracing_map_elt *elt,
 253                                   struct trace_buffer *buffer,
 254                                   struct ring_buffer_event *rbe,
 255                                   void *event)
 256 {
 257         struct hist_field *operand = hist_field->operands[0];
 258
 259         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 260         u64 val = (u64)-sval;
 261
 262         return val;
 263 }
 264
 265 #define DEFINE_HIST_FIELD_FN(type)                                      \
 266         static u64 hist_field_##type(struct hist_field *hist_field,     \
 267                                      struct tracing_map_elt *elt,       \
 268                                      struct trace_buffer *buffer,       \
 269                                      struct ring_buffer_event *rbe,     \
 270                                      void *event)                       \
 271 {                                                                       \
 272         type *addr = (type *)(event + hist_field->field->offset);       \
 273                                                                         \
 274         return (u64)(unsigned long)*addr;                               \
 275 }
 276
 277 DEFINE_HIST_FIELD_FN(s64);
 278 DEFINE_HIST_FIELD_FN(u64);
 279 DEFINE_HIST_FIELD_FN(s32);
 280 DEFINE_HIST_FIELD_FN(u32);
 281 DEFINE_HIST_FIELD_FN(s16);
 282 DEFINE_HIST_FIELD_FN(u16);
 283 DEFINE_HIST_FIELD_FN(s8);
 284 DEFINE_HIST_FIELD_FN(u8);
 285
 286 #define for_each_hist_field(i, hist_data)       \
 287         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 288
 289 #define for_each_hist_val_field(i, hist_data)   \
 290         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 291
 292 #define for_each_hist_key_field(i, hist_data)   \
 293         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 294
 295 #define HIST_STACKTRACE_DEPTH   16
 296 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 297 #define HIST_STACKTRACE_SKIP    5
 298
 299 #define HITCOUNT_IDX            0
 300 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 301
 302 enum hist_field_flags {
 303         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 304         HIST_FIELD_FL_KEY               = 1 << 1,
 305         HIST_FIELD_FL_STRING            = 1 << 2,
 306         HIST_FIELD_FL_HEX               = 1 << 3,
 307         HIST_FIELD_FL_SYM               = 1 << 4,
 308         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 309         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 310         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 311         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 312         HIST_FIELD_FL_LOG2              = 1 << 9,
 313         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 314         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 315         HIST_FIELD_FL_VAR               = 1 << 12,
 316         HIST_FIELD_FL_EXPR              = 1 << 13,
 317         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 318         HIST_FIELD_FL_CPU               = 1 << 15,
 319         HIST_FIELD_FL_ALIAS             = 1 << 16,
 320 };
 321
 322 struct var_defs {
 323         unsigned int    n_vars;
 324         char            *name[TRACING_MAP_VARS_MAX];
 325         char            *expr[TRACING_MAP_VARS_MAX];
 326 };
 327
 328 struct hist_trigger_attrs {
 329         char            *keys_str;
 330         char            *vals_str;
 331         char            *sort_key_str;
 332         char            *name;
 333         char            *clock;
 334         bool            pause;
 335         bool            cont;
 336         bool            clear;
 337         bool            ts_in_usecs;
 338         unsigned int    map_bits;
 339
 340         char            *assignment_str[TRACING_MAP_VARS_MAX];
 341         unsigned int    n_assignments;
 342
 343         char            *action_str[HIST_ACTIONS_MAX];
 344         unsigned int    n_actions;
 345
 346         struct var_defs var_defs;
 347 };
 348
 349 struct field_var {
 350         struct hist_field       *var;
 351         struct hist_field       *val;
 352 };
 353
 354 struct field_var_hist {
 355         struct hist_trigger_data        *hist_data;
 356         char                            *cmd;
 357 };
 358
 359 struct hist_trigger_data {
 360         struct hist_field               *fields[HIST_FIELDS_MAX];
 361         unsigned int                    n_vals;
 362         unsigned int                    n_keys;
 363         unsigned int                    n_fields;
 364         unsigned int                    n_vars;
 365         unsigned int                    n_var_str;
 366         unsigned int                    key_size;
 367         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 368         unsigned int                    n_sort_keys;
 369         struct trace_event_file         *event_file;
 370         struct hist_trigger_attrs       *attrs;
 371         struct tracing_map              *map;
 372         bool                            enable_timestamps;
 373         bool                            remove;
 374         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 375         unsigned int                    n_var_refs;
 376
 377         struct action_data              *actions[HIST_ACTIONS_MAX];
 378         unsigned int                    n_actions;
 379
 380         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 381         unsigned int                    n_field_vars;
 382         unsigned int                    n_field_var_str;
 383         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 384         unsigned int                    n_field_var_hists;
 385
 386         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 387         unsigned int                    n_save_vars;
 388         unsigned int                    n_save_var_str;
 389 };
 390
 391 struct action_data;
 392
 393 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 394                              struct tracing_map_elt *elt,
 395                              struct trace_buffer *buffer, void *rec,
 396                              struct ring_buffer_event *rbe, void *key,
 397                              struct action_data *data, u64 *var_ref_vals);
 398
 399 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 400
 401 enum handler_id {
 402         HANDLER_ONMATCH = 1,
 403         HANDLER_ONMAX,
 404         HANDLER_ONCHANGE,
 405 };
 406
 407 enum action_id {
 408         ACTION_SAVE = 1,
 409         ACTION_TRACE,
 410         ACTION_SNAPSHOT,
 411 };
 412
 413 struct action_data {
 414         enum handler_id         handler;
 415         enum action_id          action;
 416         char                    *action_name;
 417         action_fn_t             fn;
 418
 419         unsigned int            n_params;
 420         char                    *params[SYNTH_FIELDS_MAX];
 421
 422         /*
 423          * When a histogram trigger is hit, the values of any
 424          * references to variables, including variables being passed
 425          * as parameters to synthetic events, are collected into a
 426          * var_ref_vals array.  This var_ref_idx array is an array of
 427          * indices into the var_ref_vals array, one for each synthetic
 428          * event param, and is passed to the synthetic event
 429          * invocation.
 430          */
 431         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 432         struct synth_event      *synth_event;
 433         bool                    use_trace_keyword;
 434         char                    *synth_event_name;
 435
 436         union {
 437                 struct {
 438                         char                    *event;
 439                         char                    *event_system;
 440                 } match_data;
 441
 442                 struct {
 443                         /*
 444                          * var_str contains the $-unstripped variable
 445                          * name referenced by var_ref, and used when
 446                          * printing the action.  Because var_ref
 447                          * creation is deferred to create_actions(),
 448                          * we need a per-action way to save it until
 449                          * then, thus var_str.
 450                          */
 451                         char                    *var_str;
 452
 453                         /*
 454                          * var_ref refers to the variable being
 455                          * tracked e.g onmax($var).
 456                          */
 457                         struct hist_field       *var_ref;
 458
 459                         /*
 460                          * track_var contains the 'invisible' tracking
 461                          * variable created to keep the current
 462                          * e.g. max value.
 463                          */
 464                         struct hist_field       *track_var;
 465
 466                         check_track_val_fn_t    check_val;
 467                         action_fn_t             save_data;
 468                 } track_data;
 469         };
 470 };
 471
 472 struct track_data {
 473         u64                             track_val;
 474         bool                            updated;
 475
 476         unsigned int                    key_len;
 477         void                            *key;
 478         struct tracing_map_elt          elt;
 479
 480         struct action_data              *action_data;
 481         struct hist_trigger_data        *hist_data;
 482 };
 483
 484 struct hist_elt_data {
 485         char *comm;
 486         u64 *var_ref_vals;
 487         char *field_var_str[SYNTH_FIELDS_MAX];
 488 };
 489
 490 struct snapshot_context {
 491         struct tracing_map_elt  *elt;
 492         void                    *key;
 493 };
 494
 495 static void track_data_free(struct track_data *track_data)
 496 {
 497         struct hist_elt_data *elt_data;
 498
 499         if (!track_data)
 500                 return;
 501
 502         kfree(track_data->key);
 503
 504         elt_data = track_data->elt.private_data;
 505         if (elt_data) {
 506                 kfree(elt_data->comm);
 507                 kfree(elt_data);
 508         }
 509
 510         kfree(track_data);
 511 }
 512
 513 static struct track_data *track_data_alloc(unsigned int key_len,
 514                                            struct action_data *action_data,
 515                                            struct hist_trigger_data *hist_data)
 516 {
 517         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 518         struct hist_elt_data *elt_data;
 519
 520         if (!data)
 521                 return ERR_PTR(-ENOMEM);
 522
 523         data->key = kzalloc(key_len, GFP_KERNEL);
 524         if (!data->key) {
 525                 track_data_free(data);
 526                 return ERR_PTR(-ENOMEM);
 527         }
 528
 529         data->key_len = key_len;
 530         data->action_data = action_data;
 531         data->hist_data = hist_data;
 532
 533         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 534         if (!elt_data) {
 535                 track_data_free(data);
 536                 return ERR_PTR(-ENOMEM);
 537         }
 538
 539         data->elt.private_data = elt_data;
 540
 541         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 542         if (!elt_data->comm) {
 543                 track_data_free(data);
 544                 return ERR_PTR(-ENOMEM);
 545         }
 546
 547         return data;
 548 }
 549
 550 static char last_cmd[MAX_FILTER_STR_VAL];
 551 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 552
 553 static int errpos(char *str)
 554 {
 555         return err_pos(last_cmd, str);
 556 }
 557
 558 static void last_cmd_set(struct trace_event_file *file, char *str)
 559 {
 560         const char *system = NULL, *name = NULL;
 561         struct trace_event_call *call;
 562
 563         if (!str)
 564                 return;
 565
 566         strcpy(last_cmd, "hist:");
 567         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 568
 569         if (file) {
 570                 call = file->event_call;
 571                 system = call->class->system;
 572                 if (system) {
 573                         name = trace_event_name(call);
 574                         if (!name)
 575                                 system = NULL;
 576                 }
 577         }
 578
 579         if (system)
 580                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 581 }
 582
 583 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 584 {
 585         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 586                         err_type, err_pos);
 587 }
 588
 589 static void hist_err_clear(void)
 590 {
 591         last_cmd[0] = '\0';
 592         last_cmd_loc[0] = '\0';
 593 }
 594
 595 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 596                                     unsigned int *var_ref_idx);
 597
 598 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 599                                unsigned int *var_ref_idx)
 600 {
 601         struct tracepoint *tp = event->tp;
 602
 603         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 604                 struct tracepoint_func *probe_func_ptr;
 605                 synth_probe_func_t probe_func;
 606                 void *__data;
 607
 608                 if (!(cpu_online(raw_smp_processor_id())))
 609                         return;
 610
 611                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 612                 if (probe_func_ptr) {
 613                         do {
 614                                 probe_func = probe_func_ptr->func;
 615                                 __data = probe_func_ptr->data;
 616                                 probe_func(__data, var_ref_vals, var_ref_idx);
 617                         } while ((++probe_func_ptr)->func);
 618                 }
 619         }
 620 }
 621
 622 static void action_trace(struct hist_trigger_data *hist_data,
 623                          struct tracing_map_elt *elt,
 624                          struct trace_buffer *buffer, void *rec,
 625                          struct ring_buffer_event *rbe, void *key,
 626                          struct action_data *data, u64 *var_ref_vals)
 627 {
 628         struct synth_event *event = data->synth_event;
 629
 630         trace_synth(event, var_ref_vals, data->var_ref_idx);
 631 }
 632
 633 struct hist_var_data {
 634         struct list_head list;
 635         struct hist_trigger_data *hist_data;
 636 };
 637
 638 static u64 hist_field_timestamp(struct hist_field *hist_field,
 639                                 struct tracing_map_elt *elt,
 640                                 struct trace_buffer *buffer,
 641                                 struct ring_buffer_event *rbe,
 642                                 void *event)
 643 {
 644         struct hist_trigger_data *hist_data = hist_field->hist_data;
 645         struct trace_array *tr = hist_data->event_file->tr;
 646
 647         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 648
 649         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 650                 ts = ns2usecs(ts);
 651
 652         return ts;
 653 }
 654
 655 static u64 hist_field_cpu(struct hist_field *hist_field,
 656                           struct tracing_map_elt *elt,
 657                           struct trace_buffer *buffer,
 658                           struct ring_buffer_event *rbe,
 659                           void *event)
 660 {
 661         int cpu = smp_processor_id();
 662
 663         return cpu;
 664 }
 665
 666 /**
 667  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 668  * @hist_field: The VAR_REF field to check
 669  * @var_data: The hist trigger that owns the variable
 670  * @var_idx: The trigger variable identifier
 671  *
 672  * Check the given VAR_REF field to see whether or not it references
 673  * the given variable associated with the given trigger.
 674  *
 675  * Return: The VAR_REF field if it does reference the variable, NULL if not
 676  */
 677 static struct hist_field *
 678 check_field_for_var_ref(struct hist_field *hist_field,
 679                         struct hist_trigger_data *var_data,
 680                         unsigned int var_idx)
 681 {
 682         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 683
 684         if (hist_field && hist_field->var.idx == var_idx &&
 685             hist_field->var.hist_data == var_data)
 686                 return hist_field;
 687
 688         return NULL;
 689 }
 690
 691 /**
 692  * find_var_ref - Check if a trigger has a reference to a trigger variable
 693  * @hist_data: The hist trigger that might have a reference to the variable
 694  * @var_data: The hist trigger that owns the variable
 695  * @var_idx: The trigger variable identifier
 696  *
 697  * Check the list of var_refs[] on the first hist trigger to see
 698  * whether any of them are references to the variable on the second
 699  * trigger.
 700  *
 701  * Return: The VAR_REF field referencing the variable if so, NULL if not
 702  */
 703 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 704                                        struct hist_trigger_data *var_data,
 705                                        unsigned int var_idx)
 706 {
 707         struct hist_field *hist_field;
 708         unsigned int i;
 709
 710         for (i = 0; i < hist_data->n_var_refs; i++) {
 711                 hist_field = hist_data->var_refs[i];
 712                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 713                         return hist_field;
 714         }
 715
 716         return NULL;
 717 }
 718
 719 /**
 720  * find_any_var_ref - Check if there is a reference to a given trigger variable
 721  * @hist_data: The hist trigger
 722  * @var_idx: The trigger variable identifier
 723  *
 724  * Check to see whether the given variable is currently referenced by
 725  * any other trigger.
 726  *
 727  * The trigger the variable is defined on is explicitly excluded - the
 728  * assumption being that a self-reference doesn't prevent a trigger
 729  * from being removed.
 730  *
 731  * Return: The VAR_REF field referencing the variable if so, NULL if not
 732  */
 733 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 734                                            unsigned int var_idx)
 735 {
 736         struct trace_array *tr = hist_data->event_file->tr;
 737         struct hist_field *found = NULL;
 738         struct hist_var_data *var_data;
 739
 740         list_for_each_entry(var_data, &tr->hist_vars, list) {
 741                 if (var_data->hist_data == hist_data)
 742                         continue;
 743                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 744                 if (found)
 745                         break;
 746         }
 747
 748         return found;
 749 }
 750
 751 /**
 752  * check_var_refs - Check if there is a reference to any of trigger's variables
 753  * @hist_data: The hist trigger
 754  *
 755  * A trigger can define one or more variables.  If any one of them is
 756  * currently referenced by any other trigger, this function will
 757  * determine that.
 758
 759  * Typically used to determine whether or not a trigger can be removed
 760  * - if there are any references to a trigger's variables, it cannot.
 761  *
 762  * Return: True if there is a reference to any of trigger's variables
 763  */
 764 static bool check_var_refs(struct hist_trigger_data *hist_data)
 765 {
 766         struct hist_field *field;
 767         bool found = false;
 768         int i;
 769
 770         for_each_hist_field(i, hist_data) {
 771                 field = hist_data->fields[i];
 772                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 773                         if (find_any_var_ref(hist_data, field->var.idx)) {
 774                                 found = true;
 775                                 break;
 776                         }
 777                 }
 778         }
 779
 780         return found;
 781 }
 782
 783 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 784 {
 785         struct trace_array *tr = hist_data->event_file->tr;
 786         struct hist_var_data *var_data, *found = NULL;
 787
 788         list_for_each_entry(var_data, &tr->hist_vars, list) {
 789                 if (var_data->hist_data == hist_data) {
 790                         found = var_data;
 791                         break;
 792                 }
 793         }
 794
 795         return found;
 796 }
 797
 798 static bool field_has_hist_vars(struct hist_field *hist_field,
 799                                 unsigned int level)
 800 {
 801         int i;
 802
 803         if (level > 3)
 804                 return false;
 805
 806         if (!hist_field)
 807                 return false;
 808
 809         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 810             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 811                 return true;
 812
 813         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 814                 struct hist_field *operand;
 815
 816                 operand = hist_field->operands[i];
 817                 if (field_has_hist_vars(operand, level + 1))
 818                         return true;
 819         }
 820
 821         return false;
 822 }
 823
 824 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 825 {
 826         struct hist_field *hist_field;
 827         int i;
 828
 829         for_each_hist_field(i, hist_data) {
 830                 hist_field = hist_data->fields[i];
 831                 if (field_has_hist_vars(hist_field, 0))
 832                         return true;
 833         }
 834
 835         return false;
 836 }
 837
 838 static int save_hist_vars(struct hist_trigger_data *hist_data)
 839 {
 840         struct trace_array *tr = hist_data->event_file->tr;
 841         struct hist_var_data *var_data;
 842
 843         var_data = find_hist_vars(hist_data);
 844         if (var_data)
 845                 return 0;
 846
 847         if (tracing_check_open_get_tr(tr))
 848                 return -ENODEV;
 849
 850         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 851         if (!var_data) {
 852                 trace_array_put(tr);
 853                 return -ENOMEM;
 854         }
 855
 856         var_data->hist_data = hist_data;
 857         list_add(&var_data->list, &tr->hist_vars);
 858
 859         return 0;
 860 }
 861
 862 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 863 {
 864         struct trace_array *tr = hist_data->event_file->tr;
 865         struct hist_var_data *var_data;
 866
 867         var_data = find_hist_vars(hist_data);
 868         if (!var_data)
 869                 return;
 870
 871         if (WARN_ON(check_var_refs(hist_data)))
 872                 return;
 873
 874         list_del(&var_data->list);
 875
 876         kfree(var_data);
 877
 878         trace_array_put(tr);
 879 }
 880
 881 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 882                                          const char *var_name)
 883 {
 884         struct hist_field *hist_field, *found = NULL;
 885         int i;
 886
 887         for_each_hist_field(i, hist_data) {
 888                 hist_field = hist_data->fields[i];
 889                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 890                     strcmp(hist_field->var.name, var_name) == 0) {
 891                         found = hist_field;
 892                         break;
 893                 }
 894         }
 895
 896         return found;
 897 }
 898
 899 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 900                                    struct trace_event_file *file,
 901                                    const char *var_name)
 902 {
 903         struct hist_trigger_data *test_data;
 904         struct event_trigger_data *test;
 905         struct hist_field *hist_field;
 906
 907         lockdep_assert_held(&event_mutex);
 908
 909         hist_field = find_var_field(hist_data, var_name);
 910         if (hist_field)
 911                 return hist_field;
 912
 913         list_for_each_entry(test, &file->triggers, list) {
 914                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 915                         test_data = test->private_data;
 916                         hist_field = find_var_field(test_data, var_name);
 917                         if (hist_field)
 918                                 return hist_field;
 919                 }
 920         }
 921
 922         return NULL;
 923 }
 924
 925 static struct trace_event_file *find_var_file(struct trace_array *tr,
 926                                               char *system,
 927                                               char *event_name,
 928                                               char *var_name)
 929 {
 930         struct hist_trigger_data *var_hist_data;
 931         struct hist_var_data *var_data;
 932         struct trace_event_file *file, *found = NULL;
 933
 934         if (system)
 935                 return find_event_file(tr, system, event_name);
 936
 937         list_for_each_entry(var_data, &tr->hist_vars, list) {
 938                 var_hist_data = var_data->hist_data;
 939                 file = var_hist_data->event_file;
 940                 if (file == found)
 941                         continue;
 942
 943                 if (find_var_field(var_hist_data, var_name)) {
 944                         if (found) {
 945                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 946                                 return NULL;
 947                         }
 948
 949                         found = file;
 950                 }
 951         }
 952
 953         return found;
 954 }
 955
 956 static struct hist_field *find_file_var(struct trace_event_file *file,
 957                                         const char *var_name)
 958 {
 959         struct hist_trigger_data *test_data;
 960         struct event_trigger_data *test;
 961         struct hist_field *hist_field;
 962
 963         lockdep_assert_held(&event_mutex);
 964
 965         list_for_each_entry(test, &file->triggers, list) {
 966                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 967                         test_data = test->private_data;
 968                         hist_field = find_var_field(test_data, var_name);
 969                         if (hist_field)
 970                                 return hist_field;
 971                 }
 972         }
 973
 974         return NULL;
 975 }
 976
 977 static struct hist_field *
 978 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 979 {
 980         struct trace_array *tr = hist_data->event_file->tr;
 981         struct hist_field *hist_field, *found = NULL;
 982         struct trace_event_file *file;
 983         unsigned int i;
 984
 985         for (i = 0; i < hist_data->n_actions; i++) {
 986                 struct action_data *data = hist_data->actions[i];
 987
 988                 if (data->handler == HANDLER_ONMATCH) {
 989                         char *system = data->match_data.event_system;
 990                         char *event_name = data->match_data.event;
 991
 992                         file = find_var_file(tr, system, event_name, var_name);
 993                         if (!file)
 994                                 continue;
 995                         hist_field = find_file_var(file, var_name);
 996                         if (hist_field) {
 997                                 if (found) {
 998                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
 999                                                  errpos(var_name));
1000                                         return ERR_PTR(-EINVAL);
1001                                 }
1002
1003                                 found = hist_field;
1004                         }
1005                 }
1006         }
1007         return found;
1008 }
1009
1010 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1011                                          char *system,
1012                                          char *event_name,
1013                                          char *var_name)
1014 {
1015         struct trace_array *tr = hist_data->event_file->tr;
1016         struct hist_field *hist_field = NULL;
1017         struct trace_event_file *file;
1018
1019         if (!system || !event_name) {
1020                 hist_field = find_match_var(hist_data, var_name);
1021                 if (IS_ERR(hist_field))
1022                         return NULL;
1023                 if (hist_field)
1024                         return hist_field;
1025         }
1026
1027         file = find_var_file(tr, system, event_name, var_name);
1028         if (!file)
1029                 return NULL;
1030
1031         hist_field = find_file_var(file, var_name);
1032
1033         return hist_field;
1034 }
1035
1036 static u64 hist_field_var_ref(struct hist_field *hist_field,
1037                               struct tracing_map_elt *elt,
1038                               struct trace_buffer *buffer,
1039                               struct ring_buffer_event *rbe,
1040                               void *event)
1041 {
1042         struct hist_elt_data *elt_data;
1043         u64 var_val = 0;
1044
1045         if (WARN_ON_ONCE(!elt))
1046                 return var_val;
1047
1048         elt_data = elt->private_data;
1049         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1050
1051         return var_val;
1052 }
1053
1054 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1055                              u64 *var_ref_vals, bool self)
1056 {
1057         struct hist_trigger_data *var_data;
1058         struct tracing_map_elt *var_elt;
1059         struct hist_field *hist_field;
1060         unsigned int i, var_idx;
1061         bool resolved = true;
1062         u64 var_val = 0;
1063
1064         for (i = 0; i < hist_data->n_var_refs; i++) {
1065                 hist_field = hist_data->var_refs[i];
1066                 var_idx = hist_field->var.idx;
1067                 var_data = hist_field->var.hist_data;
1068
1069                 if (var_data == NULL) {
1070                         resolved = false;
1071                         break;
1072                 }
1073
1074                 if ((self && var_data != hist_data) ||
1075                     (!self && var_data == hist_data))
1076                         continue;
1077
1078                 var_elt = tracing_map_lookup(var_data->map, key);
1079                 if (!var_elt) {
1080                         resolved = false;
1081                         break;
1082                 }
1083
1084                 if (!tracing_map_var_set(var_elt, var_idx)) {
1085                         resolved = false;
1086                         break;
1087                 }
1088
1089                 if (self || !hist_field->read_once)
1090                         var_val = tracing_map_read_var(var_elt, var_idx);
1091                 else
1092                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1093
1094                 var_ref_vals[i] = var_val;
1095         }
1096
1097         return resolved;
1098 }
1099
1100 static const char *hist_field_name(struct hist_field *field,
1101                                    unsigned int level)
1102 {
1103         const char *field_name = "";
1104
1105         if (level > 1)
1106                 return field_name;
1107
1108         if (field->field)
1109                 field_name = field->field->name;
1110         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1111                  field->flags & HIST_FIELD_FL_ALIAS)
1112                 field_name = hist_field_name(field->operands[0], ++level);
1113         else if (field->flags & HIST_FIELD_FL_CPU)
1114                 field_name = "cpu";
1115         else if (field->flags & HIST_FIELD_FL_EXPR ||
1116                  field->flags & HIST_FIELD_FL_VAR_REF) {
1117                 if (field->system) {
1118                         static char full_name[MAX_FILTER_STR_VAL];
1119
1120                         strcat(full_name, field->system);
1121                         strcat(full_name, ".");
1122                         strcat(full_name, field->event_name);
1123                         strcat(full_name, ".");
1124                         strcat(full_name, field->name);
1125                         field_name = full_name;
1126                 } else
1127                         field_name = field->name;
1128         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1129                 field_name = "common_timestamp";
1130
1131         if (field_name == NULL)
1132                 field_name = "";
1133
1134         return field_name;
1135 }
1136
1137 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1138 {
1139         hist_field_fn_t fn = NULL;
1140
1141         switch (field_size) {
1142         case 8:
1143                 if (field_is_signed)
1144                         fn = hist_field_s64;
1145                 else
1146                         fn = hist_field_u64;
1147                 break;
1148         case 4:
1149                 if (field_is_signed)
1150                         fn = hist_field_s32;
1151                 else
1152                         fn = hist_field_u32;
1153                 break;
1154         case 2:
1155                 if (field_is_signed)
1156                         fn = hist_field_s16;
1157                 else
1158                         fn = hist_field_u16;
1159                 break;
1160         case 1:
1161                 if (field_is_signed)
1162                         fn = hist_field_s8;
1163                 else
1164                         fn = hist_field_u8;
1165                 break;
1166         }
1167
1168         return fn;
1169 }
1170
1171 static int parse_map_size(char *str)
1172 {
1173         unsigned long size, map_bits;
1174         int ret;
1175
1176         ret = kstrtoul(str, 0, &size);
1177         if (ret)
1178                 goto out;
1179
1180         map_bits = ilog2(roundup_pow_of_two(size));
1181         if (map_bits < TRACING_MAP_BITS_MIN ||
1182             map_bits > TRACING_MAP_BITS_MAX)
1183                 ret = -EINVAL;
1184         else
1185                 ret = map_bits;
1186  out:
1187         return ret;
1188 }
1189
1190 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1191 {
1192         unsigned int i;
1193
1194         if (!attrs)
1195                 return;
1196
1197         for (i = 0; i < attrs->n_assignments; i++)
1198                 kfree(attrs->assignment_str[i]);
1199
1200         for (i = 0; i < attrs->n_actions; i++)
1201                 kfree(attrs->action_str[i]);
1202
1203         kfree(attrs->name);
1204         kfree(attrs->sort_key_str);
1205         kfree(attrs->keys_str);
1206         kfree(attrs->vals_str);
1207         kfree(attrs->clock);
1208         kfree(attrs);
1209 }
1210
1211 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1212 {
1213         int ret = -EINVAL;
1214
1215         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1216                 return ret;
1217
1218         if ((str_has_prefix(str, "onmatch(")) ||
1219             (str_has_prefix(str, "onmax(")) ||
1220             (str_has_prefix(str, "onchange("))) {
1221                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1222                 if (!attrs->action_str[attrs->n_actions]) {
1223                         ret = -ENOMEM;
1224                         return ret;
1225                 }
1226                 attrs->n_actions++;
1227                 ret = 0;
1228         }
1229         return ret;
1230 }
1231
1232 static int parse_assignment(struct trace_array *tr,
1233                             char *str, struct hist_trigger_attrs *attrs)
1234 {
1235         int len, ret = 0;
1236
1237         if ((len = str_has_prefix(str, "key=")) ||
1238             (len = str_has_prefix(str, "keys="))) {
1239                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1240                 if (!attrs->keys_str) {
1241                         ret = -ENOMEM;
1242                         goto out;
1243                 }
1244         } else if ((len = str_has_prefix(str, "val=")) ||
1245                    (len = str_has_prefix(str, "vals=")) ||
1246                    (len = str_has_prefix(str, "values="))) {
1247                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1248                 if (!attrs->vals_str) {
1249                         ret = -ENOMEM;
1250                         goto out;
1251                 }
1252         } else if ((len = str_has_prefix(str, "sort="))) {
1253                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1254                 if (!attrs->sort_key_str) {
1255                         ret = -ENOMEM;
1256                         goto out;
1257                 }
1258         } else if (str_has_prefix(str, "name=")) {
1259                 attrs->name = kstrdup(str, GFP_KERNEL);
1260                 if (!attrs->name) {
1261                         ret = -ENOMEM;
1262                         goto out;
1263                 }
1264         } else if ((len = str_has_prefix(str, "clock="))) {
1265                 str += len;
1266
1267                 str = strstrip(str);
1268                 attrs->clock = kstrdup(str, GFP_KERNEL);
1269                 if (!attrs->clock) {
1270                         ret = -ENOMEM;
1271                         goto out;
1272                 }
1273         } else if ((len = str_has_prefix(str, "size="))) {
1274                 int map_bits = parse_map_size(str + len);
1275
1276                 if (map_bits < 0) {
1277                         ret = map_bits;
1278                         goto out;
1279                 }
1280                 attrs->map_bits = map_bits;
1281         } else {
1282                 char *assignment;
1283
1284                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1285                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1286                         ret = -EINVAL;
1287                         goto out;
1288                 }
1289
1290                 assignment = kstrdup(str, GFP_KERNEL);
1291                 if (!assignment) {
1292                         ret = -ENOMEM;
1293                         goto out;
1294                 }
1295
1296                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1297         }
1298  out:
1299         return ret;
1300 }
1301
1302 static struct hist_trigger_attrs *
1303 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1304 {
1305         struct hist_trigger_attrs *attrs;
1306         int ret = 0;
1307
1308         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1309         if (!attrs)
1310                 return ERR_PTR(-ENOMEM);
1311
1312         while (trigger_str) {
1313                 char *str = strsep(&trigger_str, ":");
1314                 char *rhs;
1315
1316                 rhs = strchr(str, '=');
1317                 if (rhs) {
1318                         if (!strlen(++rhs)) {
1319                                 ret = -EINVAL;
1320                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1321                                 goto free;
1322                         }
1323                         ret = parse_assignment(tr, str, attrs);
1324                         if (ret)
1325                                 goto free;
1326                 } else if (strcmp(str, "pause") == 0)
1327                         attrs->pause = true;
1328                 else if ((strcmp(str, "cont") == 0) ||
1329                          (strcmp(str, "continue") == 0))
1330                         attrs->cont = true;
1331                 else if (strcmp(str, "clear") == 0)
1332                         attrs->clear = true;
1333                 else {
1334                         ret = parse_action(str, attrs);
1335                         if (ret)
1336                                 goto free;
1337                 }
1338         }
1339
1340         if (!attrs->keys_str) {
1341                 ret = -EINVAL;
1342                 goto free;
1343         }
1344
1345         if (!attrs->clock) {
1346                 attrs->clock = kstrdup("global", GFP_KERNEL);
1347                 if (!attrs->clock) {
1348                         ret = -ENOMEM;
1349                         goto free;
1350                 }
1351         }
1352
1353         return attrs;
1354  free:
1355         destroy_hist_trigger_attrs(attrs);
1356
1357         return ERR_PTR(ret);
1358 }
1359
1360 static inline void save_comm(char *comm, struct task_struct *task)
1361 {
1362         if (!task->pid) {
1363                 strcpy(comm, "<idle>");
1364                 return;
1365         }
1366
1367         if (WARN_ON_ONCE(task->pid < 0)) {
1368                 strcpy(comm, "<XXX>");
1369                 return;
1370         }
1371
1372         strncpy(comm, task->comm, TASK_COMM_LEN);
1373 }
1374
1375 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1376 {
1377         unsigned int i;
1378
1379         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1380                 kfree(elt_data->field_var_str[i]);
1381
1382         kfree(elt_data->comm);
1383         kfree(elt_data);
1384 }
1385
1386 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1387 {
1388         struct hist_elt_data *elt_data = elt->private_data;
1389
1390         hist_elt_data_free(elt_data);
1391 }
1392
1393 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1394 {
1395         struct hist_trigger_data *hist_data = elt->map->private_data;
1396         unsigned int size = TASK_COMM_LEN;
1397         struct hist_elt_data *elt_data;
1398         struct hist_field *key_field;
1399         unsigned int i, n_str;
1400
1401         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1402         if (!elt_data)
1403                 return -ENOMEM;
1404
1405         for_each_hist_key_field(i, hist_data) {
1406                 key_field = hist_data->fields[i];
1407
1408                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1409                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1410                         if (!elt_data->comm) {
1411                                 kfree(elt_data);
1412                                 return -ENOMEM;
1413                         }
1414                         break;
1415                 }
1416         }
1417
1418         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1419                 hist_data->n_var_str;
1420         if (n_str > SYNTH_FIELDS_MAX) {
1421                 hist_elt_data_free(elt_data);
1422                 return -EINVAL;
1423         }
1424
1425         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1426
1427         size = STR_VAR_LEN_MAX;
1428
1429         for (i = 0; i < n_str; i++) {
1430                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1431                 if (!elt_data->field_var_str[i]) {
1432                         hist_elt_data_free(elt_data);
1433                         return -ENOMEM;
1434                 }
1435         }
1436
1437         elt->private_data = elt_data;
1438
1439         return 0;
1440 }
1441
1442 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1443 {
1444         struct hist_elt_data *elt_data = elt->private_data;
1445
1446         if (elt_data->comm)
1447                 save_comm(elt_data->comm, current);
1448 }
1449
1450 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1451         .elt_alloc      = hist_trigger_elt_data_alloc,
1452         .elt_free       = hist_trigger_elt_data_free,
1453         .elt_init       = hist_trigger_elt_data_init,
1454 };
1455
1456 static const char *get_hist_field_flags(struct hist_field *hist_field)
1457 {
1458         const char *flags_str = NULL;
1459
1460         if (hist_field->flags & HIST_FIELD_FL_HEX)
1461                 flags_str = "hex";
1462         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1463                 flags_str = "sym";
1464         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1465                 flags_str = "sym-offset";
1466         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1467                 flags_str = "execname";
1468         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1469                 flags_str = "syscall";
1470         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1471                 flags_str = "log2";
1472         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1473                 flags_str = "usecs";
1474
1475         return flags_str;
1476 }
1477
1478 static void expr_field_str(struct hist_field *field, char *expr)
1479 {
1480         if (field->flags & HIST_FIELD_FL_VAR_REF)
1481                 strcat(expr, "$");
1482
1483         strcat(expr, hist_field_name(field, 0));
1484
1485         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1486                 const char *flags_str = get_hist_field_flags(field);
1487
1488                 if (flags_str) {
1489                         strcat(expr, ".");
1490                         strcat(expr, flags_str);
1491                 }
1492         }
1493 }
1494
1495 static char *expr_str(struct hist_field *field, unsigned int level)
1496 {
1497         char *expr;
1498
1499         if (level > 1)
1500                 return NULL;
1501
1502         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1503         if (!expr)
1504                 return NULL;
1505
1506         if (!field->operands[0]) {
1507                 expr_field_str(field, expr);
1508                 return expr;
1509         }
1510
1511         if (field->operator == FIELD_OP_UNARY_MINUS) {
1512                 char *subexpr;
1513
1514                 strcat(expr, "-(");
1515                 subexpr = expr_str(field->operands[0], ++level);
1516                 if (!subexpr) {
1517                         kfree(expr);
1518                         return NULL;
1519                 }
1520                 strcat(expr, subexpr);
1521                 strcat(expr, ")");
1522
1523                 kfree(subexpr);
1524
1525                 return expr;
1526         }
1527
1528         expr_field_str(field->operands[0], expr);
1529
1530         switch (field->operator) {
1531         case FIELD_OP_MINUS:
1532                 strcat(expr, "-");
1533                 break;
1534         case FIELD_OP_PLUS:
1535                 strcat(expr, "+");
1536                 break;
1537         default:
1538                 kfree(expr);
1539                 return NULL;
1540         }
1541
1542         expr_field_str(field->operands[1], expr);
1543
1544         return expr;
1545 }
1546
1547 static int contains_operator(char *str)
1548 {
1549         enum field_op_id field_op = FIELD_OP_NONE;
1550         char *op;
1551
1552         op = strpbrk(str, "+-");
1553         if (!op)
1554                 return FIELD_OP_NONE;
1555
1556         switch (*op) {
1557         case '-':
1558                 /*
1559                  * Unfortunately, the modifier ".sym-offset"
1560                  * can confuse things.
1561                  */
1562                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1563                         return FIELD_OP_NONE;
1564
1565                 if (*str == '-')
1566                         field_op = FIELD_OP_UNARY_MINUS;
1567                 else
1568                         field_op = FIELD_OP_MINUS;
1569                 break;
1570         case '+':
1571                 field_op = FIELD_OP_PLUS;
1572                 break;
1573         default:
1574                 break;
1575         }
1576
1577         return field_op;
1578 }
1579
1580 static void get_hist_field(struct hist_field *hist_field)
1581 {
1582         hist_field->ref++;
1583 }
1584
1585 static void __destroy_hist_field(struct hist_field *hist_field)
1586 {
1587         if (--hist_field->ref > 1)
1588                 return;
1589
1590         kfree(hist_field->var.name);
1591         kfree(hist_field->name);
1592         kfree(hist_field->type);
1593
1594         kfree(hist_field->system);
1595         kfree(hist_field->event_name);
1596
1597         kfree(hist_field);
1598 }
1599
1600 static void destroy_hist_field(struct hist_field *hist_field,
1601                                unsigned int level)
1602 {
1603         unsigned int i;
1604
1605         if (level > 3)
1606                 return;
1607
1608         if (!hist_field)
1609                 return;
1610
1611         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1612                 return; /* var refs will be destroyed separately */
1613
1614         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1615                 destroy_hist_field(hist_field->operands[i], level + 1);
1616
1617         __destroy_hist_field(hist_field);
1618 }
1619
1620 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1621                                             struct ftrace_event_field *field,
1622                                             unsigned long flags,
1623                                             char *var_name)
1624 {
1625         struct hist_field *hist_field;
1626
1627         if (field && is_function_field(field))
1628                 return NULL;
1629
1630         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1631         if (!hist_field)
1632                 return NULL;
1633
1634         hist_field->ref = 1;
1635
1636         hist_field->hist_data = hist_data;
1637
1638         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1639                 goto out; /* caller will populate */
1640
1641         if (flags & HIST_FIELD_FL_VAR_REF) {
1642                 hist_field->fn = hist_field_var_ref;
1643                 goto out;
1644         }
1645
1646         if (flags & HIST_FIELD_FL_HITCOUNT) {
1647                 hist_field->fn = hist_field_counter;
1648                 hist_field->size = sizeof(u64);
1649                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1650                 if (!hist_field->type)
1651                         goto free;
1652                 goto out;
1653         }
1654
1655         if (flags & HIST_FIELD_FL_STACKTRACE) {
1656                 hist_field->fn = hist_field_none;
1657                 goto out;
1658         }
1659
1660         if (flags & HIST_FIELD_FL_LOG2) {
1661                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1662                 hist_field->fn = hist_field_log2;
1663                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1664                 hist_field->size = hist_field->operands[0]->size;
1665                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1666                 if (!hist_field->type)
1667                         goto free;
1668                 goto out;
1669         }
1670
1671         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1672                 hist_field->fn = hist_field_timestamp;
1673                 hist_field->size = sizeof(u64);
1674                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1675                 if (!hist_field->type)
1676                         goto free;
1677                 goto out;
1678         }
1679
1680         if (flags & HIST_FIELD_FL_CPU) {
1681                 hist_field->fn = hist_field_cpu;
1682                 hist_field->size = sizeof(int);
1683                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1684                 if (!hist_field->type)
1685                         goto free;
1686                 goto out;
1687         }
1688
1689         if (WARN_ON_ONCE(!field))
1690                 goto out;
1691
1692         if (is_string_field(field)) {
1693                 flags |= HIST_FIELD_FL_STRING;
1694
1695                 hist_field->size = MAX_FILTER_STR_VAL;
1696                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1697                 if (!hist_field->type)
1698                         goto free;
1699
1700                 if (field->filter_type == FILTER_STATIC_STRING)
1701                         hist_field->fn = hist_field_string;
1702                 else if (field->filter_type == FILTER_DYN_STRING)
1703                         hist_field->fn = hist_field_dynstring;
1704                 else
1705                         hist_field->fn = hist_field_pstring;
1706         } else {
1707                 hist_field->size = field->size;
1708                 hist_field->is_signed = field->is_signed;
1709                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1710                 if (!hist_field->type)
1711                         goto free;
1712
1713                 hist_field->fn = select_value_fn(field->size,
1714                                                  field->is_signed);
1715                 if (!hist_field->fn) {
1716                         destroy_hist_field(hist_field, 0);
1717                         return NULL;
1718                 }
1719         }
1720  out:
1721         hist_field->field = field;
1722         hist_field->flags = flags;
1723
1724         if (var_name) {
1725                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1726                 if (!hist_field->var.name)
1727                         goto free;
1728         }
1729
1730         return hist_field;
1731  free:
1732         destroy_hist_field(hist_field, 0);
1733         return NULL;
1734 }
1735
1736 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1737 {
1738         unsigned int i;
1739
1740         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1741                 if (hist_data->fields[i]) {
1742                         destroy_hist_field(hist_data->fields[i], 0);
1743                         hist_data->fields[i] = NULL;
1744                 }
1745         }
1746
1747         for (i = 0; i < hist_data->n_var_refs; i++) {
1748                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1749                 __destroy_hist_field(hist_data->var_refs[i]);
1750                 hist_data->var_refs[i] = NULL;
1751         }
1752 }
1753
1754 static int init_var_ref(struct hist_field *ref_field,
1755                         struct hist_field *var_field,
1756                         char *system, char *event_name)
1757 {
1758         int err = 0;
1759
1760         ref_field->var.idx = var_field->var.idx;
1761         ref_field->var.hist_data = var_field->hist_data;
1762         ref_field->size = var_field->size;
1763         ref_field->is_signed = var_field->is_signed;
1764         ref_field->flags |= var_field->flags &
1765                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1766
1767         if (system) {
1768                 ref_field->system = kstrdup(system, GFP_KERNEL);
1769                 if (!ref_field->system)
1770                         return -ENOMEM;
1771         }
1772
1773         if (event_name) {
1774                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1775                 if (!ref_field->event_name) {
1776                         err = -ENOMEM;
1777                         goto free;
1778                 }
1779         }
1780
1781         if (var_field->var.name) {
1782                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1783                 if (!ref_field->name) {
1784                         err = -ENOMEM;
1785                         goto free;
1786                 }
1787         } else if (var_field->name) {
1788                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1789                 if (!ref_field->name) {
1790                         err = -ENOMEM;
1791                         goto free;
1792                 }
1793         }
1794
1795         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1796         if (!ref_field->type) {
1797                 err = -ENOMEM;
1798                 goto free;
1799         }
1800  out:
1801         return err;
1802  free:
1803         kfree(ref_field->system);
1804         kfree(ref_field->event_name);
1805         kfree(ref_field->name);
1806
1807         goto out;
1808 }
1809
1810 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1811                             struct hist_field *var_field)
1812 {
1813         struct hist_field *ref_field;
1814         int i;
1815
1816         for (i = 0; i < hist_data->n_var_refs; i++) {
1817                 ref_field = hist_data->var_refs[i];
1818                 if (ref_field->var.idx == var_field->var.idx &&
1819                     ref_field->var.hist_data == var_field->hist_data)
1820                         return i;
1821         }
1822
1823         return -ENOENT;
1824 }
1825
1826 /**
1827  * create_var_ref - Create a variable reference and attach it to trigger
1828  * @hist_data: The trigger that will be referencing the variable
1829  * @var_field: The VAR field to create a reference to
1830  * @system: The optional system string
1831  * @event_name: The optional event_name string
1832  *
1833  * Given a variable hist_field, create a VAR_REF hist_field that
1834  * represents a reference to it.
1835  *
1836  * This function also adds the reference to the trigger that
1837  * now references the variable.
1838  *
1839  * Return: The VAR_REF field if successful, NULL if not
1840  */
1841 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1842                                          struct hist_field *var_field,
1843                                          char *system, char *event_name)
1844 {
1845         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1846         struct hist_field *ref_field;
1847         int i;
1848
1849         /* Check if the variable already exists */
1850         for (i = 0; i < hist_data->n_var_refs; i++) {
1851                 ref_field = hist_data->var_refs[i];
1852                 if (ref_field->var.idx == var_field->var.idx &&
1853                     ref_field->var.hist_data == var_field->hist_data) {
1854                         get_hist_field(ref_field);
1855                         return ref_field;
1856                 }
1857         }
1858
1859         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1860         if (ref_field) {
1861                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1862                         destroy_hist_field(ref_field, 0);
1863                         return NULL;
1864                 }
1865
1866                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1867                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1868         }
1869
1870         return ref_field;
1871 }
1872
1873 static bool is_var_ref(char *var_name)
1874 {
1875         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1876                 return false;
1877
1878         return true;
1879 }
1880
1881 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1882                                  char *var_name)
1883 {
1884         char *name, *field;
1885         unsigned int i;
1886
1887         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1888                 name = hist_data->attrs->var_defs.name[i];
1889
1890                 if (strcmp(var_name, name) == 0) {
1891                         field = hist_data->attrs->var_defs.expr[i];
1892                         if (contains_operator(field) || is_var_ref(field))
1893                                 continue;
1894                         return field;
1895                 }
1896         }
1897
1898         return NULL;
1899 }
1900
1901 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1902                                  char *system, char *event_name,
1903                                  char *var_name)
1904 {
1905         struct trace_event_call *call;
1906
1907         if (system && event_name) {
1908                 call = hist_data->event_file->event_call;
1909
1910                 if (strcmp(system, call->class->system) != 0)
1911                         return NULL;
1912
1913                 if (strcmp(event_name, trace_event_name(call)) != 0)
1914                         return NULL;
1915         }
1916
1917         if (!!system != !!event_name)
1918                 return NULL;
1919
1920         if (!is_var_ref(var_name))
1921                 return NULL;
1922
1923         var_name++;
1924
1925         return field_name_from_var(hist_data, var_name);
1926 }
1927
1928 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1929                                         char *system, char *event_name,
1930                                         char *var_name)
1931 {
1932         struct hist_field *var_field = NULL, *ref_field = NULL;
1933         struct trace_array *tr = hist_data->event_file->tr;
1934
1935         if (!is_var_ref(var_name))
1936                 return NULL;
1937
1938         var_name++;
1939
1940         var_field = find_event_var(hist_data, system, event_name, var_name);
1941         if (var_field)
1942                 ref_field = create_var_ref(hist_data, var_field,
1943                                            system, event_name);
1944
1945         if (!ref_field)
1946                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1947
1948         return ref_field;
1949 }
1950
1951 static struct ftrace_event_field *
1952 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1953             char *field_str, unsigned long *flags)
1954 {
1955         struct ftrace_event_field *field = NULL;
1956         char *field_name, *modifier, *str;
1957         struct trace_array *tr = file->tr;
1958
1959         modifier = str = kstrdup(field_str, GFP_KERNEL);
1960         if (!modifier)
1961                 return ERR_PTR(-ENOMEM);
1962
1963         field_name = strsep(&modifier, ".");
1964         if (modifier) {
1965                 if (strcmp(modifier, "hex") == 0)
1966                         *flags |= HIST_FIELD_FL_HEX;
1967                 else if (strcmp(modifier, "sym") == 0)
1968                         *flags |= HIST_FIELD_FL_SYM;
1969                 else if (strcmp(modifier, "sym-offset") == 0)
1970                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1971                 else if ((strcmp(modifier, "execname") == 0) &&
1972                          (strcmp(field_name, "common_pid") == 0))
1973                         *flags |= HIST_FIELD_FL_EXECNAME;
1974                 else if (strcmp(modifier, "syscall") == 0)
1975                         *flags |= HIST_FIELD_FL_SYSCALL;
1976                 else if (strcmp(modifier, "log2") == 0)
1977                         *flags |= HIST_FIELD_FL_LOG2;
1978                 else if (strcmp(modifier, "usecs") == 0)
1979                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1980                 else {
1981                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1982                         field = ERR_PTR(-EINVAL);
1983                         goto out;
1984                 }
1985         }
1986
1987         if (strcmp(field_name, "common_timestamp") == 0) {
1988                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1989                 hist_data->enable_timestamps = true;
1990                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1991                         hist_data->attrs->ts_in_usecs = true;
1992         } else if (strcmp(field_name, "cpu") == 0)
1993                 *flags |= HIST_FIELD_FL_CPU;
1994         else {
1995                 field = trace_find_event_field(file->event_call, field_name);
1996                 if (!field || !field->size) {
1997                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1998                         field = ERR_PTR(-EINVAL);
1999                         goto out;
2000                 }
2001         }
2002  out:
2003         kfree(str);
2004
2005         return field;
2006 }
2007
2008 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2009                                        struct hist_field *var_ref,
2010                                        char *var_name)
2011 {
2012         struct hist_field *alias = NULL;
2013         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2014
2015         alias = create_hist_field(hist_data, NULL, flags, var_name);
2016         if (!alias)
2017                 return NULL;
2018
2019         alias->fn = var_ref->fn;
2020         alias->operands[0] = var_ref;
2021
2022         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2023                 destroy_hist_field(alias, 0);
2024                 return NULL;
2025         }
2026
2027         alias->var_ref_idx = var_ref->var_ref_idx;
2028
2029         return alias;
2030 }
2031
2032 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2033                                      struct trace_event_file *file, char *str,
2034                                      unsigned long *flags, char *var_name)
2035 {
2036         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2037         struct ftrace_event_field *field = NULL;
2038         struct hist_field *hist_field = NULL;
2039         int ret = 0;
2040
2041         s = strchr(str, '.');
2042         if (s) {
2043                 s = strchr(++s, '.');
2044                 if (s) {
2045                         ref_system = strsep(&str, ".");
2046                         if (!str) {
2047                                 ret = -EINVAL;
2048                                 goto out;
2049                         }
2050                         ref_event = strsep(&str, ".");
2051                         if (!str) {
2052                                 ret = -EINVAL;
2053                                 goto out;
2054                         }
2055                         ref_var = str;
2056                 }
2057         }
2058
2059         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2060         if (!s) {
2061                 hist_field = parse_var_ref(hist_data, ref_system,
2062                                            ref_event, ref_var);
2063                 if (hist_field) {
2064                         if (var_name) {
2065                                 hist_field = create_alias(hist_data, hist_field, var_name);
2066                                 if (!hist_field) {
2067                                         ret = -ENOMEM;
2068                                         goto out;
2069                                 }
2070                         }
2071                         return hist_field;
2072                 }
2073         } else
2074                 str = s;
2075
2076         field = parse_field(hist_data, file, str, flags);
2077         if (IS_ERR(field)) {
2078                 ret = PTR_ERR(field);
2079                 goto out;
2080         }
2081
2082         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2083         if (!hist_field) {
2084                 ret = -ENOMEM;
2085                 goto out;
2086         }
2087
2088         return hist_field;
2089  out:
2090         return ERR_PTR(ret);
2091 }
2092
2093 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2094                                      struct trace_event_file *file,
2095                                      char *str, unsigned long flags,
2096                                      char *var_name, unsigned int level);
2097
2098 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2099                                       struct trace_event_file *file,
2100                                       char *str, unsigned long flags,
2101                                       char *var_name, unsigned int level)
2102 {
2103         struct hist_field *operand1, *expr = NULL;
2104         unsigned long operand_flags;
2105         int ret = 0;
2106         char *s;
2107
2108         /* we support only -(xxx) i.e. explicit parens required */
2109
2110         if (level > 3) {
2111                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2112                 ret = -EINVAL;
2113                 goto free;
2114         }
2115
2116         str++; /* skip leading '-' */
2117
2118         s = strchr(str, '(');
2119         if (s)
2120                 str++;
2121         else {
2122                 ret = -EINVAL;
2123                 goto free;
2124         }
2125
2126         s = strrchr(str, ')');
2127         if (s)
2128                 *s = '\0';
2129         else {
2130                 ret = -EINVAL; /* no closing ')' */
2131                 goto free;
2132         }
2133
2134         flags |= HIST_FIELD_FL_EXPR;
2135         expr = create_hist_field(hist_data, NULL, flags, var_name);
2136         if (!expr) {
2137                 ret = -ENOMEM;
2138                 goto free;
2139         }
2140
2141         operand_flags = 0;
2142         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2143         if (IS_ERR(operand1)) {
2144                 ret = PTR_ERR(operand1);
2145                 goto free;
2146         }
2147
2148         expr->flags |= operand1->flags &
2149                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2150         expr->fn = hist_field_unary_minus;
2151         expr->operands[0] = operand1;
2152         expr->operator = FIELD_OP_UNARY_MINUS;
2153         expr->name = expr_str(expr, 0);
2154         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2155         if (!expr->type) {
2156                 ret = -ENOMEM;
2157                 goto free;
2158         }
2159
2160         return expr;
2161  free:
2162         destroy_hist_field(expr, 0);
2163         return ERR_PTR(ret);
2164 }
2165
2166 static int check_expr_operands(struct trace_array *tr,
2167                                struct hist_field *operand1,
2168                                struct hist_field *operand2)
2169 {
2170         unsigned long operand1_flags = operand1->flags;
2171         unsigned long operand2_flags = operand2->flags;
2172
2173         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2174             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2175                 struct hist_field *var;
2176
2177                 var = find_var_field(operand1->var.hist_data, operand1->name);
2178                 if (!var)
2179                         return -EINVAL;
2180                 operand1_flags = var->flags;
2181         }
2182
2183         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2184             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2185                 struct hist_field *var;
2186
2187                 var = find_var_field(operand2->var.hist_data, operand2->name);
2188                 if (!var)
2189                         return -EINVAL;
2190                 operand2_flags = var->flags;
2191         }
2192
2193         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2194             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2195                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2196                 return -EINVAL;
2197         }
2198
2199         return 0;
2200 }
2201
2202 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2203                                      struct trace_event_file *file,
2204                                      char *str, unsigned long flags,
2205                                      char *var_name, unsigned int level)
2206 {
2207         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2208         unsigned long operand_flags;
2209         int field_op, ret = -EINVAL;
2210         char *sep, *operand1_str;
2211
2212         if (level > 3) {
2213                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2214                 return ERR_PTR(-EINVAL);
2215         }
2216
2217         field_op = contains_operator(str);
2218
2219         if (field_op == FIELD_OP_NONE)
2220                 return parse_atom(hist_data, file, str, &flags, var_name);
2221
2222         if (field_op == FIELD_OP_UNARY_MINUS)
2223                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2224
2225         switch (field_op) {
2226         case FIELD_OP_MINUS:
2227                 sep = "-";
2228                 break;
2229         case FIELD_OP_PLUS:
2230                 sep = "+";
2231                 break;
2232         default:
2233                 goto free;
2234         }
2235
2236         operand1_str = strsep(&str, sep);
2237         if (!operand1_str || !str)
2238                 goto free;
2239
2240         operand_flags = 0;
2241         operand1 = parse_atom(hist_data, file, operand1_str,
2242                               &operand_flags, NULL);
2243         if (IS_ERR(operand1)) {
2244                 ret = PTR_ERR(operand1);
2245                 operand1 = NULL;
2246                 goto free;
2247         }
2248
2249         /* rest of string could be another expression e.g. b+c in a+b+c */
2250         operand_flags = 0;
2251         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2252         if (IS_ERR(operand2)) {
2253                 ret = PTR_ERR(operand2);
2254                 operand2 = NULL;
2255                 goto free;
2256         }
2257
2258         ret = check_expr_operands(file->tr, operand1, operand2);
2259         if (ret)
2260                 goto free;
2261
2262         flags |= HIST_FIELD_FL_EXPR;
2263
2264         flags |= operand1->flags &
2265                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2266
2267         expr = create_hist_field(hist_data, NULL, flags, var_name);
2268         if (!expr) {
2269                 ret = -ENOMEM;
2270                 goto free;
2271         }
2272
2273         operand1->read_once = true;
2274         operand2->read_once = true;
2275
2276         expr->operands[0] = operand1;
2277         expr->operands[1] = operand2;
2278         expr->operator = field_op;
2279         expr->name = expr_str(expr, 0);
2280         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2281         if (!expr->type) {
2282                 ret = -ENOMEM;
2283                 goto free;
2284         }
2285
2286         switch (field_op) {
2287         case FIELD_OP_MINUS:
2288                 expr->fn = hist_field_minus;
2289                 break;
2290         case FIELD_OP_PLUS:
2291                 expr->fn = hist_field_plus;
2292                 break;
2293         default:
2294                 ret = -EINVAL;
2295                 goto free;
2296         }
2297
2298         return expr;
2299  free:
2300         destroy_hist_field(operand1, 0);
2301         destroy_hist_field(operand2, 0);
2302         destroy_hist_field(expr, 0);
2303
2304         return ERR_PTR(ret);
2305 }
2306
2307 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2308                                  struct trace_event_file *file)
2309 {
2310         struct event_trigger_data *test;
2311
2312         lockdep_assert_held(&event_mutex);
2313
2314         list_for_each_entry(test, &file->triggers, list) {
2315                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2316                         if (test->private_data == hist_data)
2317                                 return test->filter_str;
2318                 }
2319         }
2320
2321         return NULL;
2322 }
2323
2324 static struct event_command trigger_hist_cmd;
2325 static int event_hist_trigger_func(struct event_command *cmd_ops,
2326                                    struct trace_event_file *file,
2327                                    char *glob, char *cmd, char *param);
2328
2329 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2330                             struct hist_trigger_data *hist_data,
2331                             unsigned int n_keys)
2332 {
2333         struct hist_field *target_hist_field, *hist_field;
2334         unsigned int n, i, j;
2335
2336         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2337                 return false;
2338
2339         i = hist_data->n_vals;
2340         j = target_hist_data->n_vals;
2341
2342         for (n = 0; n < n_keys; n++) {
2343                 hist_field = hist_data->fields[i + n];
2344                 target_hist_field = target_hist_data->fields[j + n];
2345
2346                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2347                         return false;
2348                 if (hist_field->size != target_hist_field->size)
2349                         return false;
2350                 if (hist_field->is_signed != target_hist_field->is_signed)
2351                         return false;
2352         }
2353
2354         return true;
2355 }
2356
2357 static struct hist_trigger_data *
2358 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2359                      struct trace_event_file *file)
2360 {
2361         struct hist_trigger_data *hist_data;
2362         struct event_trigger_data *test;
2363         unsigned int n_keys;
2364
2365         lockdep_assert_held(&event_mutex);
2366
2367         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2368
2369         list_for_each_entry(test, &file->triggers, list) {
2370                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2371                         hist_data = test->private_data;
2372
2373                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2374                                 return hist_data;
2375                 }
2376         }
2377
2378         return NULL;
2379 }
2380
2381 static struct trace_event_file *event_file(struct trace_array *tr,
2382                                            char *system, char *event_name)
2383 {
2384         struct trace_event_file *file;
2385
2386         file = __find_event_file(tr, system, event_name);
2387         if (!file)
2388                 return ERR_PTR(-EINVAL);
2389
2390         return file;
2391 }
2392
2393 static struct hist_field *
2394 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2395                          char *system, char *event_name, char *field_name)
2396 {
2397         struct hist_field *event_var;
2398         char *synthetic_name;
2399
2400         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2401         if (!synthetic_name)
2402                 return ERR_PTR(-ENOMEM);
2403
2404         strcpy(synthetic_name, "synthetic_");
2405         strcat(synthetic_name, field_name);
2406
2407         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2408
2409         kfree(synthetic_name);
2410
2411         return event_var;
2412 }
2413
2414 /**
2415  * create_field_var_hist - Automatically create a histogram and var for a field
2416  * @target_hist_data: The target hist trigger
2417  * @subsys_name: Optional subsystem name
2418  * @event_name: Optional event name
2419  * @field_name: The name of the field (and the resulting variable)
2420  *
2421  * Hist trigger actions fetch data from variables, not directly from
2422  * events.  However, for convenience, users are allowed to directly
2423  * specify an event field in an action, which will be automatically
2424  * converted into a variable on their behalf.
2425
2426  * If a user specifies a field on an event that isn't the event the
2427  * histogram currently being defined (the target event histogram), the
2428  * only way that can be accomplished is if a new hist trigger is
2429  * created and the field variable defined on that.
2430  *
2431  * This function creates a new histogram compatible with the target
2432  * event (meaning a histogram with the same key as the target
2433  * histogram), and creates a variable for the specified field, but
2434  * with 'synthetic_' prepended to the variable name in order to avoid
2435  * collision with normal field variables.
2436  *
2437  * Return: The variable created for the field.
2438  */
2439 static struct hist_field *
2440 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2441                       char *subsys_name, char *event_name, char *field_name)
2442 {
2443         struct trace_array *tr = target_hist_data->event_file->tr;
2444         struct hist_trigger_data *hist_data;
2445         unsigned int i, n, first = true;
2446         struct field_var_hist *var_hist;
2447         struct trace_event_file *file;
2448         struct hist_field *key_field;
2449         struct hist_field *event_var;
2450         char *saved_filter;
2451         char *cmd;
2452         int ret;
2453
2454         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2455                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2456                 return ERR_PTR(-EINVAL);
2457         }
2458
2459         file = event_file(tr, subsys_name, event_name);
2460
2461         if (IS_ERR(file)) {
2462                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2463                 ret = PTR_ERR(file);
2464                 return ERR_PTR(ret);
2465         }
2466
2467         /*
2468          * Look for a histogram compatible with target.  We'll use the
2469          * found histogram specification to create a new matching
2470          * histogram with our variable on it.  target_hist_data is not
2471          * yet a registered histogram so we can't use that.
2472          */
2473         hist_data = find_compatible_hist(target_hist_data, file);
2474         if (!hist_data) {
2475                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2476                 return ERR_PTR(-EINVAL);
2477         }
2478
2479         /* See if a synthetic field variable has already been created */
2480         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2481                                              event_name, field_name);
2482         if (!IS_ERR_OR_NULL(event_var))
2483                 return event_var;
2484
2485         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2486         if (!var_hist)
2487                 return ERR_PTR(-ENOMEM);
2488
2489         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2490         if (!cmd) {
2491                 kfree(var_hist);
2492                 return ERR_PTR(-ENOMEM);
2493         }
2494
2495         /* Use the same keys as the compatible histogram */
2496         strcat(cmd, "keys=");
2497
2498         for_each_hist_key_field(i, hist_data) {
2499                 key_field = hist_data->fields[i];
2500                 if (!first)
2501                         strcat(cmd, ",");
2502                 strcat(cmd, key_field->field->name);
2503                 first = false;
2504         }
2505
2506         /* Create the synthetic field variable specification */
2507         strcat(cmd, ":synthetic_");
2508         strcat(cmd, field_name);
2509         strcat(cmd, "=");
2510         strcat(cmd, field_name);
2511
2512         /* Use the same filter as the compatible histogram */
2513         saved_filter = find_trigger_filter(hist_data, file);
2514         if (saved_filter) {
2515                 strcat(cmd, " if ");
2516                 strcat(cmd, saved_filter);
2517         }
2518
2519         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2520         if (!var_hist->cmd) {
2521                 kfree(cmd);
2522                 kfree(var_hist);
2523                 return ERR_PTR(-ENOMEM);
2524         }
2525
2526         /* Save the compatible histogram information */
2527         var_hist->hist_data = hist_data;
2528
2529         /* Create the new histogram with our variable */
2530         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2531                                       "", "hist", cmd);
2532         if (ret) {
2533                 kfree(cmd);
2534                 kfree(var_hist->cmd);
2535                 kfree(var_hist);
2536                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2537                 return ERR_PTR(ret);
2538         }
2539
2540         kfree(cmd);
2541
2542         /* If we can't find the variable, something went wrong */
2543         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2544                                              event_name, field_name);
2545         if (IS_ERR_OR_NULL(event_var)) {
2546                 kfree(var_hist->cmd);
2547                 kfree(var_hist);
2548                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2549                 return ERR_PTR(-EINVAL);
2550         }
2551
2552         n = target_hist_data->n_field_var_hists;
2553         target_hist_data->field_var_hists[n] = var_hist;
2554         target_hist_data->n_field_var_hists++;
2555
2556         return event_var;
2557 }
2558
2559 static struct hist_field *
2560 find_target_event_var(struct hist_trigger_data *hist_data,
2561                       char *subsys_name, char *event_name, char *var_name)
2562 {
2563         struct trace_event_file *file = hist_data->event_file;
2564         struct hist_field *hist_field = NULL;
2565
2566         if (subsys_name) {
2567                 struct trace_event_call *call;
2568
2569                 if (!event_name)
2570                         return NULL;
2571
2572                 call = file->event_call;
2573
2574                 if (strcmp(subsys_name, call->class->system) != 0)
2575                         return NULL;
2576
2577                 if (strcmp(event_name, trace_event_name(call)) != 0)
2578                         return NULL;
2579         }
2580
2581         hist_field = find_var_field(hist_data, var_name);
2582
2583         return hist_field;
2584 }
2585
2586 static inline void __update_field_vars(struct tracing_map_elt *elt,
2587                                        struct trace_buffer *buffer,
2588                                        struct ring_buffer_event *rbe,
2589                                        void *rec,
2590                                        struct field_var **field_vars,
2591                                        unsigned int n_field_vars,
2592                                        unsigned int field_var_str_start)
2593 {
2594         struct hist_elt_data *elt_data = elt->private_data;
2595         unsigned int i, j, var_idx;
2596         u64 var_val;
2597
2598         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2599                 struct field_var *field_var = field_vars[i];
2600                 struct hist_field *var = field_var->var;
2601                 struct hist_field *val = field_var->val;
2602
2603                 var_val = val->fn(val, elt, buffer, rbe, rec);
2604                 var_idx = var->var.idx;
2605
2606                 if (val->flags & HIST_FIELD_FL_STRING) {
2607                         char *str = elt_data->field_var_str[j++];
2608                         char *val_str = (char *)(uintptr_t)var_val;
2609
2610                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2611                         var_val = (u64)(uintptr_t)str;
2612                 }
2613                 tracing_map_set_var(elt, var_idx, var_val);
2614         }
2615 }
2616
2617 static void update_field_vars(struct hist_trigger_data *hist_data,
2618                               struct tracing_map_elt *elt,
2619                               struct trace_buffer *buffer,
2620                               struct ring_buffer_event *rbe,
2621                               void *rec)
2622 {
2623         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2624                             hist_data->n_field_vars, 0);
2625 }
2626
2627 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2628                                  struct tracing_map_elt *elt,
2629                                  struct trace_buffer *buffer,  void *rec,
2630                                  struct ring_buffer_event *rbe, void *key,
2631                                  struct action_data *data, u64 *var_ref_vals)
2632 {
2633         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2634                             hist_data->n_save_vars, hist_data->n_field_var_str);
2635 }
2636
2637 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2638                                      struct trace_event_file *file,
2639                                      char *name, int size, const char *type)
2640 {
2641         struct hist_field *var;
2642         int idx;
2643
2644         if (find_var(hist_data, file, name) && !hist_data->remove) {
2645                 var = ERR_PTR(-EINVAL);
2646                 goto out;
2647         }
2648
2649         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2650         if (!var) {
2651                 var = ERR_PTR(-ENOMEM);
2652                 goto out;
2653         }
2654
2655         idx = tracing_map_add_var(hist_data->map);
2656         if (idx < 0) {
2657                 kfree(var);
2658                 var = ERR_PTR(-EINVAL);
2659                 goto out;
2660         }
2661
2662         var->ref = 1;
2663         var->flags = HIST_FIELD_FL_VAR;
2664         var->var.idx = idx;
2665         var->var.hist_data = var->hist_data = hist_data;
2666         var->size = size;
2667         var->var.name = kstrdup(name, GFP_KERNEL);
2668         var->type = kstrdup(type, GFP_KERNEL);
2669         if (!var->var.name || !var->type) {
2670                 kfree(var->var.name);
2671                 kfree(var->type);
2672                 kfree(var);
2673                 var = ERR_PTR(-ENOMEM);
2674         }
2675  out:
2676         return var;
2677 }
2678
2679 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2680                                           struct trace_event_file *file,
2681                                           char *field_name)
2682 {
2683         struct hist_field *val = NULL, *var = NULL;
2684         unsigned long flags = HIST_FIELD_FL_VAR;
2685         struct trace_array *tr = file->tr;
2686         struct field_var *field_var;
2687         int ret = 0;
2688
2689         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2690                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2691                 ret = -EINVAL;
2692                 goto err;
2693         }
2694
2695         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2696         if (IS_ERR(val)) {
2697                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2698                 ret = PTR_ERR(val);
2699                 goto err;
2700         }
2701
2702         var = create_var(hist_data, file, field_name, val->size, val->type);
2703         if (IS_ERR(var)) {
2704                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2705                 kfree(val);
2706                 ret = PTR_ERR(var);
2707                 goto err;
2708         }
2709
2710         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2711         if (!field_var) {
2712                 kfree(val);
2713                 kfree(var);
2714                 ret =  -ENOMEM;
2715                 goto err;
2716         }
2717
2718         field_var->var = var;
2719         field_var->val = val;
2720  out:
2721         return field_var;
2722  err:
2723         field_var = ERR_PTR(ret);
2724         goto out;
2725 }
2726
2727 /**
2728  * create_target_field_var - Automatically create a variable for a field
2729  * @target_hist_data: The target hist trigger
2730  * @subsys_name: Optional subsystem name
2731  * @event_name: Optional event name
2732  * @var_name: The name of the field (and the resulting variable)
2733  *
2734  * Hist trigger actions fetch data from variables, not directly from
2735  * events.  However, for convenience, users are allowed to directly
2736  * specify an event field in an action, which will be automatically
2737  * converted into a variable on their behalf.
2738
2739  * This function creates a field variable with the name var_name on
2740  * the hist trigger currently being defined on the target event.  If
2741  * subsys_name and event_name are specified, this function simply
2742  * verifies that they do in fact match the target event subsystem and
2743  * event name.
2744  *
2745  * Return: The variable created for the field.
2746  */
2747 static struct field_var *
2748 create_target_field_var(struct hist_trigger_data *target_hist_data,
2749                         char *subsys_name, char *event_name, char *var_name)
2750 {
2751         struct trace_event_file *file = target_hist_data->event_file;
2752
2753         if (subsys_name) {
2754                 struct trace_event_call *call;
2755
2756                 if (!event_name)
2757                         return NULL;
2758
2759                 call = file->event_call;
2760
2761                 if (strcmp(subsys_name, call->class->system) != 0)
2762                         return NULL;
2763
2764                 if (strcmp(event_name, trace_event_name(call)) != 0)
2765                         return NULL;
2766         }
2767
2768         return create_field_var(target_hist_data, file, var_name);
2769 }
2770
2771 static bool check_track_val_max(u64 track_val, u64 var_val)
2772 {
2773         if (var_val <= track_val)
2774                 return false;
2775
2776         return true;
2777 }
2778
2779 static bool check_track_val_changed(u64 track_val, u64 var_val)
2780 {
2781         if (var_val == track_val)
2782                 return false;
2783
2784         return true;
2785 }
2786
2787 static u64 get_track_val(struct hist_trigger_data *hist_data,
2788                          struct tracing_map_elt *elt,
2789                          struct action_data *data)
2790 {
2791         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2792         u64 track_val;
2793
2794         track_val = tracing_map_read_var(elt, track_var_idx);
2795
2796         return track_val;
2797 }
2798
2799 static void save_track_val(struct hist_trigger_data *hist_data,
2800                            struct tracing_map_elt *elt,
2801                            struct action_data *data, u64 var_val)
2802 {
2803         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2804
2805         tracing_map_set_var(elt, track_var_idx, var_val);
2806 }
2807
2808 static void save_track_data(struct hist_trigger_data *hist_data,
2809                             struct tracing_map_elt *elt,
2810                             struct trace_buffer *buffer, void *rec,
2811                             struct ring_buffer_event *rbe, void *key,
2812                             struct action_data *data, u64 *var_ref_vals)
2813 {
2814         if (data->track_data.save_data)
2815                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2816                                            key, data, var_ref_vals);
2817 }
2818
2819 static bool check_track_val(struct tracing_map_elt *elt,
2820                             struct action_data *data,
2821                             u64 var_val)
2822 {
2823         struct hist_trigger_data *hist_data;
2824         u64 track_val;
2825
2826         hist_data = data->track_data.track_var->hist_data;
2827         track_val = get_track_val(hist_data, elt, data);
2828
2829         return data->track_data.check_val(track_val, var_val);
2830 }
2831
2832 #ifdef CONFIG_TRACER_SNAPSHOT
2833 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2834 {
2835         /* called with tr->max_lock held */
2836         struct track_data *track_data = tr->cond_snapshot->cond_data;
2837         struct hist_elt_data *elt_data, *track_elt_data;
2838         struct snapshot_context *context = cond_data;
2839         struct action_data *action;
2840         u64 track_val;
2841
2842         if (!track_data)
2843                 return false;
2844
2845         action = track_data->action_data;
2846
2847         track_val = get_track_val(track_data->hist_data, context->elt,
2848                                   track_data->action_data);
2849
2850         if (!action->track_data.check_val(track_data->track_val, track_val))
2851                 return false;
2852
2853         track_data->track_val = track_val;
2854         memcpy(track_data->key, context->key, track_data->key_len);
2855
2856         elt_data = context->elt->private_data;
2857         track_elt_data = track_data->elt.private_data;
2858         if (elt_data->comm)
2859                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2860
2861         track_data->updated = true;
2862
2863         return true;
2864 }
2865
2866 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2867                                      struct tracing_map_elt *elt,
2868                                      struct trace_buffer *buffer, void *rec,
2869                                      struct ring_buffer_event *rbe, void *key,
2870                                      struct action_data *data,
2871                                      u64 *var_ref_vals)
2872 {
2873         struct trace_event_file *file = hist_data->event_file;
2874         struct snapshot_context context;
2875
2876         context.elt = elt;
2877         context.key = key;
2878
2879         tracing_snapshot_cond(file->tr, &context);
2880 }
2881
2882 static void hist_trigger_print_key(struct seq_file *m,
2883                                    struct hist_trigger_data *hist_data,
2884                                    void *key,
2885                                    struct tracing_map_elt *elt);
2886
2887 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2888 {
2889         unsigned int i;
2890
2891         if (!hist_data->n_actions)
2892                 return NULL;
2893
2894         for (i = 0; i < hist_data->n_actions; i++) {
2895                 struct action_data *data = hist_data->actions[i];
2896
2897                 if (data->action == ACTION_SNAPSHOT)
2898                         return data;
2899         }
2900
2901         return NULL;
2902 }
2903
2904 static void track_data_snapshot_print(struct seq_file *m,
2905                                       struct hist_trigger_data *hist_data)
2906 {
2907         struct trace_event_file *file = hist_data->event_file;
2908         struct track_data *track_data;
2909         struct action_data *action;
2910
2911         track_data = tracing_cond_snapshot_data(file->tr);
2912         if (!track_data)
2913                 return;
2914
2915         if (!track_data->updated)
2916                 return;
2917
2918         action = snapshot_action(hist_data);
2919         if (!action)
2920                 return;
2921
2922         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2923         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2924                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2925                    action->track_data.var_str, track_data->track_val);
2926
2927         seq_puts(m, "\ttriggered by event with key: ");
2928         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2929         seq_putc(m, '\n');
2930 }
2931 #else
2932 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2933 {
2934         return false;
2935 }
2936 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2937                                      struct tracing_map_elt *elt,
2938                                      struct trace_buffer *buffer, void *rec,
2939                                      struct ring_buffer_event *rbe, void *key,
2940                                      struct action_data *data,
2941                                      u64 *var_ref_vals) {}
2942 static void track_data_snapshot_print(struct seq_file *m,
2943                                       struct hist_trigger_data *hist_data) {}
2944 #endif /* CONFIG_TRACER_SNAPSHOT */
2945
2946 static void track_data_print(struct seq_file *m,
2947                              struct hist_trigger_data *hist_data,
2948                              struct tracing_map_elt *elt,
2949                              struct action_data *data)
2950 {
2951         u64 track_val = get_track_val(hist_data, elt, data);
2952         unsigned int i, save_var_idx;
2953
2954         if (data->handler == HANDLER_ONMAX)
2955                 seq_printf(m, "\n\tmax: %10llu", track_val);
2956         else if (data->handler == HANDLER_ONCHANGE)
2957                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2958
2959         if (data->action == ACTION_SNAPSHOT)
2960                 return;
2961
2962         for (i = 0; i < hist_data->n_save_vars; i++) {
2963                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2964                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2965                 u64 val;
2966
2967                 save_var_idx = save_var->var.idx;
2968
2969                 val = tracing_map_read_var(elt, save_var_idx);
2970
2971                 if (save_val->flags & HIST_FIELD_FL_STRING) {
2972                         seq_printf(m, "  %s: %-32s", save_var->var.name,
2973                                    (char *)(uintptr_t)(val));
2974                 } else
2975                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2976         }
2977 }
2978
2979 static void ontrack_action(struct hist_trigger_data *hist_data,
2980                            struct tracing_map_elt *elt,
2981                            struct trace_buffer *buffer, void *rec,
2982                            struct ring_buffer_event *rbe, void *key,
2983                            struct action_data *data, u64 *var_ref_vals)
2984 {
2985         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2986
2987         if (check_track_val(elt, data, var_val)) {
2988                 save_track_val(hist_data, elt, data, var_val);
2989                 save_track_data(hist_data, elt, buffer, rec, rbe,
2990                                 key, data, var_ref_vals);
2991         }
2992 }
2993
2994 static void action_data_destroy(struct action_data *data)
2995 {
2996         unsigned int i;
2997
2998         lockdep_assert_held(&event_mutex);
2999
3000         kfree(data->action_name);
3001
3002         for (i = 0; i < data->n_params; i++)
3003                 kfree(data->params[i]);
3004
3005         if (data->synth_event)
3006                 data->synth_event->ref--;
3007
3008         kfree(data->synth_event_name);
3009
3010         kfree(data);
3011 }
3012
3013 static void track_data_destroy(struct hist_trigger_data *hist_data,
3014                                struct action_data *data)
3015 {
3016         struct trace_event_file *file = hist_data->event_file;
3017
3018         destroy_hist_field(data->track_data.track_var, 0);
3019
3020         if (data->action == ACTION_SNAPSHOT) {
3021                 struct track_data *track_data;
3022
3023                 track_data = tracing_cond_snapshot_data(file->tr);
3024                 if (track_data && track_data->hist_data == hist_data) {
3025                         tracing_snapshot_cond_disable(file->tr);
3026                         track_data_free(track_data);
3027                 }
3028         }
3029
3030         kfree(data->track_data.var_str);
3031
3032         action_data_destroy(data);
3033 }
3034
3035 static int action_create(struct hist_trigger_data *hist_data,
3036                          struct action_data *data);
3037
3038 static int track_data_create(struct hist_trigger_data *hist_data,
3039                              struct action_data *data)
3040 {
3041         struct hist_field *var_field, *ref_field, *track_var = NULL;
3042         struct trace_event_file *file = hist_data->event_file;
3043         struct trace_array *tr = file->tr;
3044         char *track_data_var_str;
3045         int ret = 0;
3046
3047         track_data_var_str = data->track_data.var_str;
3048         if (track_data_var_str[0] != '$') {
3049                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3050                 return -EINVAL;
3051         }
3052         track_data_var_str++;
3053
3054         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3055         if (!var_field) {
3056                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3057                 return -EINVAL;
3058         }
3059
3060         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3061         if (!ref_field)
3062                 return -ENOMEM;
3063
3064         data->track_data.var_ref = ref_field;
3065
3066         if (data->handler == HANDLER_ONMAX)
3067                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3068         if (IS_ERR(track_var)) {
3069                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3070                 ret = PTR_ERR(track_var);
3071                 goto out;
3072         }
3073
3074         if (data->handler == HANDLER_ONCHANGE)
3075                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3076         if (IS_ERR(track_var)) {
3077                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3078                 ret = PTR_ERR(track_var);
3079                 goto out;
3080         }
3081         data->track_data.track_var = track_var;
3082
3083         ret = action_create(hist_data, data);
3084  out:
3085         return ret;
3086 }
3087
3088 static int parse_action_params(struct trace_array *tr, char *params,
3089                                struct action_data *data)
3090 {
3091         char *param, *saved_param;
3092         bool first_param = true;
3093         int ret = 0;
3094
3095         while (params) {
3096                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3097                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3098                         goto out;
3099                 }
3100
3101                 param = strsep(&params, ",");
3102                 if (!param) {
3103                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3104                         ret = -EINVAL;
3105                         goto out;
3106                 }
3107
3108                 param = strstrip(param);
3109                 if (strlen(param) < 2) {
3110                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3111                         ret = -EINVAL;
3112                         goto out;
3113                 }
3114
3115                 saved_param = kstrdup(param, GFP_KERNEL);
3116                 if (!saved_param) {
3117                         ret = -ENOMEM;
3118                         goto out;
3119                 }
3120
3121                 if (first_param && data->use_trace_keyword) {
3122                         data->synth_event_name = saved_param;
3123                         first_param = false;
3124                         continue;
3125                 }
3126                 first_param = false;
3127
3128                 data->params[data->n_params++] = saved_param;
3129         }
3130  out:
3131         return ret;
3132 }
3133
3134 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3135                         enum handler_id handler)
3136 {
3137         char *action_name;
3138         int ret = 0;
3139
3140         strsep(&str, ".");
3141         if (!str) {
3142                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3143                 ret = -EINVAL;
3144                 goto out;
3145         }
3146
3147         action_name = strsep(&str, "(");
3148         if (!action_name || !str) {
3149                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3150                 ret = -EINVAL;
3151                 goto out;
3152         }
3153
3154         if (str_has_prefix(action_name, "save")) {
3155                 char *params = strsep(&str, ")");
3156
3157                 if (!params) {
3158                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3159                         ret = -EINVAL;
3160                         goto out;
3161                 }
3162
3163                 ret = parse_action_params(tr, params, data);
3164                 if (ret)
3165                         goto out;
3166
3167                 if (handler == HANDLER_ONMAX)
3168                         data->track_data.check_val = check_track_val_max;
3169                 else if (handler == HANDLER_ONCHANGE)
3170                         data->track_data.check_val = check_track_val_changed;
3171                 else {
3172                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3173                         ret = -EINVAL;
3174                         goto out;
3175                 }
3176
3177                 data->track_data.save_data = save_track_data_vars;
3178                 data->fn = ontrack_action;
3179                 data->action = ACTION_SAVE;
3180         } else if (str_has_prefix(action_name, "snapshot")) {
3181                 char *params = strsep(&str, ")");
3182
3183                 if (!str) {
3184                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3185                         ret = -EINVAL;
3186                         goto out;
3187                 }
3188
3189                 if (handler == HANDLER_ONMAX)
3190                         data->track_data.check_val = check_track_val_max;
3191                 else if (handler == HANDLER_ONCHANGE)
3192                         data->track_data.check_val = check_track_val_changed;
3193                 else {
3194                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3195                         ret = -EINVAL;
3196                         goto out;
3197                 }
3198
3199                 data->track_data.save_data = save_track_data_snapshot;
3200                 data->fn = ontrack_action;
3201                 data->action = ACTION_SNAPSHOT;
3202         } else {
3203                 char *params = strsep(&str, ")");
3204
3205                 if (str_has_prefix(action_name, "trace"))
3206                         data->use_trace_keyword = true;
3207
3208                 if (params) {
3209                         ret = parse_action_params(tr, params, data);
3210                         if (ret)
3211                                 goto out;
3212                 }
3213
3214                 if (handler == HANDLER_ONMAX)
3215                         data->track_data.check_val = check_track_val_max;
3216                 else if (handler == HANDLER_ONCHANGE)
3217                         data->track_data.check_val = check_track_val_changed;
3218
3219                 if (handler != HANDLER_ONMATCH) {
3220                         data->track_data.save_data = action_trace;
3221                         data->fn = ontrack_action;
3222                 } else
3223                         data->fn = action_trace;
3224
3225                 data->action = ACTION_TRACE;
3226         }
3227
3228         data->action_name = kstrdup(action_name, GFP_KERNEL);
3229         if (!data->action_name) {
3230                 ret = -ENOMEM;
3231                 goto out;
3232         }
3233
3234         data->handler = handler;
3235  out:
3236         return ret;
3237 }
3238
3239 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3240                                             char *str, enum handler_id handler)
3241 {
3242         struct action_data *data;
3243         int ret = -EINVAL;
3244         char *var_str;
3245
3246         data = kzalloc(sizeof(*data), GFP_KERNEL);
3247         if (!data)
3248                 return ERR_PTR(-ENOMEM);
3249
3250         var_str = strsep(&str, ")");
3251         if (!var_str || !str) {
3252                 ret = -EINVAL;
3253                 goto free;
3254         }
3255
3256         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3257         if (!data->track_data.var_str) {
3258                 ret = -ENOMEM;
3259                 goto free;
3260         }
3261
3262         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3263         if (ret)
3264                 goto free;
3265  out:
3266         return data;
3267  free:
3268         track_data_destroy(hist_data, data);
3269         data = ERR_PTR(ret);
3270         goto out;
3271 }
3272
3273 static void onmatch_destroy(struct action_data *data)
3274 {
3275         kfree(data->match_data.event);
3276         kfree(data->match_data.event_system);
3277
3278         action_data_destroy(data);
3279 }
3280
3281 static void destroy_field_var(struct field_var *field_var)
3282 {
3283         if (!field_var)
3284                 return;
3285
3286         destroy_hist_field(field_var->var, 0);
3287         destroy_hist_field(field_var->val, 0);
3288
3289         kfree(field_var);
3290 }
3291
3292 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3293 {
3294         unsigned int i;
3295
3296         for (i = 0; i < hist_data->n_field_vars; i++)
3297                 destroy_field_var(hist_data->field_vars[i]);
3298
3299         for (i = 0; i < hist_data->n_save_vars; i++)
3300                 destroy_field_var(hist_data->save_vars[i]);
3301 }
3302
3303 static void save_field_var(struct hist_trigger_data *hist_data,
3304                            struct field_var *field_var)
3305 {
3306         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3307
3308         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3309                 hist_data->n_field_var_str++;
3310 }
3311
3312
3313 static int check_synth_field(struct synth_event *event,
3314                              struct hist_field *hist_field,
3315                              unsigned int field_pos)
3316 {
3317         struct synth_field *field;
3318
3319         if (field_pos >= event->n_fields)
3320                 return -EINVAL;
3321
3322         field = event->fields[field_pos];
3323
3324         /*
3325          * A dynamic string synth field can accept static or
3326          * dynamic. A static string synth field can only accept a
3327          * same-sized static string, which is checked for later.
3328          */
3329         if (strstr(hist_field->type, "char[") && field->is_string
3330             && field->is_dynamic)
3331                 return 0;
3332
3333         if (strcmp(field->type, hist_field->type) != 0) {
3334                 if (field->size != hist_field->size ||
3335                     field->is_signed != hist_field->is_signed)
3336                         return -EINVAL;
3337         }
3338
3339         return 0;
3340 }
3341
3342 static struct hist_field *
3343 trace_action_find_var(struct hist_trigger_data *hist_data,
3344                       struct action_data *data,
3345                       char *system, char *event, char *var)
3346 {
3347         struct trace_array *tr = hist_data->event_file->tr;
3348         struct hist_field *hist_field;
3349
3350         var++; /* skip '$' */
3351
3352         hist_field = find_target_event_var(hist_data, system, event, var);
3353         if (!hist_field) {
3354                 if (!system && data->handler == HANDLER_ONMATCH) {
3355                         system = data->match_data.event_system;
3356                         event = data->match_data.event;
3357                 }
3358
3359                 hist_field = find_event_var(hist_data, system, event, var);
3360         }
3361
3362         if (!hist_field)
3363                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3364
3365         return hist_field;
3366 }
3367
3368 static struct hist_field *
3369 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3370                               struct action_data *data, char *system,
3371                               char *event, char *var)
3372 {
3373         struct hist_field *hist_field = NULL;
3374         struct field_var *field_var;
3375
3376         /*
3377          * First try to create a field var on the target event (the
3378          * currently being defined).  This will create a variable for
3379          * unqualified fields on the target event, or if qualified,
3380          * target fields that have qualified names matching the target.
3381          */
3382         field_var = create_target_field_var(hist_data, system, event, var);
3383
3384         if (field_var && !IS_ERR(field_var)) {
3385                 save_field_var(hist_data, field_var);
3386                 hist_field = field_var->var;
3387         } else {
3388                 field_var = NULL;
3389                 /*
3390                  * If no explicit system.event is specified, default to
3391                  * looking for fields on the onmatch(system.event.xxx)
3392                  * event.
3393                  */
3394                 if (!system && data->handler == HANDLER_ONMATCH) {
3395                         system = data->match_data.event_system;
3396                         event = data->match_data.event;
3397                 }
3398
3399                 /*
3400                  * At this point, we're looking at a field on another
3401                  * event.  Because we can't modify a hist trigger on
3402                  * another event to add a variable for a field, we need
3403                  * to create a new trigger on that event and create the
3404                  * variable at the same time.
3405                  */
3406                 hist_field = create_field_var_hist(hist_data, system, event, var);
3407                 if (IS_ERR(hist_field))
3408                         goto free;
3409         }
3410  out:
3411         return hist_field;
3412  free:
3413         destroy_field_var(field_var);
3414         hist_field = NULL;
3415         goto out;
3416 }
3417
3418 static int trace_action_create(struct hist_trigger_data *hist_data,
3419                                struct action_data *data)
3420 {
3421         struct trace_array *tr = hist_data->event_file->tr;
3422         char *event_name, *param, *system = NULL;
3423         struct hist_field *hist_field, *var_ref;
3424         unsigned int i;
3425         unsigned int field_pos = 0;
3426         struct synth_event *event;
3427         char *synth_event_name;
3428         int var_ref_idx, ret = 0;
3429
3430         lockdep_assert_held(&event_mutex);
3431
3432         if (data->use_trace_keyword)
3433                 synth_event_name = data->synth_event_name;
3434         else
3435                 synth_event_name = data->action_name;
3436
3437         event = find_synth_event(synth_event_name);
3438         if (!event) {
3439                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3440                 return -EINVAL;
3441         }
3442
3443         event->ref++;
3444
3445         for (i = 0; i < data->n_params; i++) {
3446                 char *p;
3447
3448                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3449                 if (!param) {
3450                         ret = -ENOMEM;
3451                         goto err;
3452                 }
3453
3454                 system = strsep(&param, ".");
3455                 if (!param) {
3456                         param = (char *)system;
3457                         system = event_name = NULL;
3458                 } else {
3459                         event_name = strsep(&param, ".");
3460                         if (!param) {
3461                                 kfree(p);
3462                                 ret = -EINVAL;
3463                                 goto err;
3464                         }
3465                 }
3466
3467                 if (param[0] == '$')
3468                         hist_field = trace_action_find_var(hist_data, data,
3469                                                            system, event_name,
3470                                                            param);
3471                 else
3472                         hist_field = trace_action_create_field_var(hist_data,
3473                                                                    data,
3474                                                                    system,
3475                                                                    event_name,
3476                                                                    param);
3477
3478                 if (!hist_field) {
3479                         kfree(p);
3480                         ret = -EINVAL;
3481                         goto err;
3482                 }
3483
3484                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3485                         var_ref = create_var_ref(hist_data, hist_field,
3486                                                  system, event_name);
3487                         if (!var_ref) {
3488                                 kfree(p);
3489                                 ret = -ENOMEM;
3490                                 goto err;
3491                         }
3492
3493                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3494                         if (WARN_ON(var_ref_idx < 0)) {
3495                                 ret = var_ref_idx;
3496                                 goto err;
3497                         }
3498
3499                         data->var_ref_idx[i] = var_ref_idx;
3500
3501                         field_pos++;
3502                         kfree(p);
3503                         continue;
3504                 }
3505
3506                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3507                 kfree(p);
3508                 ret = -EINVAL;
3509                 goto err;
3510         }
3511
3512         if (field_pos != event->n_fields) {
3513                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3514                 ret = -EINVAL;
3515                 goto err;
3516         }
3517
3518         data->synth_event = event;
3519  out:
3520         return ret;
3521  err:
3522         event->ref--;
3523
3524         goto out;
3525 }
3526
3527 static int action_create(struct hist_trigger_data *hist_data,
3528                          struct action_data *data)
3529 {
3530         struct trace_event_file *file = hist_data->event_file;
3531         struct trace_array *tr = file->tr;
3532         struct track_data *track_data;
3533         struct field_var *field_var;
3534         unsigned int i;
3535         char *param;
3536         int ret = 0;
3537
3538         if (data->action == ACTION_TRACE)
3539                 return trace_action_create(hist_data, data);
3540
3541         if (data->action == ACTION_SNAPSHOT) {
3542                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3543                 if (IS_ERR(track_data)) {
3544                         ret = PTR_ERR(track_data);
3545                         goto out;
3546                 }
3547
3548                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3549                                                    cond_snapshot_update);
3550                 if (ret)
3551                         track_data_free(track_data);
3552
3553                 goto out;
3554         }
3555
3556         if (data->action == ACTION_SAVE) {
3557                 if (hist_data->n_save_vars) {
3558                         ret = -EEXIST;
3559                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3560                         goto out;
3561                 }
3562
3563                 for (i = 0; i < data->n_params; i++) {
3564                         param = kstrdup(data->params[i], GFP_KERNEL);
3565                         if (!param) {
3566                                 ret = -ENOMEM;
3567                                 goto out;
3568                         }
3569
3570                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3571                         if (IS_ERR(field_var)) {
3572                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3573                                          errpos(param));
3574                                 ret = PTR_ERR(field_var);
3575                                 kfree(param);
3576                                 goto out;
3577                         }
3578
3579                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3580                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3581                                 hist_data->n_save_var_str++;
3582                         kfree(param);
3583                 }
3584         }
3585  out:
3586         return ret;
3587 }
3588
3589 static int onmatch_create(struct hist_trigger_data *hist_data,
3590                           struct action_data *data)
3591 {
3592         return action_create(hist_data, data);
3593 }
3594
3595 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3596 {
3597         char *match_event, *match_event_system;
3598         struct action_data *data;
3599         int ret = -EINVAL;
3600
3601         data = kzalloc(sizeof(*data), GFP_KERNEL);
3602         if (!data)
3603                 return ERR_PTR(-ENOMEM);
3604
3605         match_event = strsep(&str, ")");
3606         if (!match_event || !str) {
3607                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3608                 goto free;
3609         }
3610
3611         match_event_system = strsep(&match_event, ".");
3612         if (!match_event) {
3613                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3614                 goto free;
3615         }
3616
3617         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3618                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3619                 goto free;
3620         }
3621
3622         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3623         if (!data->match_data.event) {
3624                 ret = -ENOMEM;
3625                 goto free;
3626         }
3627
3628         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3629         if (!data->match_data.event_system) {
3630                 ret = -ENOMEM;
3631                 goto free;
3632         }
3633
3634         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3635         if (ret)
3636                 goto free;
3637  out:
3638         return data;
3639  free:
3640         onmatch_destroy(data);
3641         data = ERR_PTR(ret);
3642         goto out;
3643 }
3644
3645 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3646 {
3647         hist_data->fields[HITCOUNT_IDX] =
3648                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3649         if (!hist_data->fields[HITCOUNT_IDX])
3650                 return -ENOMEM;
3651
3652         hist_data->n_vals++;
3653         hist_data->n_fields++;
3654
3655         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3656                 return -EINVAL;
3657
3658         return 0;
3659 }
3660
3661 static int __create_val_field(struct hist_trigger_data *hist_data,
3662                               unsigned int val_idx,
3663                               struct trace_event_file *file,
3664                               char *var_name, char *field_str,
3665                               unsigned long flags)
3666 {
3667         struct hist_field *hist_field;
3668         int ret = 0;
3669
3670         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3671         if (IS_ERR(hist_field)) {
3672                 ret = PTR_ERR(hist_field);
3673                 goto out;
3674         }
3675
3676         hist_data->fields[val_idx] = hist_field;
3677
3678         ++hist_data->n_vals;
3679         ++hist_data->n_fields;
3680
3681         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3682                 ret = -EINVAL;
3683  out:
3684         return ret;
3685 }
3686
3687 static int create_val_field(struct hist_trigger_data *hist_data,
3688                             unsigned int val_idx,
3689                             struct trace_event_file *file,
3690                             char *field_str)
3691 {
3692         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3693                 return -EINVAL;
3694
3695         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3696 }
3697
3698 static int create_var_field(struct hist_trigger_data *hist_data,
3699                             unsigned int val_idx,
3700                             struct trace_event_file *file,
3701                             char *var_name, char *expr_str)
3702 {
3703         struct trace_array *tr = hist_data->event_file->tr;
3704         unsigned long flags = 0;
3705         int ret;
3706
3707         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3708                 return -EINVAL;
3709
3710         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3711                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3712                 return -EINVAL;
3713         }
3714
3715         flags |= HIST_FIELD_FL_VAR;
3716         hist_data->n_vars++;
3717         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3718                 return -EINVAL;
3719
3720         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3721
3722         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3723                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3724
3725         return ret;
3726 }
3727
3728 static int create_val_fields(struct hist_trigger_data *hist_data,
3729                              struct trace_event_file *file)
3730 {
3731         char *fields_str, *field_str;
3732         unsigned int i, j = 1;
3733         int ret;
3734
3735         ret = create_hitcount_val(hist_data);
3736         if (ret)
3737                 goto out;
3738
3739         fields_str = hist_data->attrs->vals_str;
3740         if (!fields_str)
3741                 goto out;
3742
3743         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3744                      j < TRACING_MAP_VALS_MAX; i++) {
3745                 field_str = strsep(&fields_str, ",");
3746                 if (!field_str)
3747                         break;
3748
3749                 if (strcmp(field_str, "hitcount") == 0)
3750                         continue;
3751
3752                 ret = create_val_field(hist_data, j++, file, field_str);
3753                 if (ret)
3754                         goto out;
3755         }
3756
3757         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3758                 ret = -EINVAL;
3759  out:
3760         return ret;
3761 }
3762
3763 static int create_key_field(struct hist_trigger_data *hist_data,
3764                             unsigned int key_idx,
3765                             unsigned int key_offset,
3766                             struct trace_event_file *file,
3767                             char *field_str)
3768 {
3769         struct trace_array *tr = hist_data->event_file->tr;
3770         struct hist_field *hist_field = NULL;
3771         unsigned long flags = 0;
3772         unsigned int key_size;
3773         int ret = 0;
3774
3775         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3776                 return -EINVAL;
3777
3778         flags |= HIST_FIELD_FL_KEY;
3779
3780         if (strcmp(field_str, "stacktrace") == 0) {
3781                 flags |= HIST_FIELD_FL_STACKTRACE;
3782                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3783                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3784         } else {
3785                 hist_field = parse_expr(hist_data, file, field_str, flags,
3786                                         NULL, 0);
3787                 if (IS_ERR(hist_field)) {
3788                         ret = PTR_ERR(hist_field);
3789                         goto out;
3790                 }
3791
3792                 if (field_has_hist_vars(hist_field, 0)) {
3793                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3794                         destroy_hist_field(hist_field, 0);
3795                         ret = -EINVAL;
3796                         goto out;
3797                 }
3798
3799                 key_size = hist_field->size;
3800         }
3801
3802         hist_data->fields[key_idx] = hist_field;
3803
3804         key_size = ALIGN(key_size, sizeof(u64));
3805         hist_data->fields[key_idx]->size = key_size;
3806         hist_data->fields[key_idx]->offset = key_offset;
3807
3808         hist_data->key_size += key_size;
3809
3810         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3811                 ret = -EINVAL;
3812                 goto out;
3813         }
3814
3815         hist_data->n_keys++;
3816         hist_data->n_fields++;
3817
3818         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3819                 return -EINVAL;
3820
3821         ret = key_size;
3822  out:
3823         return ret;
3824 }
3825
3826 static int create_key_fields(struct hist_trigger_data *hist_data,
3827                              struct trace_event_file *file)
3828 {
3829         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3830         char *fields_str, *field_str;
3831         int ret = -EINVAL;
3832
3833         fields_str = hist_data->attrs->keys_str;
3834         if (!fields_str)
3835                 goto out;
3836
3837         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3838                 field_str = strsep(&fields_str, ",");
3839                 if (!field_str)
3840                         break;
3841                 ret = create_key_field(hist_data, i, key_offset,
3842                                        file, field_str);
3843                 if (ret < 0)
3844                         goto out;
3845                 key_offset += ret;
3846         }
3847         if (fields_str) {
3848                 ret = -EINVAL;
3849                 goto out;
3850         }
3851         ret = 0;
3852  out:
3853         return ret;
3854 }
3855
3856 static int create_var_fields(struct hist_trigger_data *hist_data,
3857                              struct trace_event_file *file)
3858 {
3859         unsigned int i, j = hist_data->n_vals;
3860         int ret = 0;
3861
3862         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3863
3864         for (i = 0; i < n_vars; i++) {
3865                 char *var_name = hist_data->attrs->var_defs.name[i];
3866                 char *expr = hist_data->attrs->var_defs.expr[i];
3867
3868                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3869                 if (ret)
3870                         goto out;
3871         }
3872  out:
3873         return ret;
3874 }
3875
3876 static void free_var_defs(struct hist_trigger_data *hist_data)
3877 {
3878         unsigned int i;
3879
3880         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3881                 kfree(hist_data->attrs->var_defs.name[i]);
3882                 kfree(hist_data->attrs->var_defs.expr[i]);
3883         }
3884
3885         hist_data->attrs->var_defs.n_vars = 0;
3886 }
3887
3888 static int parse_var_defs(struct hist_trigger_data *hist_data)
3889 {
3890         struct trace_array *tr = hist_data->event_file->tr;
3891         char *s, *str, *var_name, *field_str;
3892         unsigned int i, j, n_vars = 0;
3893         int ret = 0;
3894
3895         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3896                 str = hist_data->attrs->assignment_str[i];
3897                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3898                         field_str = strsep(&str, ",");
3899                         if (!field_str)
3900                                 break;
3901
3902                         var_name = strsep(&field_str, "=");
3903                         if (!var_name || !field_str) {
3904                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3905                                          errpos(var_name));
3906                                 ret = -EINVAL;
3907                                 goto free;
3908                         }
3909
3910                         if (n_vars == TRACING_MAP_VARS_MAX) {
3911                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3912                                 ret = -EINVAL;
3913                                 goto free;
3914                         }
3915
3916                         s = kstrdup(var_name, GFP_KERNEL);
3917                         if (!s) {
3918                                 ret = -ENOMEM;
3919                                 goto free;
3920                         }
3921                         hist_data->attrs->var_defs.name[n_vars] = s;
3922
3923                         s = kstrdup(field_str, GFP_KERNEL);
3924                         if (!s) {
3925                                 ret = -ENOMEM;
3926                                 goto free;
3927                         }
3928                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3929
3930                         hist_data->attrs->var_defs.n_vars = n_vars;
3931                 }
3932         }
3933
3934         return ret;
3935  free:
3936         free_var_defs(hist_data);
3937
3938         return ret;
3939 }
3940
3941 static int create_hist_fields(struct hist_trigger_data *hist_data,
3942                               struct trace_event_file *file)
3943 {
3944         int ret;
3945
3946         ret = parse_var_defs(hist_data);
3947         if (ret)
3948                 goto out;
3949
3950         ret = create_val_fields(hist_data, file);
3951         if (ret)
3952                 goto out;
3953
3954         ret = create_var_fields(hist_data, file);
3955         if (ret)
3956                 goto out;
3957
3958         ret = create_key_fields(hist_data, file);
3959         if (ret)
3960                 goto out;
3961  out:
3962         free_var_defs(hist_data);
3963
3964         return ret;
3965 }
3966
3967 static int is_descending(struct trace_array *tr, const char *str)
3968 {
3969         if (!str)
3970                 return 0;
3971
3972         if (strcmp(str, "descending") == 0)
3973                 return 1;
3974
3975         if (strcmp(str, "ascending") == 0)
3976                 return 0;
3977
3978         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3979
3980         return -EINVAL;
3981 }
3982
3983 static int create_sort_keys(struct hist_trigger_data *hist_data)
3984 {
3985         struct trace_array *tr = hist_data->event_file->tr;
3986         char *fields_str = hist_data->attrs->sort_key_str;
3987         struct tracing_map_sort_key *sort_key;
3988         int descending, ret = 0;
3989         unsigned int i, j, k;
3990
3991         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3992
3993         if (!fields_str)
3994                 goto out;
3995
3996         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3997                 struct hist_field *hist_field;
3998                 char *field_str, *field_name;
3999                 const char *test_name;
4000
4001                 sort_key = &hist_data->sort_keys[i];
4002
4003                 field_str = strsep(&fields_str, ",");
4004                 if (!field_str)
4005                         break;
4006
4007                 if (!*field_str) {
4008                         ret = -EINVAL;
4009                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4010                         break;
4011                 }
4012
4013                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4014                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4015                         ret = -EINVAL;
4016                         break;
4017                 }
4018
4019                 field_name = strsep(&field_str, ".");
4020                 if (!field_name || !*field_name) {
4021                         ret = -EINVAL;
4022                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4023                         break;
4024                 }
4025
4026                 if (strcmp(field_name, "hitcount") == 0) {
4027                         descending = is_descending(tr, field_str);
4028                         if (descending < 0) {
4029                                 ret = descending;
4030                                 break;
4031                         }
4032                         sort_key->descending = descending;
4033                         continue;
4034                 }
4035
4036                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4037                         unsigned int idx;
4038
4039                         hist_field = hist_data->fields[j];
4040                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4041                                 continue;
4042
4043                         idx = k++;
4044
4045                         test_name = hist_field_name(hist_field, 0);
4046
4047                         if (strcmp(field_name, test_name) == 0) {
4048                                 sort_key->field_idx = idx;
4049                                 descending = is_descending(tr, field_str);
4050                                 if (descending < 0) {
4051                                         ret = descending;
4052                                         goto out;
4053                                 }
4054                                 sort_key->descending = descending;
4055                                 break;
4056                         }
4057                 }
4058                 if (j == hist_data->n_fields) {
4059                         ret = -EINVAL;
4060                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4061                         break;
4062                 }
4063         }
4064
4065         hist_data->n_sort_keys = i;
4066  out:
4067         return ret;
4068 }
4069
4070 static void destroy_actions(struct hist_trigger_data *hist_data)
4071 {
4072         unsigned int i;
4073
4074         for (i = 0; i < hist_data->n_actions; i++) {
4075                 struct action_data *data = hist_data->actions[i];
4076
4077                 if (data->handler == HANDLER_ONMATCH)
4078                         onmatch_destroy(data);
4079                 else if (data->handler == HANDLER_ONMAX ||
4080                          data->handler == HANDLER_ONCHANGE)
4081                         track_data_destroy(hist_data, data);
4082                 else
4083                         kfree(data);
4084         }
4085 }
4086
4087 static int parse_actions(struct hist_trigger_data *hist_data)
4088 {
4089         struct trace_array *tr = hist_data->event_file->tr;
4090         struct action_data *data;
4091         unsigned int i;
4092         int ret = 0;
4093         char *str;
4094         int len;
4095
4096         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4097                 str = hist_data->attrs->action_str[i];
4098
4099                 if ((len = str_has_prefix(str, "onmatch("))) {
4100                         char *action_str = str + len;
4101
4102                         data = onmatch_parse(tr, action_str);
4103                         if (IS_ERR(data)) {
4104                                 ret = PTR_ERR(data);
4105                                 break;
4106                         }
4107                 } else if ((len = str_has_prefix(str, "onmax("))) {
4108                         char *action_str = str + len;
4109
4110                         data = track_data_parse(hist_data, action_str,
4111                                                 HANDLER_ONMAX);
4112                         if (IS_ERR(data)) {
4113                                 ret = PTR_ERR(data);
4114                                 break;
4115                         }
4116                 } else if ((len = str_has_prefix(str, "onchange("))) {
4117                         char *action_str = str + len;
4118
4119                         data = track_data_parse(hist_data, action_str,
4120                                                 HANDLER_ONCHANGE);
4121                         if (IS_ERR(data)) {
4122                                 ret = PTR_ERR(data);
4123                                 break;
4124                         }
4125                 } else {
4126                         ret = -EINVAL;
4127                         break;
4128                 }
4129
4130                 hist_data->actions[hist_data->n_actions++] = data;
4131         }
4132
4133         return ret;
4134 }
4135
4136 static int create_actions(struct hist_trigger_data *hist_data)
4137 {
4138         struct action_data *data;
4139         unsigned int i;
4140         int ret = 0;
4141
4142         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4143                 data = hist_data->actions[i];
4144
4145                 if (data->handler == HANDLER_ONMATCH) {
4146                         ret = onmatch_create(hist_data, data);
4147                         if (ret)
4148                                 break;
4149                 } else if (data->handler == HANDLER_ONMAX ||
4150                            data->handler == HANDLER_ONCHANGE) {
4151                         ret = track_data_create(hist_data, data);
4152                         if (ret)
4153                                 break;
4154                 } else {
4155                         ret = -EINVAL;
4156                         break;
4157                 }
4158         }
4159
4160         return ret;
4161 }
4162
4163 static void print_actions(struct seq_file *m,
4164                           struct hist_trigger_data *hist_data,
4165                           struct tracing_map_elt *elt)
4166 {
4167         unsigned int i;
4168
4169         for (i = 0; i < hist_data->n_actions; i++) {
4170                 struct action_data *data = hist_data->actions[i];
4171
4172                 if (data->action == ACTION_SNAPSHOT)
4173                         continue;
4174
4175                 if (data->handler == HANDLER_ONMAX ||
4176                     data->handler == HANDLER_ONCHANGE)
4177                         track_data_print(m, hist_data, elt, data);
4178         }
4179 }
4180
4181 static void print_action_spec(struct seq_file *m,
4182                               struct hist_trigger_data *hist_data,
4183                               struct action_data *data)
4184 {
4185         unsigned int i;
4186
4187         if (data->action == ACTION_SAVE) {
4188                 for (i = 0; i < hist_data->n_save_vars; i++) {
4189                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4190                         if (i < hist_data->n_save_vars - 1)
4191                                 seq_puts(m, ",");
4192                 }
4193         } else if (data->action == ACTION_TRACE) {
4194                 if (data->use_trace_keyword)
4195                         seq_printf(m, "%s", data->synth_event_name);
4196                 for (i = 0; i < data->n_params; i++) {
4197                         if (i || data->use_trace_keyword)
4198                                 seq_puts(m, ",");
4199                         seq_printf(m, "%s", data->params[i]);
4200                 }
4201         }
4202 }
4203
4204 static void print_track_data_spec(struct seq_file *m,
4205                                   struct hist_trigger_data *hist_data,
4206                                   struct action_data *data)
4207 {
4208         if (data->handler == HANDLER_ONMAX)
4209                 seq_puts(m, ":onmax(");
4210         else if (data->handler == HANDLER_ONCHANGE)
4211                 seq_puts(m, ":onchange(");
4212         seq_printf(m, "%s", data->track_data.var_str);
4213         seq_printf(m, ").%s(", data->action_name);
4214
4215         print_action_spec(m, hist_data, data);
4216
4217         seq_puts(m, ")");
4218 }
4219
4220 static void print_onmatch_spec(struct seq_file *m,
4221                                struct hist_trigger_data *hist_data,
4222                                struct action_data *data)
4223 {
4224         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4225                    data->match_data.event);
4226
4227         seq_printf(m, "%s(", data->action_name);
4228
4229         print_action_spec(m, hist_data, data);
4230
4231         seq_puts(m, ")");
4232 }
4233
4234 static bool actions_match(struct hist_trigger_data *hist_data,
4235                           struct hist_trigger_data *hist_data_test)
4236 {
4237         unsigned int i, j;
4238
4239         if (hist_data->n_actions != hist_data_test->n_actions)
4240                 return false;
4241
4242         for (i = 0; i < hist_data->n_actions; i++) {
4243                 struct action_data *data = hist_data->actions[i];
4244                 struct action_data *data_test = hist_data_test->actions[i];
4245                 char *action_name, *action_name_test;
4246
4247                 if (data->handler != data_test->handler)
4248                         return false;
4249                 if (data->action != data_test->action)
4250                         return false;
4251
4252                 if (data->n_params != data_test->n_params)
4253                         return false;
4254
4255                 for (j = 0; j < data->n_params; j++) {
4256                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4257                                 return false;
4258                 }
4259
4260                 if (data->use_trace_keyword)
4261                         action_name = data->synth_event_name;
4262                 else
4263                         action_name = data->action_name;
4264
4265                 if (data_test->use_trace_keyword)
4266                         action_name_test = data_test->synth_event_name;
4267                 else
4268                         action_name_test = data_test->action_name;
4269
4270                 if (strcmp(action_name, action_name_test) != 0)
4271                         return false;
4272
4273                 if (data->handler == HANDLER_ONMATCH) {
4274                         if (strcmp(data->match_data.event_system,
4275                                    data_test->match_data.event_system) != 0)
4276                                 return false;
4277                         if (strcmp(data->match_data.event,
4278                                    data_test->match_data.event) != 0)
4279                                 return false;
4280                 } else if (data->handler == HANDLER_ONMAX ||
4281                            data->handler == HANDLER_ONCHANGE) {
4282                         if (strcmp(data->track_data.var_str,
4283                                    data_test->track_data.var_str) != 0)
4284                                 return false;
4285                 }
4286         }
4287
4288         return true;
4289 }
4290
4291
4292 static void print_actions_spec(struct seq_file *m,
4293                                struct hist_trigger_data *hist_data)
4294 {
4295         unsigned int i;
4296
4297         for (i = 0; i < hist_data->n_actions; i++) {
4298                 struct action_data *data = hist_data->actions[i];
4299
4300                 if (data->handler == HANDLER_ONMATCH)
4301                         print_onmatch_spec(m, hist_data, data);
4302                 else if (data->handler == HANDLER_ONMAX ||
4303                          data->handler == HANDLER_ONCHANGE)
4304                         print_track_data_spec(m, hist_data, data);
4305         }
4306 }
4307
4308 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4309 {
4310         unsigned int i;
4311
4312         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4313                 kfree(hist_data->field_var_hists[i]->cmd);
4314                 kfree(hist_data->field_var_hists[i]);
4315         }
4316 }
4317
4318 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4319 {
4320         if (!hist_data)
4321                 return;
4322
4323         destroy_hist_trigger_attrs(hist_data->attrs);
4324         destroy_hist_fields(hist_data);
4325         tracing_map_destroy(hist_data->map);
4326
4327         destroy_actions(hist_data);
4328         destroy_field_vars(hist_data);
4329         destroy_field_var_hists(hist_data);
4330
4331         kfree(hist_data);
4332 }
4333
4334 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4335 {
4336         struct tracing_map *map = hist_data->map;
4337         struct ftrace_event_field *field;
4338         struct hist_field *hist_field;
4339         int i, idx = 0;
4340
4341         for_each_hist_field(i, hist_data) {
4342                 hist_field = hist_data->fields[i];
4343                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4344                         tracing_map_cmp_fn_t cmp_fn;
4345
4346                         field = hist_field->field;
4347
4348                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4349                                 cmp_fn = tracing_map_cmp_none;
4350                         else if (!field)
4351                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4352                                                              hist_field->is_signed);
4353                         else if (is_string_field(field))
4354                                 cmp_fn = tracing_map_cmp_string;
4355                         else
4356                                 cmp_fn = tracing_map_cmp_num(field->size,
4357                                                              field->is_signed);
4358                         idx = tracing_map_add_key_field(map,
4359                                                         hist_field->offset,
4360                                                         cmp_fn);
4361                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4362                         idx = tracing_map_add_sum_field(map);
4363
4364                 if (idx < 0)
4365                         return idx;
4366
4367                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4368                         idx = tracing_map_add_var(map);
4369                         if (idx < 0)
4370                                 return idx;
4371                         hist_field->var.idx = idx;
4372                         hist_field->var.hist_data = hist_data;
4373                 }
4374         }
4375
4376         return 0;
4377 }
4378
4379 static struct hist_trigger_data *
4380 create_hist_data(unsigned int map_bits,
4381                  struct hist_trigger_attrs *attrs,
4382                  struct trace_event_file *file,
4383                  bool remove)
4384 {
4385         const struct tracing_map_ops *map_ops = NULL;
4386         struct hist_trigger_data *hist_data;
4387         int ret = 0;
4388
4389         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4390         if (!hist_data)
4391                 return ERR_PTR(-ENOMEM);
4392
4393         hist_data->attrs = attrs;
4394         hist_data->remove = remove;
4395         hist_data->event_file = file;
4396
4397         ret = parse_actions(hist_data);
4398         if (ret)
4399                 goto free;
4400
4401         ret = create_hist_fields(hist_data, file);
4402         if (ret)
4403                 goto free;
4404
4405         ret = create_sort_keys(hist_data);
4406         if (ret)
4407                 goto free;
4408
4409         map_ops = &hist_trigger_elt_data_ops;
4410
4411         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4412                                             map_ops, hist_data);
4413         if (IS_ERR(hist_data->map)) {
4414                 ret = PTR_ERR(hist_data->map);
4415                 hist_data->map = NULL;
4416                 goto free;
4417         }
4418
4419         ret = create_tracing_map_fields(hist_data);
4420         if (ret)
4421                 goto free;
4422  out:
4423         return hist_data;
4424  free:
4425         hist_data->attrs = NULL;
4426
4427         destroy_hist_data(hist_data);
4428
4429         hist_data = ERR_PTR(ret);
4430
4431         goto out;
4432 }
4433
4434 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4435                                     struct tracing_map_elt *elt,
4436                                     struct trace_buffer *buffer, void *rec,
4437                                     struct ring_buffer_event *rbe,
4438                                     u64 *var_ref_vals)
4439 {
4440         struct hist_elt_data *elt_data;
4441         struct hist_field *hist_field;
4442         unsigned int i, var_idx;
4443         u64 hist_val;
4444
4445         elt_data = elt->private_data;
4446         elt_data->var_ref_vals = var_ref_vals;
4447
4448         for_each_hist_val_field(i, hist_data) {
4449                 hist_field = hist_data->fields[i];
4450                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4451                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4452                         var_idx = hist_field->var.idx;
4453
4454                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4455                                 unsigned int str_start, var_str_idx, idx;
4456                                 char *str, *val_str;
4457
4458                                 str_start = hist_data->n_field_var_str +
4459                                         hist_data->n_save_var_str;
4460                                 var_str_idx = hist_field->var_str_idx;
4461                                 idx = str_start + var_str_idx;
4462
4463                                 str = elt_data->field_var_str[idx];
4464                                 val_str = (char *)(uintptr_t)hist_val;
4465                                 strscpy(str, val_str, STR_VAR_LEN_MAX);
4466
4467                                 hist_val = (u64)(uintptr_t)str;
4468                         }
4469                         tracing_map_set_var(elt, var_idx, hist_val);
4470                         continue;
4471                 }
4472                 tracing_map_update_sum(elt, i, hist_val);
4473         }
4474
4475         for_each_hist_key_field(i, hist_data) {
4476                 hist_field = hist_data->fields[i];
4477                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4478                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4479                         var_idx = hist_field->var.idx;
4480                         tracing_map_set_var(elt, var_idx, hist_val);
4481                 }
4482         }
4483
4484         update_field_vars(hist_data, elt, buffer, rbe, rec);
4485 }
4486
4487 static inline void add_to_key(char *compound_key, void *key,
4488                               struct hist_field *key_field, void *rec)
4489 {
4490         size_t size = key_field->size;
4491
4492         if (key_field->flags & HIST_FIELD_FL_STRING) {
4493                 struct ftrace_event_field *field;
4494
4495                 field = key_field->field;
4496                 if (field->filter_type == FILTER_DYN_STRING)
4497                         size = *(u32 *)(rec + field->offset) >> 16;
4498                 else if (field->filter_type == FILTER_PTR_STRING)
4499                         size = strlen(key);
4500                 else if (field->filter_type == FILTER_STATIC_STRING)
4501                         size = field->size;
4502
4503                 /* ensure NULL-termination */
4504                 if (size > key_field->size - 1)
4505                         size = key_field->size - 1;
4506
4507                 strncpy(compound_key + key_field->offset, (char *)key, size);
4508         } else
4509                 memcpy(compound_key + key_field->offset, key, size);
4510 }
4511
4512 static void
4513 hist_trigger_actions(struct hist_trigger_data *hist_data,
4514                      struct tracing_map_elt *elt,
4515                      struct trace_buffer *buffer, void *rec,
4516                      struct ring_buffer_event *rbe, void *key,
4517                      u64 *var_ref_vals)
4518 {
4519         struct action_data *data;
4520         unsigned int i;
4521
4522         for (i = 0; i < hist_data->n_actions; i++) {
4523                 data = hist_data->actions[i];
4524                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4525         }
4526 }
4527
4528 static void event_hist_trigger(struct event_trigger_data *data,
4529                                struct trace_buffer *buffer, void *rec,
4530                                struct ring_buffer_event *rbe)
4531 {
4532         struct hist_trigger_data *hist_data = data->private_data;
4533         bool use_compound_key = (hist_data->n_keys > 1);
4534         unsigned long entries[HIST_STACKTRACE_DEPTH];
4535         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4536         char compound_key[HIST_KEY_SIZE_MAX];
4537         struct tracing_map_elt *elt = NULL;
4538         struct hist_field *key_field;
4539         u64 field_contents;
4540         void *key = NULL;
4541         unsigned int i;
4542
4543         memset(compound_key, 0, hist_data->key_size);
4544
4545         for_each_hist_key_field(i, hist_data) {
4546                 key_field = hist_data->fields[i];
4547
4548                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4549                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4550                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4551                                          HIST_STACKTRACE_SKIP);
4552                         key = entries;
4553                 } else {
4554                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4555                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4556                                 key = (void *)(unsigned long)field_contents;
4557                                 use_compound_key = true;
4558                         } else
4559                                 key = (void *)&field_contents;
4560                 }
4561
4562                 if (use_compound_key)
4563                         add_to_key(compound_key, key, key_field, rec);
4564         }
4565
4566         if (use_compound_key)
4567                 key = compound_key;
4568
4569         if (hist_data->n_var_refs &&
4570             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4571                 return;
4572
4573         elt = tracing_map_insert(hist_data->map, key);
4574         if (!elt)
4575                 return;
4576
4577         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4578
4579         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4580                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4581 }
4582
4583 static void hist_trigger_stacktrace_print(struct seq_file *m,
4584                                           unsigned long *stacktrace_entries,
4585                                           unsigned int max_entries)
4586 {
4587         char str[KSYM_SYMBOL_LEN];
4588         unsigned int spaces = 8;
4589         unsigned int i;
4590
4591         for (i = 0; i < max_entries; i++) {
4592                 if (!stacktrace_entries[i])
4593                         return;
4594
4595                 seq_printf(m, "%*c", 1 + spaces, ' ');
4596                 sprint_symbol(str, stacktrace_entries[i]);
4597                 seq_printf(m, "%s\n", str);
4598         }
4599 }
4600
4601 static void hist_trigger_print_key(struct seq_file *m,
4602                                    struct hist_trigger_data *hist_data,
4603                                    void *key,
4604                                    struct tracing_map_elt *elt)
4605 {
4606         struct hist_field *key_field;
4607         char str[KSYM_SYMBOL_LEN];
4608         bool multiline = false;
4609         const char *field_name;
4610         unsigned int i;
4611         u64 uval;
4612
4613         seq_puts(m, "{ ");
4614
4615         for_each_hist_key_field(i, hist_data) {
4616                 key_field = hist_data->fields[i];
4617
4618                 if (i > hist_data->n_vals)
4619                         seq_puts(m, ", ");
4620
4621                 field_name = hist_field_name(key_field, 0);
4622
4623                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4624                         uval = *(u64 *)(key + key_field->offset);
4625                         seq_printf(m, "%s: %llx", field_name, uval);
4626                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4627                         uval = *(u64 *)(key + key_field->offset);
4628                         sprint_symbol_no_offset(str, uval);
4629                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4630                                    uval, str);
4631                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4632                         uval = *(u64 *)(key + key_field->offset);
4633                         sprint_symbol(str, uval);
4634                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4635                                    uval, str);
4636                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4637                         struct hist_elt_data *elt_data = elt->private_data;
4638                         char *comm;
4639
4640                         if (WARN_ON_ONCE(!elt_data))
4641                                 return;
4642
4643                         comm = elt_data->comm;
4644
4645                         uval = *(u64 *)(key + key_field->offset);
4646                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4647                                    comm, uval);
4648                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4649                         const char *syscall_name;
4650
4651                         uval = *(u64 *)(key + key_field->offset);
4652                         syscall_name = get_syscall_name(uval);
4653                         if (!syscall_name)
4654                                 syscall_name = "unknown_syscall";
4655
4656                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4657                                    syscall_name, uval);
4658                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4659                         seq_puts(m, "stacktrace:\n");
4660                         hist_trigger_stacktrace_print(m,
4661                                                       key + key_field->offset,
4662                                                       HIST_STACKTRACE_DEPTH);
4663                         multiline = true;
4664                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4665                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4666                                    *(u64 *)(key + key_field->offset));
4667                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4668                         seq_printf(m, "%s: %-50s", field_name,
4669                                    (char *)(key + key_field->offset));
4670                 } else {
4671                         uval = *(u64 *)(key + key_field->offset);
4672                         seq_printf(m, "%s: %10llu", field_name, uval);
4673                 }
4674         }
4675
4676         if (!multiline)
4677                 seq_puts(m, " ");
4678
4679         seq_puts(m, "}");
4680 }
4681
4682 static void hist_trigger_entry_print(struct seq_file *m,
4683                                      struct hist_trigger_data *hist_data,
4684                                      void *key,
4685                                      struct tracing_map_elt *elt)
4686 {
4687         const char *field_name;
4688         unsigned int i;
4689
4690         hist_trigger_print_key(m, hist_data, key, elt);
4691
4692         seq_printf(m, " hitcount: %10llu",
4693                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4694
4695         for (i = 1; i < hist_data->n_vals; i++) {
4696                 field_name = hist_field_name(hist_data->fields[i], 0);
4697
4698                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4699                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4700                         continue;
4701
4702                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4703                         seq_printf(m, "  %s: %10llx", field_name,
4704                                    tracing_map_read_sum(elt, i));
4705                 } else {
4706                         seq_printf(m, "  %s: %10llu", field_name,
4707                                    tracing_map_read_sum(elt, i));
4708                 }
4709         }
4710
4711         print_actions(m, hist_data, elt);
4712
4713         seq_puts(m, "\n");
4714 }
4715
4716 static int print_entries(struct seq_file *m,
4717                          struct hist_trigger_data *hist_data)
4718 {
4719         struct tracing_map_sort_entry **sort_entries = NULL;
4720         struct tracing_map *map = hist_data->map;
4721         int i, n_entries;
4722
4723         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4724                                              hist_data->n_sort_keys,
4725                                              &sort_entries);
4726         if (n_entries < 0)
4727                 return n_entries;
4728
4729         for (i = 0; i < n_entries; i++)
4730                 hist_trigger_entry_print(m, hist_data,
4731                                          sort_entries[i]->key,
4732                                          sort_entries[i]->elt);
4733
4734         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4735
4736         return n_entries;
4737 }
4738
4739 static void hist_trigger_show(struct seq_file *m,
4740                               struct event_trigger_data *data, int n)
4741 {
4742         struct hist_trigger_data *hist_data;
4743         int n_entries;
4744
4745         if (n > 0)
4746                 seq_puts(m, "\n\n");
4747
4748         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4749         data->ops->print(m, data->ops, data);
4750         seq_puts(m, "#\n\n");
4751
4752         hist_data = data->private_data;
4753         n_entries = print_entries(m, hist_data);
4754         if (n_entries < 0)
4755                 n_entries = 0;
4756
4757         track_data_snapshot_print(m, hist_data);
4758
4759         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4760                    (u64)atomic64_read(&hist_data->map->hits),
4761                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4762 }
4763
4764 static int hist_show(struct seq_file *m, void *v)
4765 {
4766         struct event_trigger_data *data;
4767         struct trace_event_file *event_file;
4768         int n = 0, ret = 0;
4769
4770         mutex_lock(&event_mutex);
4771
4772         event_file = event_file_data(m->private);
4773         if (unlikely(!event_file)) {
4774                 ret = -ENODEV;
4775                 goto out_unlock;
4776         }
4777
4778         list_for_each_entry(data, &event_file->triggers, list) {
4779                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4780                         hist_trigger_show(m, data, n++);
4781         }
4782
4783  out_unlock:
4784         mutex_unlock(&event_mutex);
4785
4786         return ret;
4787 }
4788
4789 static int event_hist_open(struct inode *inode, struct file *file)
4790 {
4791         int ret;
4792
4793         ret = security_locked_down(LOCKDOWN_TRACEFS);
4794         if (ret)
4795                 return ret;
4796
4797         return single_open(file, hist_show, file);
4798 }
4799
4800 const struct file_operations event_hist_fops = {
4801         .open = event_hist_open,
4802         .read = seq_read,
4803         .llseek = seq_lseek,
4804         .release = single_release,
4805 };
4806
4807 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4808 static void hist_field_debug_show_flags(struct seq_file *m,
4809                                         unsigned long flags)
4810 {
4811         seq_puts(m, "      flags:\n");
4812
4813         if (flags & HIST_FIELD_FL_KEY)
4814                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4815         else if (flags & HIST_FIELD_FL_HITCOUNT)
4816                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4817         else if (flags & HIST_FIELD_FL_VAR)
4818                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4819         else if (flags & HIST_FIELD_FL_VAR_REF)
4820                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4821         else
4822                 seq_puts(m, "        VAL: normal u64 value\n");
4823
4824         if (flags & HIST_FIELD_FL_ALIAS)
4825                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4826 }
4827
4828 static int hist_field_debug_show(struct seq_file *m,
4829                                  struct hist_field *field, unsigned long flags)
4830 {
4831         if ((field->flags & flags) != flags) {
4832                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4833                 return -EINVAL;
4834         }
4835
4836         hist_field_debug_show_flags(m, field->flags);
4837         if (field->field)
4838                 seq_printf(m, "      ftrace_event_field name: %s\n",
4839                            field->field->name);
4840
4841         if (field->flags & HIST_FIELD_FL_VAR) {
4842                 seq_printf(m, "      var.name: %s\n", field->var.name);
4843                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4844                            field->var.idx);
4845         }
4846
4847         if (field->flags & HIST_FIELD_FL_ALIAS)
4848                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4849                            field->var_ref_idx);
4850
4851         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4852                 seq_printf(m, "      name: %s\n", field->name);
4853                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4854                            field->var.idx);
4855                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4856                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4857                            field->var_ref_idx);
4858                 if (field->system)
4859                         seq_printf(m, "      system: %s\n", field->system);
4860                 if (field->event_name)
4861                         seq_printf(m, "      event_name: %s\n", field->event_name);
4862         }
4863
4864         seq_printf(m, "      type: %s\n", field->type);
4865         seq_printf(m, "      size: %u\n", field->size);
4866         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4867
4868         return 0;
4869 }
4870
4871 static int field_var_debug_show(struct seq_file *m,
4872                                 struct field_var *field_var, unsigned int i,
4873                                 bool save_vars)
4874 {
4875         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4876         struct hist_field *field;
4877         int ret = 0;
4878
4879         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4880
4881         field = field_var->var;
4882
4883         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4884
4885         hist_field_debug_show_flags(m, field->flags);
4886         seq_printf(m, "      var.name: %s\n", field->var.name);
4887         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4888                    field->var.idx);
4889
4890         field = field_var->val;
4891
4892         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4893         if (field->field)
4894                 seq_printf(m, "      ftrace_event_field name: %s\n",
4895                            field->field->name);
4896         else {
4897                 ret = -EINVAL;
4898                 goto out;
4899         }
4900
4901         seq_printf(m, "      type: %s\n", field->type);
4902         seq_printf(m, "      size: %u\n", field->size);
4903         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4904 out:
4905         return ret;
4906 }
4907
4908 static int hist_action_debug_show(struct seq_file *m,
4909                                   struct action_data *data, int i)
4910 {
4911         int ret = 0;
4912
4913         if (data->handler == HANDLER_ONMAX ||
4914             data->handler == HANDLER_ONCHANGE) {
4915                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4916                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4917                                             HIST_FIELD_FL_VAR_REF);
4918                 if (ret)
4919                         goto out;
4920
4921                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4922                 ret = hist_field_debug_show(m, data->track_data.track_var,
4923                                             HIST_FIELD_FL_VAR);
4924                 if (ret)
4925                         goto out;
4926         }
4927
4928         if (data->handler == HANDLER_ONMATCH) {
4929                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4930                            i, data->match_data.event_system);
4931                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4932                            i, data->match_data.event);
4933         }
4934 out:
4935         return ret;
4936 }
4937
4938 static int hist_actions_debug_show(struct seq_file *m,
4939                                    struct hist_trigger_data *hist_data)
4940 {
4941         int i, ret = 0;
4942
4943         if (hist_data->n_actions)
4944                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4945
4946         for (i = 0; i < hist_data->n_actions; i++) {
4947                 struct action_data *action = hist_data->actions[i];
4948
4949                 ret = hist_action_debug_show(m, action, i);
4950                 if (ret)
4951                         goto out;
4952         }
4953
4954         if (hist_data->n_save_vars)
4955                 seq_puts(m, "\n  save action variables (save() params):\n");
4956
4957         for (i = 0; i < hist_data->n_save_vars; i++) {
4958                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4959                 if (ret)
4960                         goto out;
4961         }
4962 out:
4963         return ret;
4964 }
4965
4966 static void hist_trigger_debug_show(struct seq_file *m,
4967                                     struct event_trigger_data *data, int n)
4968 {
4969         struct hist_trigger_data *hist_data;
4970         int i, ret;
4971
4972         if (n > 0)
4973                 seq_puts(m, "\n\n");
4974
4975         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4976         data->ops->print(m, data->ops, data);
4977         seq_puts(m, "#\n\n");
4978
4979         hist_data = data->private_data;
4980
4981         seq_printf(m, "hist_data: %p\n\n", hist_data);
4982         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4983         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4984         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4985
4986         seq_puts(m, "\n  val fields:\n\n");
4987
4988         seq_puts(m, "    hist_data->fields[0]:\n");
4989         ret = hist_field_debug_show(m, hist_data->fields[0],
4990                                     HIST_FIELD_FL_HITCOUNT);
4991         if (ret)
4992                 return;
4993
4994         for (i = 1; i < hist_data->n_vals; i++) {
4995                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4996                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4997                 if (ret)
4998                         return;
4999         }
5000
5001         seq_puts(m, "\n  key fields:\n");
5002
5003         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5004                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5005                 ret = hist_field_debug_show(m, hist_data->fields[i],
5006                                             HIST_FIELD_FL_KEY);
5007                 if (ret)
5008                         return;
5009         }
5010
5011         if (hist_data->n_var_refs)
5012                 seq_puts(m, "\n  variable reference fields:\n");
5013
5014         for (i = 0; i < hist_data->n_var_refs; i++) {
5015                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5016                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5017                                             HIST_FIELD_FL_VAR_REF);
5018                 if (ret)
5019                         return;
5020         }
5021
5022         if (hist_data->n_field_vars)
5023                 seq_puts(m, "\n  field variables:\n");
5024
5025         for (i = 0; i < hist_data->n_field_vars; i++) {
5026                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5027                 if (ret)
5028                         return;
5029         }
5030
5031         ret = hist_actions_debug_show(m, hist_data);
5032         if (ret)
5033                 return;
5034 }
5035
5036 static int hist_debug_show(struct seq_file *m, void *v)
5037 {
5038         struct event_trigger_data *data;
5039         struct trace_event_file *event_file;
5040         int n = 0, ret = 0;
5041
5042         mutex_lock(&event_mutex);
5043
5044         event_file = event_file_data(m->private);
5045         if (unlikely(!event_file)) {
5046                 ret = -ENODEV;
5047                 goto out_unlock;
5048         }
5049
5050         list_for_each_entry(data, &event_file->triggers, list) {
5051                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5052                         hist_trigger_debug_show(m, data, n++);
5053         }
5054
5055  out_unlock:
5056         mutex_unlock(&event_mutex);
5057
5058         return ret;
5059 }
5060
5061 static int event_hist_debug_open(struct inode *inode, struct file *file)
5062 {
5063         int ret;
5064
5065         ret = security_locked_down(LOCKDOWN_TRACEFS);
5066         if (ret)
5067                 return ret;
5068
5069         return single_open(file, hist_debug_show, file);
5070 }
5071
5072 const struct file_operations event_hist_debug_fops = {
5073         .open = event_hist_debug_open,
5074         .read = seq_read,
5075         .llseek = seq_lseek,
5076         .release = single_release,
5077 };
5078 #endif
5079
5080 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5081 {
5082         const char *field_name = hist_field_name(hist_field, 0);
5083
5084         if (hist_field->var.name)
5085                 seq_printf(m, "%s=", hist_field->var.name);
5086
5087         if (hist_field->flags & HIST_FIELD_FL_CPU)
5088                 seq_puts(m, "cpu");
5089         else if (field_name) {
5090                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5091                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5092                         seq_putc(m, '$');
5093                 seq_printf(m, "%s", field_name);
5094         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5095                 seq_puts(m, "common_timestamp");
5096
5097         if (hist_field->flags) {
5098                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5099                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5100                         const char *flags = get_hist_field_flags(hist_field);
5101
5102                         if (flags)
5103                                 seq_printf(m, ".%s", flags);
5104                 }
5105         }
5106 }
5107
5108 static int event_hist_trigger_print(struct seq_file *m,
5109                                     struct event_trigger_ops *ops,
5110                                     struct event_trigger_data *data)
5111 {
5112         struct hist_trigger_data *hist_data = data->private_data;
5113         struct hist_field *field;
5114         bool have_var = false;
5115         unsigned int i;
5116
5117         seq_puts(m, "hist:");
5118
5119         if (data->name)
5120                 seq_printf(m, "%s:", data->name);
5121
5122         seq_puts(m, "keys=");
5123
5124         for_each_hist_key_field(i, hist_data) {
5125                 field = hist_data->fields[i];
5126
5127                 if (i > hist_data->n_vals)
5128                         seq_puts(m, ",");
5129
5130                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5131                         seq_puts(m, "stacktrace");
5132                 else
5133                         hist_field_print(m, field);
5134         }
5135
5136         seq_puts(m, ":vals=");
5137
5138         for_each_hist_val_field(i, hist_data) {
5139                 field = hist_data->fields[i];
5140                 if (field->flags & HIST_FIELD_FL_VAR) {
5141                         have_var = true;
5142                         continue;
5143                 }
5144
5145                 if (i == HITCOUNT_IDX)
5146                         seq_puts(m, "hitcount");
5147                 else {
5148                         seq_puts(m, ",");
5149                         hist_field_print(m, field);
5150                 }
5151         }
5152
5153         if (have_var) {
5154                 unsigned int n = 0;
5155
5156                 seq_puts(m, ":");
5157
5158                 for_each_hist_val_field(i, hist_data) {
5159                         field = hist_data->fields[i];
5160
5161                         if (field->flags & HIST_FIELD_FL_VAR) {
5162                                 if (n++)
5163                                         seq_puts(m, ",");
5164                                 hist_field_print(m, field);
5165                         }
5166                 }
5167         }
5168
5169         seq_puts(m, ":sort=");
5170
5171         for (i = 0; i < hist_data->n_sort_keys; i++) {
5172                 struct tracing_map_sort_key *sort_key;
5173                 unsigned int idx, first_key_idx;
5174
5175                 /* skip VAR vals */
5176                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5177
5178                 sort_key = &hist_data->sort_keys[i];
5179                 idx = sort_key->field_idx;
5180
5181                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5182                         return -EINVAL;
5183
5184                 if (i > 0)
5185                         seq_puts(m, ",");
5186
5187                 if (idx == HITCOUNT_IDX)
5188                         seq_puts(m, "hitcount");
5189                 else {
5190                         if (idx >= first_key_idx)
5191                                 idx += hist_data->n_vars;
5192                         hist_field_print(m, hist_data->fields[idx]);
5193                 }
5194
5195                 if (sort_key->descending)
5196                         seq_puts(m, ".descending");
5197         }
5198         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5199         if (hist_data->enable_timestamps)
5200                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5201
5202         print_actions_spec(m, hist_data);
5203
5204         if (data->filter_str)
5205                 seq_printf(m, " if %s", data->filter_str);
5206
5207         if (data->paused)
5208                 seq_puts(m, " [paused]");
5209         else
5210                 seq_puts(m, " [active]");
5211
5212         seq_putc(m, '\n');
5213
5214         return 0;
5215 }
5216
5217 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5218                                    struct event_trigger_data *data)
5219 {
5220         struct hist_trigger_data *hist_data = data->private_data;
5221
5222         if (!data->ref && hist_data->attrs->name)
5223                 save_named_trigger(hist_data->attrs->name, data);
5224
5225         data->ref++;
5226
5227         return 0;
5228 }
5229
5230 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5231 {
5232         struct trace_event_file *file;
5233         unsigned int i;
5234         char *cmd;
5235         int ret;
5236
5237         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5238                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5239                 cmd = hist_data->field_var_hists[i]->cmd;
5240                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5241                                               "!hist", "hist", cmd);
5242                 WARN_ON_ONCE(ret < 0);
5243         }
5244 }
5245
5246 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5247                                     struct event_trigger_data *data)
5248 {
5249         struct hist_trigger_data *hist_data = data->private_data;
5250
5251         if (WARN_ON_ONCE(data->ref <= 0))
5252                 return;
5253
5254         data->ref--;
5255         if (!data->ref) {
5256                 if (data->name)
5257                         del_named_trigger(data);
5258
5259                 trigger_data_free(data);
5260
5261                 remove_hist_vars(hist_data);
5262
5263                 unregister_field_var_hists(hist_data);
5264
5265                 destroy_hist_data(hist_data);
5266         }
5267 }
5268
5269 static struct event_trigger_ops event_hist_trigger_ops = {
5270         .func                   = event_hist_trigger,
5271         .print                  = event_hist_trigger_print,
5272         .init                   = event_hist_trigger_init,
5273         .free                   = event_hist_trigger_free,
5274 };
5275
5276 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5277                                          struct event_trigger_data *data)
5278 {
5279         data->ref++;
5280
5281         save_named_trigger(data->named_data->name, data);
5282
5283         event_hist_trigger_init(ops, data->named_data);
5284
5285         return 0;
5286 }
5287
5288 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5289                                           struct event_trigger_data *data)
5290 {
5291         if (WARN_ON_ONCE(data->ref <= 0))
5292                 return;
5293
5294         event_hist_trigger_free(ops, data->named_data);
5295
5296         data->ref--;
5297         if (!data->ref) {
5298                 del_named_trigger(data);
5299                 trigger_data_free(data);
5300         }
5301 }
5302
5303 static struct event_trigger_ops event_hist_trigger_named_ops = {
5304         .func                   = event_hist_trigger,
5305         .print                  = event_hist_trigger_print,
5306         .init                   = event_hist_trigger_named_init,
5307         .free                   = event_hist_trigger_named_free,
5308 };
5309
5310 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5311                                                             char *param)
5312 {
5313         return &event_hist_trigger_ops;
5314 }
5315
5316 static void hist_clear(struct event_trigger_data *data)
5317 {
5318         struct hist_trigger_data *hist_data = data->private_data;
5319
5320         if (data->name)
5321                 pause_named_trigger(data);
5322
5323         tracepoint_synchronize_unregister();
5324
5325         tracing_map_clear(hist_data->map);
5326
5327         if (data->name)
5328                 unpause_named_trigger(data);
5329 }
5330
5331 static bool compatible_field(struct ftrace_event_field *field,
5332                              struct ftrace_event_field *test_field)
5333 {
5334         if (field == test_field)
5335                 return true;
5336         if (field == NULL || test_field == NULL)
5337                 return false;
5338         if (strcmp(field->name, test_field->name) != 0)
5339                 return false;
5340         if (strcmp(field->type, test_field->type) != 0)
5341                 return false;
5342         if (field->size != test_field->size)
5343                 return false;
5344         if (field->is_signed != test_field->is_signed)
5345                 return false;
5346
5347         return true;
5348 }
5349
5350 static bool hist_trigger_match(struct event_trigger_data *data,
5351                                struct event_trigger_data *data_test,
5352                                struct event_trigger_data *named_data,
5353                                bool ignore_filter)
5354 {
5355         struct tracing_map_sort_key *sort_key, *sort_key_test;
5356         struct hist_trigger_data *hist_data, *hist_data_test;
5357         struct hist_field *key_field, *key_field_test;
5358         unsigned int i;
5359
5360         if (named_data && (named_data != data_test) &&
5361             (named_data != data_test->named_data))
5362                 return false;
5363
5364         if (!named_data && is_named_trigger(data_test))
5365                 return false;
5366
5367         hist_data = data->private_data;
5368         hist_data_test = data_test->private_data;
5369
5370         if (hist_data->n_vals != hist_data_test->n_vals ||
5371             hist_data->n_fields != hist_data_test->n_fields ||
5372             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5373                 return false;
5374
5375         if (!ignore_filter) {
5376                 if ((data->filter_str && !data_test->filter_str) ||
5377                    (!data->filter_str && data_test->filter_str))
5378                         return false;
5379         }
5380
5381         for_each_hist_field(i, hist_data) {
5382                 key_field = hist_data->fields[i];
5383                 key_field_test = hist_data_test->fields[i];
5384
5385                 if (key_field->flags != key_field_test->flags)
5386                         return false;
5387                 if (!compatible_field(key_field->field, key_field_test->field))
5388                         return false;
5389                 if (key_field->offset != key_field_test->offset)
5390                         return false;
5391                 if (key_field->size != key_field_test->size)
5392                         return false;
5393                 if (key_field->is_signed != key_field_test->is_signed)
5394                         return false;
5395                 if (!!key_field->var.name != !!key_field_test->var.name)
5396                         return false;
5397                 if (key_field->var.name &&
5398                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5399                         return false;
5400         }
5401
5402         for (i = 0; i < hist_data->n_sort_keys; i++) {
5403                 sort_key = &hist_data->sort_keys[i];
5404                 sort_key_test = &hist_data_test->sort_keys[i];
5405
5406                 if (sort_key->field_idx != sort_key_test->field_idx ||
5407                     sort_key->descending != sort_key_test->descending)
5408                         return false;
5409         }
5410
5411         if (!ignore_filter && data->filter_str &&
5412             (strcmp(data->filter_str, data_test->filter_str) != 0))
5413                 return false;
5414
5415         if (!actions_match(hist_data, hist_data_test))
5416                 return false;
5417
5418         return true;
5419 }
5420
5421 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5422                                  struct event_trigger_data *data,
5423                                  struct trace_event_file *file)
5424 {
5425         struct hist_trigger_data *hist_data = data->private_data;
5426         struct event_trigger_data *test, *named_data = NULL;
5427         struct trace_array *tr = file->tr;
5428         int ret = 0;
5429
5430         if (hist_data->attrs->name) {
5431                 named_data = find_named_trigger(hist_data->attrs->name);
5432                 if (named_data) {
5433                         if (!hist_trigger_match(data, named_data, named_data,
5434                                                 true)) {
5435                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5436                                 ret = -EINVAL;
5437                                 goto out;
5438                         }
5439                 }
5440         }
5441
5442         if (hist_data->attrs->name && !named_data)
5443                 goto new;
5444
5445         lockdep_assert_held(&event_mutex);
5446
5447         list_for_each_entry(test, &file->triggers, list) {
5448                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5449                         if (!hist_trigger_match(data, test, named_data, false))
5450                                 continue;
5451                         if (hist_data->attrs->pause)
5452                                 test->paused = true;
5453                         else if (hist_data->attrs->cont)
5454                                 test->paused = false;
5455                         else if (hist_data->attrs->clear)
5456                                 hist_clear(test);
5457                         else {
5458                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5459                                 ret = -EEXIST;
5460                         }
5461                         goto out;
5462                 }
5463         }
5464  new:
5465         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5466                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5467                 ret = -ENOENT;
5468                 goto out;
5469         }
5470
5471         if (hist_data->attrs->pause)
5472                 data->paused = true;
5473
5474         if (named_data) {
5475                 data->private_data = named_data->private_data;
5476                 set_named_trigger_data(data, named_data);
5477                 data->ops = &event_hist_trigger_named_ops;
5478         }
5479
5480         if (data->ops->init) {
5481                 ret = data->ops->init(data->ops, data);
5482                 if (ret < 0)
5483                         goto out;
5484         }
5485
5486         if (hist_data->enable_timestamps) {
5487                 char *clock = hist_data->attrs->clock;
5488
5489                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5490                 if (ret) {
5491                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5492                         goto out;
5493                 }
5494
5495                 tracing_set_filter_buffering(file->tr, true);
5496         }
5497
5498         if (named_data)
5499                 destroy_hist_data(hist_data);
5500
5501         ret++;
5502  out:
5503         return ret;
5504 }
5505
5506 static int hist_trigger_enable(struct event_trigger_data *data,
5507                                struct trace_event_file *file)
5508 {
5509         int ret = 0;
5510
5511         list_add_tail_rcu(&data->list, &file->triggers);
5512
5513         update_cond_flag(file);
5514
5515         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5516                 list_del_rcu(&data->list);
5517                 update_cond_flag(file);
5518                 ret--;
5519         }
5520
5521         return ret;
5522 }
5523
5524 static bool have_hist_trigger_match(struct event_trigger_data *data,
5525                                     struct trace_event_file *file)
5526 {
5527         struct hist_trigger_data *hist_data = data->private_data;
5528         struct event_trigger_data *test, *named_data = NULL;
5529         bool match = false;
5530
5531         lockdep_assert_held(&event_mutex);
5532
5533         if (hist_data->attrs->name)
5534                 named_data = find_named_trigger(hist_data->attrs->name);
5535
5536         list_for_each_entry(test, &file->triggers, list) {
5537                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5538                         if (hist_trigger_match(data, test, named_data, false)) {
5539                                 match = true;
5540                                 break;
5541                         }
5542                 }
5543         }
5544
5545         return match;
5546 }
5547
5548 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5549                                     struct trace_event_file *file)
5550 {
5551         struct hist_trigger_data *hist_data = data->private_data;
5552         struct event_trigger_data *test, *named_data = NULL;
5553
5554         lockdep_assert_held(&event_mutex);
5555
5556         if (hist_data->attrs->name)
5557                 named_data = find_named_trigger(hist_data->attrs->name);
5558
5559         list_for_each_entry(test, &file->triggers, list) {
5560                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5561                         if (!hist_trigger_match(data, test, named_data, false))
5562                                 continue;
5563                         hist_data = test->private_data;
5564                         if (check_var_refs(hist_data))
5565                                 return true;
5566                         break;
5567                 }
5568         }
5569
5570         return false;
5571 }
5572
5573 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5574                                     struct event_trigger_data *data,
5575                                     struct trace_event_file *file)
5576 {
5577         struct hist_trigger_data *hist_data = data->private_data;
5578         struct event_trigger_data *test, *named_data = NULL;
5579         bool unregistered = false;
5580
5581         lockdep_assert_held(&event_mutex);
5582
5583         if (hist_data->attrs->name)
5584                 named_data = find_named_trigger(hist_data->attrs->name);
5585
5586         list_for_each_entry(test, &file->triggers, list) {
5587                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5588                         if (!hist_trigger_match(data, test, named_data, false))
5589                                 continue;
5590                         unregistered = true;
5591                         list_del_rcu(&test->list);
5592                         trace_event_trigger_enable_disable(file, 0);
5593                         update_cond_flag(file);
5594                         break;
5595                 }
5596         }
5597
5598         if (unregistered && test->ops->free)
5599                 test->ops->free(test->ops, test);
5600
5601         if (hist_data->enable_timestamps) {
5602                 if (!hist_data->remove || unregistered)
5603                         tracing_set_filter_buffering(file->tr, false);
5604         }
5605 }
5606
5607 static bool hist_file_check_refs(struct trace_event_file *file)
5608 {
5609         struct hist_trigger_data *hist_data;
5610         struct event_trigger_data *test;
5611
5612         lockdep_assert_held(&event_mutex);
5613
5614         list_for_each_entry(test, &file->triggers, list) {
5615                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5616                         hist_data = test->private_data;
5617                         if (check_var_refs(hist_data))
5618                                 return true;
5619                 }
5620         }
5621
5622         return false;
5623 }
5624
5625 static void hist_unreg_all(struct trace_event_file *file)
5626 {
5627         struct event_trigger_data *test, *n;
5628         struct hist_trigger_data *hist_data;
5629         struct synth_event *se;
5630         const char *se_name;
5631
5632         lockdep_assert_held(&event_mutex);
5633
5634         if (hist_file_check_refs(file))
5635                 return;
5636
5637         list_for_each_entry_safe(test, n, &file->triggers, list) {
5638                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5639                         hist_data = test->private_data;
5640                         list_del_rcu(&test->list);
5641                         trace_event_trigger_enable_disable(file, 0);
5642
5643                         se_name = trace_event_name(file->event_call);
5644                         se = find_synth_event(se_name);
5645                         if (se)
5646                                 se->ref--;
5647
5648                         update_cond_flag(file);
5649                         if (hist_data->enable_timestamps)
5650                                 tracing_set_filter_buffering(file->tr, false);
5651                         if (test->ops->free)
5652                                 test->ops->free(test->ops, test);
5653                 }
5654         }
5655 }
5656
5657 static int event_hist_trigger_func(struct event_command *cmd_ops,
5658                                    struct trace_event_file *file,
5659                                    char *glob, char *cmd, char *param)
5660 {
5661         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5662         struct event_trigger_data *trigger_data;
5663         struct hist_trigger_attrs *attrs;
5664         struct event_trigger_ops *trigger_ops;
5665         struct hist_trigger_data *hist_data;
5666         struct synth_event *se;
5667         const char *se_name;
5668         bool remove = false;
5669         char *trigger, *p;
5670         int ret = 0;
5671
5672         lockdep_assert_held(&event_mutex);
5673
5674         if (glob && strlen(glob)) {
5675                 hist_err_clear();
5676                 last_cmd_set(file, param);
5677         }
5678
5679         if (!param)
5680                 return -EINVAL;
5681
5682         if (glob[0] == '!')
5683                 remove = true;
5684
5685         /*
5686          * separate the trigger from the filter (k:v [if filter])
5687          * allowing for whitespace in the trigger
5688          */
5689         p = trigger = param;
5690         do {
5691                 p = strstr(p, "if");
5692                 if (!p)
5693                         break;
5694                 if (p == param)
5695                         return -EINVAL;
5696                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5697                         p++;
5698                         continue;
5699                 }
5700                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5701                         return -EINVAL;
5702                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5703                         p++;
5704                         continue;
5705                 }
5706                 break;
5707         } while (p);
5708
5709         if (!p)
5710                 param = NULL;
5711         else {
5712                 *(p - 1) = '\0';
5713                 param = strstrip(p);
5714                 trigger = strstrip(trigger);
5715         }
5716
5717         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5718         if (IS_ERR(attrs))
5719                 return PTR_ERR(attrs);
5720
5721         if (attrs->map_bits)
5722                 hist_trigger_bits = attrs->map_bits;
5723
5724         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5725         if (IS_ERR(hist_data)) {
5726                 destroy_hist_trigger_attrs(attrs);
5727                 return PTR_ERR(hist_data);
5728         }
5729
5730         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5731
5732         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5733         if (!trigger_data) {
5734                 ret = -ENOMEM;
5735                 goto out_free;
5736         }
5737
5738         trigger_data->count = -1;
5739         trigger_data->ops = trigger_ops;
5740         trigger_data->cmd_ops = cmd_ops;
5741
5742         INIT_LIST_HEAD(&trigger_data->list);
5743         RCU_INIT_POINTER(trigger_data->filter, NULL);
5744
5745         trigger_data->private_data = hist_data;
5746
5747         /* if param is non-empty, it's supposed to be a filter */
5748         if (param && cmd_ops->set_filter) {
5749                 ret = cmd_ops->set_filter(param, trigger_data, file);
5750                 if (ret < 0)
5751                         goto out_free;
5752         }
5753
5754         if (remove) {
5755                 if (!have_hist_trigger_match(trigger_data, file))
5756                         goto out_free;
5757
5758                 if (hist_trigger_check_refs(trigger_data, file)) {
5759                         ret = -EBUSY;
5760                         goto out_free;
5761                 }
5762
5763                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5764                 se_name = trace_event_name(file->event_call);
5765                 se = find_synth_event(se_name);
5766                 if (se)
5767                         se->ref--;
5768                 ret = 0;
5769                 goto out_free;
5770         }
5771
5772         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5773         /*
5774          * The above returns on success the # of triggers registered,
5775          * but if it didn't register any it returns zero.  Consider no
5776          * triggers registered a failure too.
5777          */
5778         if (!ret) {
5779                 if (!(attrs->pause || attrs->cont || attrs->clear))
5780                         ret = -ENOENT;
5781                 goto out_free;
5782         } else if (ret < 0)
5783                 goto out_free;
5784
5785         if (get_named_trigger_data(trigger_data))
5786                 goto enable;
5787
5788         if (has_hist_vars(hist_data))
5789                 save_hist_vars(hist_data);
5790
5791         ret = create_actions(hist_data);
5792         if (ret)
5793                 goto out_unreg;
5794
5795         ret = tracing_map_init(hist_data->map);
5796         if (ret)
5797                 goto out_unreg;
5798 enable:
5799         ret = hist_trigger_enable(trigger_data, file);
5800         if (ret)
5801                 goto out_unreg;
5802
5803         se_name = trace_event_name(file->event_call);
5804         se = find_synth_event(se_name);
5805         if (se)
5806                 se->ref++;
5807         /* Just return zero, not the number of registered triggers */
5808         ret = 0;
5809  out:
5810         if (ret == 0)
5811                 hist_err_clear();
5812
5813         return ret;
5814  out_unreg:
5815         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5816  out_free:
5817         if (cmd_ops->set_filter)
5818                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5819
5820         remove_hist_vars(hist_data);
5821
5822         kfree(trigger_data);
5823
5824         destroy_hist_data(hist_data);
5825         goto out;
5826 }
5827
5828 static struct event_command trigger_hist_cmd = {
5829         .name                   = "hist",
5830         .trigger_type           = ETT_EVENT_HIST,
5831         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5832         .func                   = event_hist_trigger_func,
5833         .reg                    = hist_register_trigger,
5834         .unreg                  = hist_unregister_trigger,
5835         .unreg_all              = hist_unreg_all,
5836         .get_trigger_ops        = event_hist_get_trigger_ops,
5837         .set_filter             = set_trigger_filter,
5838 };
5839
5840 __init int register_trigger_hist_cmd(void)
5841 {
5842         int ret;
5843
5844         ret = register_event_command(&trigger_hist_cmd);
5845         WARN_ON(ret < 0);
5846
5847         return ret;
5848 }
5849
5850 static void
5851 hist_enable_trigger(struct event_trigger_data *data,
5852                     struct trace_buffer *buffer,  void *rec,
5853                     struct ring_buffer_event *event)
5854 {
5855         struct enable_trigger_data *enable_data = data->private_data;
5856         struct event_trigger_data *test;
5857
5858         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5859                                 lockdep_is_held(&event_mutex)) {
5860                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5861                         if (enable_data->enable)
5862                                 test->paused = false;
5863                         else
5864                                 test->paused = true;
5865                 }
5866         }
5867 }
5868
5869 static void
5870 hist_enable_count_trigger(struct event_trigger_data *data,
5871                           struct trace_buffer *buffer,  void *rec,
5872                           struct ring_buffer_event *event)
5873 {
5874         if (!data->count)
5875                 return;
5876
5877         if (data->count != -1)
5878                 (data->count)--;
5879
5880         hist_enable_trigger(data, buffer, rec, event);
5881 }
5882
5883 static struct event_trigger_ops hist_enable_trigger_ops = {
5884         .func                   = hist_enable_trigger,
5885         .print                  = event_enable_trigger_print,
5886         .init                   = event_trigger_init,
5887         .free                   = event_enable_trigger_free,
5888 };
5889
5890 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5891         .func                   = hist_enable_count_trigger,
5892         .print                  = event_enable_trigger_print,
5893         .init                   = event_trigger_init,
5894         .free                   = event_enable_trigger_free,
5895 };
5896
5897 static struct event_trigger_ops hist_disable_trigger_ops = {
5898         .func                   = hist_enable_trigger,
5899         .print                  = event_enable_trigger_print,
5900         .init                   = event_trigger_init,
5901         .free                   = event_enable_trigger_free,
5902 };
5903
5904 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5905         .func                   = hist_enable_count_trigger,
5906         .print                  = event_enable_trigger_print,
5907         .init                   = event_trigger_init,
5908         .free                   = event_enable_trigger_free,
5909 };
5910
5911 static struct event_trigger_ops *
5912 hist_enable_get_trigger_ops(char *cmd, char *param)
5913 {
5914         struct event_trigger_ops *ops;
5915         bool enable;
5916
5917         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5918
5919         if (enable)
5920                 ops = param ? &hist_enable_count_trigger_ops :
5921                         &hist_enable_trigger_ops;
5922         else
5923                 ops = param ? &hist_disable_count_trigger_ops :
5924                         &hist_disable_trigger_ops;
5925
5926         return ops;
5927 }
5928
5929 static void hist_enable_unreg_all(struct trace_event_file *file)
5930 {
5931         struct event_trigger_data *test, *n;
5932
5933         list_for_each_entry_safe(test, n, &file->triggers, list) {
5934                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5935                         list_del_rcu(&test->list);
5936                         update_cond_flag(file);
5937                         trace_event_trigger_enable_disable(file, 0);
5938                         if (test->ops->free)
5939                                 test->ops->free(test->ops, test);
5940                 }
5941         }
5942 }
5943
5944 static struct event_command trigger_hist_enable_cmd = {
5945         .name                   = ENABLE_HIST_STR,
5946         .trigger_type           = ETT_HIST_ENABLE,
5947         .func                   = event_enable_trigger_func,
5948         .reg                    = event_enable_register_trigger,
5949         .unreg                  = event_enable_unregister_trigger,
5950         .unreg_all              = hist_enable_unreg_all,
5951         .get_trigger_ops        = hist_enable_get_trigger_ops,
5952         .set_filter             = set_trigger_filter,
5953 };
5954
5955 static struct event_command trigger_hist_disable_cmd = {
5956         .name                   = DISABLE_HIST_STR,
5957         .trigger_type           = ETT_HIST_ENABLE,
5958         .func                   = event_enable_trigger_func,
5959         .reg                    = event_enable_register_trigger,
5960         .unreg                  = event_enable_unregister_trigger,
5961         .unreg_all              = hist_enable_unreg_all,
5962         .get_trigger_ops        = hist_enable_get_trigger_ops,
5963         .set_filter             = set_trigger_filter,
5964 };
5965
5966 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5967 {
5968         unregister_event_command(&trigger_hist_enable_cmd);
5969         unregister_event_command(&trigger_hist_disable_cmd);
5970 }
5971
5972 __init int register_trigger_hist_enable_disable_cmds(void)
5973 {
5974         int ret;
5975
5976         ret = register_event_command(&trigger_hist_enable_cmd);
5977         if (WARN_ON(ret < 0))
5978                 return ret;
5979         ret = register_event_command(&trigger_hist_disable_cmd);
5980         if (WARN_ON(ret < 0))
5981                 unregister_trigger_hist_enable_disable_cmds();
5982
5983         return ret;
5984 }