kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  69
  70 #undef C
  71 #define C(a, b)         HIST_ERR_##a
  72
  73 enum { ERRORS };
  74
  75 #undef C
  76 #define C(a, b)         b
  77
  78 static const char *err_text[] = { ERRORS };
  79
  80 struct hist_field;
  81
  82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  83                                 struct tracing_map_elt *elt,
  84                                 struct trace_buffer *buffer,
  85                                 struct ring_buffer_event *rbe,
  86                                 void *event);
  87
  88 #define HIST_FIELD_OPERANDS_MAX 2
  89 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  90 #define HIST_ACTIONS_MAX        8
  91
  92 enum field_op_id {
  93         FIELD_OP_NONE,
  94         FIELD_OP_PLUS,
  95         FIELD_OP_MINUS,
  96         FIELD_OP_UNARY_MINUS,
  97 };
  98
  99 /*
 100  * A hist_var (histogram variable) contains variable information for
 101  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 102  * flag set.  A hist_var has a variable name e.g. ts0, and is
 103  * associated with a given histogram trigger, as specified by
 104  * hist_data.  The hist_var idx is the unique index assigned to the
 105  * variable by the hist trigger's tracing_map.  The idx is what is
 106  * used to set a variable's value and, by a variable reference, to
 107  * retrieve it.
 108  */
 109 struct hist_var {
 110         char                            *name;
 111         struct hist_trigger_data        *hist_data;
 112         unsigned int                    idx;
 113 };
 114
 115 struct hist_field {
 116         struct ftrace_event_field       *field;
 117         unsigned long                   flags;
 118         hist_field_fn_t                 fn;
 119         unsigned int                    ref;
 120         unsigned int                    size;
 121         unsigned int                    offset;
 122         unsigned int                    is_signed;
 123         const char                      *type;
 124         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 125         struct hist_trigger_data        *hist_data;
 126
 127         /*
 128          * Variable fields contain variable-specific info in var.
 129          */
 130         struct hist_var                 var;
 131         enum field_op_id                operator;
 132         char                            *system;
 133         char                            *event_name;
 134
 135         /*
 136          * The name field is used for EXPR and VAR_REF fields.  VAR
 137          * fields contain the variable name in var.name.
 138          */
 139         char                            *name;
 140
 141         /*
 142          * When a histogram trigger is hit, if it has any references
 143          * to variables, the values of those variables are collected
 144          * into a var_ref_vals array by resolve_var_refs().  The
 145          * current value of each variable is read from the tracing_map
 146          * using the hist field's hist_var.idx and entered into the
 147          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 148          */
 149         unsigned int                    var_ref_idx;
 150         bool                            read_once;
 151
 152         unsigned int                    var_str_idx;
 153 };
 154
 155 static u64 hist_field_none(struct hist_field *field,
 156                            struct tracing_map_elt *elt,
 157                            struct trace_buffer *buffer,
 158                            struct ring_buffer_event *rbe,
 159                            void *event)
 160 {
 161         return 0;
 162 }
 163
 164 static u64 hist_field_counter(struct hist_field *field,
 165                               struct tracing_map_elt *elt,
 166                               struct trace_buffer *buffer,
 167                               struct ring_buffer_event *rbe,
 168                               void *event)
 169 {
 170         return 1;
 171 }
 172
 173 static u64 hist_field_string(struct hist_field *hist_field,
 174                              struct tracing_map_elt *elt,
 175                              struct trace_buffer *buffer,
 176                              struct ring_buffer_event *rbe,
 177                              void *event)
 178 {
 179         char *addr = (char *)(event + hist_field->field->offset);
 180
 181         return (u64)(unsigned long)addr;
 182 }
 183
 184 static u64 hist_field_dynstring(struct hist_field *hist_field,
 185                                 struct tracing_map_elt *elt,
 186                                 struct trace_buffer *buffer,
 187                                 struct ring_buffer_event *rbe,
 188                                 void *event)
 189 {
 190         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 191         int str_loc = str_item & 0xffff;
 192         char *addr = (char *)(event + str_loc);
 193
 194         return (u64)(unsigned long)addr;
 195 }
 196
 197 static u64 hist_field_pstring(struct hist_field *hist_field,
 198                               struct tracing_map_elt *elt,
 199                               struct trace_buffer *buffer,
 200                               struct ring_buffer_event *rbe,
 201                               void *event)
 202 {
 203         char **addr = (char **)(event + hist_field->field->offset);
 204
 205         return (u64)(unsigned long)*addr;
 206 }
 207
 208 static u64 hist_field_log2(struct hist_field *hist_field,
 209                            struct tracing_map_elt *elt,
 210                            struct trace_buffer *buffer,
 211                            struct ring_buffer_event *rbe,
 212                            void *event)
 213 {
 214         struct hist_field *operand = hist_field->operands[0];
 215
 216         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 217
 218         return (u64) ilog2(roundup_pow_of_two(val));
 219 }
 220
 221 static u64 hist_field_plus(struct hist_field *hist_field,
 222                            struct tracing_map_elt *elt,
 223                            struct trace_buffer *buffer,
 224                            struct ring_buffer_event *rbe,
 225                            void *event)
 226 {
 227         struct hist_field *operand1 = hist_field->operands[0];
 228         struct hist_field *operand2 = hist_field->operands[1];
 229
 230         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 231         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 232
 233         return val1 + val2;
 234 }
 235
 236 static u64 hist_field_minus(struct hist_field *hist_field,
 237                             struct tracing_map_elt *elt,
 238                             struct trace_buffer *buffer,
 239                             struct ring_buffer_event *rbe,
 240                             void *event)
 241 {
 242         struct hist_field *operand1 = hist_field->operands[0];
 243         struct hist_field *operand2 = hist_field->operands[1];
 244
 245         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 246         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 247
 248         return val1 - val2;
 249 }
 250
 251 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 252                                   struct tracing_map_elt *elt,
 253                                   struct trace_buffer *buffer,
 254                                   struct ring_buffer_event *rbe,
 255                                   void *event)
 256 {
 257         struct hist_field *operand = hist_field->operands[0];
 258
 259         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 260         u64 val = (u64)-sval;
 261
 262         return val;
 263 }
 264
 265 #define DEFINE_HIST_FIELD_FN(type)                                      \
 266         static u64 hist_field_##type(struct hist_field *hist_field,     \
 267                                      struct tracing_map_elt *elt,       \
 268                                      struct trace_buffer *buffer,       \
 269                                      struct ring_buffer_event *rbe,     \
 270                                      void *event)                       \
 271 {                                                                       \
 272         type *addr = (type *)(event + hist_field->field->offset);       \
 273                                                                         \
 274         return (u64)(unsigned long)*addr;                               \
 275 }
 276
 277 DEFINE_HIST_FIELD_FN(s64);
 278 DEFINE_HIST_FIELD_FN(u64);
 279 DEFINE_HIST_FIELD_FN(s32);
 280 DEFINE_HIST_FIELD_FN(u32);
 281 DEFINE_HIST_FIELD_FN(s16);
 282 DEFINE_HIST_FIELD_FN(u16);
 283 DEFINE_HIST_FIELD_FN(s8);
 284 DEFINE_HIST_FIELD_FN(u8);
 285
 286 #define for_each_hist_field(i, hist_data)       \
 287         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 288
 289 #define for_each_hist_val_field(i, hist_data)   \
 290         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 291
 292 #define for_each_hist_key_field(i, hist_data)   \
 293         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 294
 295 #define HIST_STACKTRACE_DEPTH   16
 296 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 297 #define HIST_STACKTRACE_SKIP    5
 298
 299 #define HITCOUNT_IDX            0
 300 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 301
 302 enum hist_field_flags {
 303         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 304         HIST_FIELD_FL_KEY               = 1 << 1,
 305         HIST_FIELD_FL_STRING            = 1 << 2,
 306         HIST_FIELD_FL_HEX               = 1 << 3,
 307         HIST_FIELD_FL_SYM               = 1 << 4,
 308         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 309         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 310         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 311         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 312         HIST_FIELD_FL_LOG2              = 1 << 9,
 313         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 314         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 315         HIST_FIELD_FL_VAR               = 1 << 12,
 316         HIST_FIELD_FL_EXPR              = 1 << 13,
 317         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 318         HIST_FIELD_FL_CPU               = 1 << 15,
 319         HIST_FIELD_FL_ALIAS             = 1 << 16,
 320 };
 321
 322 struct var_defs {
 323         unsigned int    n_vars;
 324         char            *name[TRACING_MAP_VARS_MAX];
 325         char            *expr[TRACING_MAP_VARS_MAX];
 326 };
 327
 328 struct hist_trigger_attrs {
 329         char            *keys_str;
 330         char            *vals_str;
 331         char            *sort_key_str;
 332         char            *name;
 333         char            *clock;
 334         bool            pause;
 335         bool            cont;
 336         bool            clear;
 337         bool            ts_in_usecs;
 338         unsigned int    map_bits;
 339
 340         char            *assignment_str[TRACING_MAP_VARS_MAX];
 341         unsigned int    n_assignments;
 342
 343         char            *action_str[HIST_ACTIONS_MAX];
 344         unsigned int    n_actions;
 345
 346         struct var_defs var_defs;
 347 };
 348
 349 struct field_var {
 350         struct hist_field       *var;
 351         struct hist_field       *val;
 352 };
 353
 354 struct field_var_hist {
 355         struct hist_trigger_data        *hist_data;
 356         char                            *cmd;
 357 };
 358
 359 struct hist_trigger_data {
 360         struct hist_field               *fields[HIST_FIELDS_MAX];
 361         unsigned int                    n_vals;
 362         unsigned int                    n_keys;
 363         unsigned int                    n_fields;
 364         unsigned int                    n_vars;
 365         unsigned int                    n_var_str;
 366         unsigned int                    key_size;
 367         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 368         unsigned int                    n_sort_keys;
 369         struct trace_event_file         *event_file;
 370         struct hist_trigger_attrs       *attrs;
 371         struct tracing_map              *map;
 372         bool                            enable_timestamps;
 373         bool                            remove;
 374         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 375         unsigned int                    n_var_refs;
 376
 377         struct action_data              *actions[HIST_ACTIONS_MAX];
 378         unsigned int                    n_actions;
 379
 380         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 381         unsigned int                    n_field_vars;
 382         unsigned int                    n_field_var_str;
 383         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 384         unsigned int                    n_field_var_hists;
 385
 386         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 387         unsigned int                    n_save_vars;
 388         unsigned int                    n_save_var_str;
 389 };
 390
 391 struct action_data;
 392
 393 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 394                              struct tracing_map_elt *elt,
 395                              struct trace_buffer *buffer, void *rec,
 396                              struct ring_buffer_event *rbe, void *key,
 397                              struct action_data *data, u64 *var_ref_vals);
 398
 399 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 400
 401 enum handler_id {
 402         HANDLER_ONMATCH = 1,
 403         HANDLER_ONMAX,
 404         HANDLER_ONCHANGE,
 405 };
 406
 407 enum action_id {
 408         ACTION_SAVE = 1,
 409         ACTION_TRACE,
 410         ACTION_SNAPSHOT,
 411 };
 412
 413 struct action_data {
 414         enum handler_id         handler;
 415         enum action_id          action;
 416         char                    *action_name;
 417         action_fn_t             fn;
 418
 419         unsigned int            n_params;
 420         char                    *params[SYNTH_FIELDS_MAX];
 421
 422         /*
 423          * When a histogram trigger is hit, the values of any
 424          * references to variables, including variables being passed
 425          * as parameters to synthetic events, are collected into a
 426          * var_ref_vals array.  This var_ref_idx array is an array of
 427          * indices into the var_ref_vals array, one for each synthetic
 428          * event param, and is passed to the synthetic event
 429          * invocation.
 430          */
 431         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 432         struct synth_event      *synth_event;
 433         bool                    use_trace_keyword;
 434         char                    *synth_event_name;
 435
 436         union {
 437                 struct {
 438                         char                    *event;
 439                         char                    *event_system;
 440                 } match_data;
 441
 442                 struct {
 443                         /*
 444                          * var_str contains the $-unstripped variable
 445                          * name referenced by var_ref, and used when
 446                          * printing the action.  Because var_ref
 447                          * creation is deferred to create_actions(),
 448                          * we need a per-action way to save it until
 449                          * then, thus var_str.
 450                          */
 451                         char                    *var_str;
 452
 453                         /*
 454                          * var_ref refers to the variable being
 455                          * tracked e.g onmax($var).
 456                          */
 457                         struct hist_field       *var_ref;
 458
 459                         /*
 460                          * track_var contains the 'invisible' tracking
 461                          * variable created to keep the current
 462                          * e.g. max value.
 463                          */
 464                         struct hist_field       *track_var;
 465
 466                         check_track_val_fn_t    check_val;
 467                         action_fn_t             save_data;
 468                 } track_data;
 469         };
 470 };
 471
 472 struct track_data {
 473         u64                             track_val;
 474         bool                            updated;
 475
 476         unsigned int                    key_len;
 477         void                            *key;
 478         struct tracing_map_elt          elt;
 479
 480         struct action_data              *action_data;
 481         struct hist_trigger_data        *hist_data;
 482 };
 483
 484 struct hist_elt_data {
 485         char *comm;
 486         u64 *var_ref_vals;
 487         char *field_var_str[SYNTH_FIELDS_MAX];
 488 };
 489
 490 struct snapshot_context {
 491         struct tracing_map_elt  *elt;
 492         void                    *key;
 493 };
 494
 495 static void track_data_free(struct track_data *track_data)
 496 {
 497         struct hist_elt_data *elt_data;
 498
 499         if (!track_data)
 500                 return;
 501
 502         kfree(track_data->key);
 503
 504         elt_data = track_data->elt.private_data;
 505         if (elt_data) {
 506                 kfree(elt_data->comm);
 507                 kfree(elt_data);
 508         }
 509
 510         kfree(track_data);
 511 }
 512
 513 static struct track_data *track_data_alloc(unsigned int key_len,
 514                                            struct action_data *action_data,
 515                                            struct hist_trigger_data *hist_data)
 516 {
 517         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 518         struct hist_elt_data *elt_data;
 519
 520         if (!data)
 521                 return ERR_PTR(-ENOMEM);
 522
 523         data->key = kzalloc(key_len, GFP_KERNEL);
 524         if (!data->key) {
 525                 track_data_free(data);
 526                 return ERR_PTR(-ENOMEM);
 527         }
 528
 529         data->key_len = key_len;
 530         data->action_data = action_data;
 531         data->hist_data = hist_data;
 532
 533         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 534         if (!elt_data) {
 535                 track_data_free(data);
 536                 return ERR_PTR(-ENOMEM);
 537         }
 538
 539         data->elt.private_data = elt_data;
 540
 541         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 542         if (!elt_data->comm) {
 543                 track_data_free(data);
 544                 return ERR_PTR(-ENOMEM);
 545         }
 546
 547         return data;
 548 }
 549
 550 static char last_cmd[MAX_FILTER_STR_VAL];
 551 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 552
 553 static int errpos(char *str)
 554 {
 555         return err_pos(last_cmd, str);
 556 }
 557
 558 static void last_cmd_set(struct trace_event_file *file, char *str)
 559 {
 560         const char *system = NULL, *name = NULL;
 561         struct trace_event_call *call;
 562
 563         if (!str)
 564                 return;
 565
 566         strcpy(last_cmd, "hist:");
 567         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 568
 569         if (file) {
 570                 call = file->event_call;
 571                 system = call->class->system;
 572                 if (system) {
 573                         name = trace_event_name(call);
 574                         if (!name)
 575                                 system = NULL;
 576                 }
 577         }
 578
 579         if (system)
 580                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 581 }
 582
 583 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 584 {
 585         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 586                         err_type, err_pos);
 587 }
 588
 589 static void hist_err_clear(void)
 590 {
 591         last_cmd[0] = '\0';
 592         last_cmd_loc[0] = '\0';
 593 }
 594
 595 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 596                                     unsigned int *var_ref_idx);
 597
 598 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 599                                unsigned int *var_ref_idx)
 600 {
 601         struct tracepoint *tp = event->tp;
 602
 603         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 604                 struct tracepoint_func *probe_func_ptr;
 605                 synth_probe_func_t probe_func;
 606                 void *__data;
 607
 608                 if (!(cpu_online(raw_smp_processor_id())))
 609                         return;
 610
 611                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 612                 if (probe_func_ptr) {
 613                         do {
 614                                 probe_func = probe_func_ptr->func;
 615                                 __data = probe_func_ptr->data;
 616                                 probe_func(__data, var_ref_vals, var_ref_idx);
 617                         } while ((++probe_func_ptr)->func);
 618                 }
 619         }
 620 }
 621
 622 static void action_trace(struct hist_trigger_data *hist_data,
 623                          struct tracing_map_elt *elt,
 624                          struct trace_buffer *buffer, void *rec,
 625                          struct ring_buffer_event *rbe, void *key,
 626                          struct action_data *data, u64 *var_ref_vals)
 627 {
 628         struct synth_event *event = data->synth_event;
 629
 630         trace_synth(event, var_ref_vals, data->var_ref_idx);
 631 }
 632
 633 struct hist_var_data {
 634         struct list_head list;
 635         struct hist_trigger_data *hist_data;
 636 };
 637
 638 static u64 hist_field_timestamp(struct hist_field *hist_field,
 639                                 struct tracing_map_elt *elt,
 640                                 struct trace_buffer *buffer,
 641                                 struct ring_buffer_event *rbe,
 642                                 void *event)
 643 {
 644         struct hist_trigger_data *hist_data = hist_field->hist_data;
 645         struct trace_array *tr = hist_data->event_file->tr;
 646
 647         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 648
 649         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 650                 ts = ns2usecs(ts);
 651
 652         return ts;
 653 }
 654
 655 static u64 hist_field_cpu(struct hist_field *hist_field,
 656                           struct tracing_map_elt *elt,
 657                           struct trace_buffer *buffer,
 658                           struct ring_buffer_event *rbe,
 659                           void *event)
 660 {
 661         int cpu = smp_processor_id();
 662
 663         return cpu;
 664 }
 665
 666 /**
 667  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 668  * @hist_field: The VAR_REF field to check
 669  * @var_data: The hist trigger that owns the variable
 670  * @var_idx: The trigger variable identifier
 671  *
 672  * Check the given VAR_REF field to see whether or not it references
 673  * the given variable associated with the given trigger.
 674  *
 675  * Return: The VAR_REF field if it does reference the variable, NULL if not
 676  */
 677 static struct hist_field *
 678 check_field_for_var_ref(struct hist_field *hist_field,
 679                         struct hist_trigger_data *var_data,
 680                         unsigned int var_idx)
 681 {
 682         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 683
 684         if (hist_field && hist_field->var.idx == var_idx &&
 685             hist_field->var.hist_data == var_data)
 686                 return hist_field;
 687
 688         return NULL;
 689 }
 690
 691 /**
 692  * find_var_ref - Check if a trigger has a reference to a trigger variable
 693  * @hist_data: The hist trigger that might have a reference to the variable
 694  * @var_data: The hist trigger that owns the variable
 695  * @var_idx: The trigger variable identifier
 696  *
 697  * Check the list of var_refs[] on the first hist trigger to see
 698  * whether any of them are references to the variable on the second
 699  * trigger.
 700  *
 701  * Return: The VAR_REF field referencing the variable if so, NULL if not
 702  */
 703 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 704                                        struct hist_trigger_data *var_data,
 705                                        unsigned int var_idx)
 706 {
 707         struct hist_field *hist_field;
 708         unsigned int i;
 709
 710         for (i = 0; i < hist_data->n_var_refs; i++) {
 711                 hist_field = hist_data->var_refs[i];
 712                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 713                         return hist_field;
 714         }
 715
 716         return NULL;
 717 }
 718
 719 /**
 720  * find_any_var_ref - Check if there is a reference to a given trigger variable
 721  * @hist_data: The hist trigger
 722  * @var_idx: The trigger variable identifier
 723  *
 724  * Check to see whether the given variable is currently referenced by
 725  * any other trigger.
 726  *
 727  * The trigger the variable is defined on is explicitly excluded - the
 728  * assumption being that a self-reference doesn't prevent a trigger
 729  * from being removed.
 730  *
 731  * Return: The VAR_REF field referencing the variable if so, NULL if not
 732  */
 733 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 734                                            unsigned int var_idx)
 735 {
 736         struct trace_array *tr = hist_data->event_file->tr;
 737         struct hist_field *found = NULL;
 738         struct hist_var_data *var_data;
 739
 740         list_for_each_entry(var_data, &tr->hist_vars, list) {
 741                 if (var_data->hist_data == hist_data)
 742                         continue;
 743                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 744                 if (found)
 745                         break;
 746         }
 747
 748         return found;
 749 }
 750
 751 /**
 752  * check_var_refs - Check if there is a reference to any of trigger's variables
 753  * @hist_data: The hist trigger
 754  *
 755  * A trigger can define one or more variables.  If any one of them is
 756  * currently referenced by any other trigger, this function will
 757  * determine that.
 758
 759  * Typically used to determine whether or not a trigger can be removed
 760  * - if there are any references to a trigger's variables, it cannot.
 761  *
 762  * Return: True if there is a reference to any of trigger's variables
 763  */
 764 static bool check_var_refs(struct hist_trigger_data *hist_data)
 765 {
 766         struct hist_field *field;
 767         bool found = false;
 768         int i;
 769
 770         for_each_hist_field(i, hist_data) {
 771                 field = hist_data->fields[i];
 772                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 773                         if (find_any_var_ref(hist_data, field->var.idx)) {
 774                                 found = true;
 775                                 break;
 776                         }
 777                 }
 778         }
 779
 780         return found;
 781 }
 782
 783 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 784 {
 785         struct trace_array *tr = hist_data->event_file->tr;
 786         struct hist_var_data *var_data, *found = NULL;
 787
 788         list_for_each_entry(var_data, &tr->hist_vars, list) {
 789                 if (var_data->hist_data == hist_data) {
 790                         found = var_data;
 791                         break;
 792                 }
 793         }
 794
 795         return found;
 796 }
 797
 798 static bool field_has_hist_vars(struct hist_field *hist_field,
 799                                 unsigned int level)
 800 {
 801         int i;
 802
 803         if (level > 3)
 804                 return false;
 805
 806         if (!hist_field)
 807                 return false;
 808
 809         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 810             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 811                 return true;
 812
 813         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 814                 struct hist_field *operand;
 815
 816                 operand = hist_field->operands[i];
 817                 if (field_has_hist_vars(operand, level + 1))
 818                         return true;
 819         }
 820
 821         return false;
 822 }
 823
 824 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 825 {
 826         struct hist_field *hist_field;
 827         int i;
 828
 829         for_each_hist_field(i, hist_data) {
 830                 hist_field = hist_data->fields[i];
 831                 if (field_has_hist_vars(hist_field, 0))
 832                         return true;
 833         }
 834
 835         return false;
 836 }
 837
 838 static int save_hist_vars(struct hist_trigger_data *hist_data)
 839 {
 840         struct trace_array *tr = hist_data->event_file->tr;
 841         struct hist_var_data *var_data;
 842
 843         var_data = find_hist_vars(hist_data);
 844         if (var_data)
 845                 return 0;
 846
 847         if (tracing_check_open_get_tr(tr))
 848                 return -ENODEV;
 849
 850         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 851         if (!var_data) {
 852                 trace_array_put(tr);
 853                 return -ENOMEM;
 854         }
 855
 856         var_data->hist_data = hist_data;
 857         list_add(&var_data->list, &tr->hist_vars);
 858
 859         return 0;
 860 }
 861
 862 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 863 {
 864         struct trace_array *tr = hist_data->event_file->tr;
 865         struct hist_var_data *var_data;
 866
 867         var_data = find_hist_vars(hist_data);
 868         if (!var_data)
 869                 return;
 870
 871         if (WARN_ON(check_var_refs(hist_data)))
 872                 return;
 873
 874         list_del(&var_data->list);
 875
 876         kfree(var_data);
 877
 878         trace_array_put(tr);
 879 }
 880
 881 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 882                                          const char *var_name)
 883 {
 884         struct hist_field *hist_field, *found = NULL;
 885         int i;
 886
 887         for_each_hist_field(i, hist_data) {
 888                 hist_field = hist_data->fields[i];
 889                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 890                     strcmp(hist_field->var.name, var_name) == 0) {
 891                         found = hist_field;
 892                         break;
 893                 }
 894         }
 895
 896         return found;
 897 }
 898
 899 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 900                                    struct trace_event_file *file,
 901                                    const char *var_name)
 902 {
 903         struct hist_trigger_data *test_data;
 904         struct event_trigger_data *test;
 905         struct hist_field *hist_field;
 906
 907         lockdep_assert_held(&event_mutex);
 908
 909         hist_field = find_var_field(hist_data, var_name);
 910         if (hist_field)
 911                 return hist_field;
 912
 913         list_for_each_entry(test, &file->triggers, list) {
 914                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 915                         test_data = test->private_data;
 916                         hist_field = find_var_field(test_data, var_name);
 917                         if (hist_field)
 918                                 return hist_field;
 919                 }
 920         }
 921
 922         return NULL;
 923 }
 924
 925 static struct trace_event_file *find_var_file(struct trace_array *tr,
 926                                               char *system,
 927                                               char *event_name,
 928                                               char *var_name)
 929 {
 930         struct hist_trigger_data *var_hist_data;
 931         struct hist_var_data *var_data;
 932         struct trace_event_file *file, *found = NULL;
 933
 934         if (system)
 935                 return find_event_file(tr, system, event_name);
 936
 937         list_for_each_entry(var_data, &tr->hist_vars, list) {
 938                 var_hist_data = var_data->hist_data;
 939                 file = var_hist_data->event_file;
 940                 if (file == found)
 941                         continue;
 942
 943                 if (find_var_field(var_hist_data, var_name)) {
 944                         if (found) {
 945                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 946                                 return NULL;
 947                         }
 948
 949                         found = file;
 950                 }
 951         }
 952
 953         return found;
 954 }
 955
 956 static struct hist_field *find_file_var(struct trace_event_file *file,
 957                                         const char *var_name)
 958 {
 959         struct hist_trigger_data *test_data;
 960         struct event_trigger_data *test;
 961         struct hist_field *hist_field;
 962
 963         lockdep_assert_held(&event_mutex);
 964
 965         list_for_each_entry(test, &file->triggers, list) {
 966                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 967                         test_data = test->private_data;
 968                         hist_field = find_var_field(test_data, var_name);
 969                         if (hist_field)
 970                                 return hist_field;
 971                 }
 972         }
 973
 974         return NULL;
 975 }
 976
 977 static struct hist_field *
 978 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 979 {
 980         struct trace_array *tr = hist_data->event_file->tr;
 981         struct hist_field *hist_field, *found = NULL;
 982         struct trace_event_file *file;
 983         unsigned int i;
 984
 985         for (i = 0; i < hist_data->n_actions; i++) {
 986                 struct action_data *data = hist_data->actions[i];
 987
 988                 if (data->handler == HANDLER_ONMATCH) {
 989                         char *system = data->match_data.event_system;
 990                         char *event_name = data->match_data.event;
 991
 992                         file = find_var_file(tr, system, event_name, var_name);
 993                         if (!file)
 994                                 continue;
 995                         hist_field = find_file_var(file, var_name);
 996                         if (hist_field) {
 997                                 if (found) {
 998                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
 999                                                  errpos(var_name));
1000                                         return ERR_PTR(-EINVAL);
1001                                 }
1002
1003                                 found = hist_field;
1004                         }
1005                 }
1006         }
1007         return found;
1008 }
1009
1010 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1011                                          char *system,
1012                                          char *event_name,
1013                                          char *var_name)
1014 {
1015         struct trace_array *tr = hist_data->event_file->tr;
1016         struct hist_field *hist_field = NULL;
1017         struct trace_event_file *file;
1018
1019         if (!system || !event_name) {
1020                 hist_field = find_match_var(hist_data, var_name);
1021                 if (IS_ERR(hist_field))
1022                         return NULL;
1023                 if (hist_field)
1024                         return hist_field;
1025         }
1026
1027         file = find_var_file(tr, system, event_name, var_name);
1028         if (!file)
1029                 return NULL;
1030
1031         hist_field = find_file_var(file, var_name);
1032
1033         return hist_field;
1034 }
1035
1036 static u64 hist_field_var_ref(struct hist_field *hist_field,
1037                               struct tracing_map_elt *elt,
1038                               struct trace_buffer *buffer,
1039                               struct ring_buffer_event *rbe,
1040                               void *event)
1041 {
1042         struct hist_elt_data *elt_data;
1043         u64 var_val = 0;
1044
1045         if (WARN_ON_ONCE(!elt))
1046                 return var_val;
1047
1048         elt_data = elt->private_data;
1049         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1050
1051         return var_val;
1052 }
1053
1054 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1055                              u64 *var_ref_vals, bool self)
1056 {
1057         struct hist_trigger_data *var_data;
1058         struct tracing_map_elt *var_elt;
1059         struct hist_field *hist_field;
1060         unsigned int i, var_idx;
1061         bool resolved = true;
1062         u64 var_val = 0;
1063
1064         for (i = 0; i < hist_data->n_var_refs; i++) {
1065                 hist_field = hist_data->var_refs[i];
1066                 var_idx = hist_field->var.idx;
1067                 var_data = hist_field->var.hist_data;
1068
1069                 if (var_data == NULL) {
1070                         resolved = false;
1071                         break;
1072                 }
1073
1074                 if ((self && var_data != hist_data) ||
1075                     (!self && var_data == hist_data))
1076                         continue;
1077
1078                 var_elt = tracing_map_lookup(var_data->map, key);
1079                 if (!var_elt) {
1080                         resolved = false;
1081                         break;
1082                 }
1083
1084                 if (!tracing_map_var_set(var_elt, var_idx)) {
1085                         resolved = false;
1086                         break;
1087                 }
1088
1089                 if (self || !hist_field->read_once)
1090                         var_val = tracing_map_read_var(var_elt, var_idx);
1091                 else
1092                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1093
1094                 var_ref_vals[i] = var_val;
1095         }
1096
1097         return resolved;
1098 }
1099
1100 static const char *hist_field_name(struct hist_field *field,
1101                                    unsigned int level)
1102 {
1103         const char *field_name = "";
1104
1105         if (level > 1)
1106                 return field_name;
1107
1108         if (field->field)
1109                 field_name = field->field->name;
1110         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1111                  field->flags & HIST_FIELD_FL_ALIAS)
1112                 field_name = hist_field_name(field->operands[0], ++level);
1113         else if (field->flags & HIST_FIELD_FL_CPU)
1114                 field_name = "cpu";
1115         else if (field->flags & HIST_FIELD_FL_EXPR ||
1116                  field->flags & HIST_FIELD_FL_VAR_REF) {
1117                 if (field->system) {
1118                         static char full_name[MAX_FILTER_STR_VAL];
1119
1120                         strcat(full_name, field->system);
1121                         strcat(full_name, ".");
1122                         strcat(full_name, field->event_name);
1123                         strcat(full_name, ".");
1124                         strcat(full_name, field->name);
1125                         field_name = full_name;
1126                 } else
1127                         field_name = field->name;
1128         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1129                 field_name = "common_timestamp";
1130
1131         if (field_name == NULL)
1132                 field_name = "";
1133
1134         return field_name;
1135 }
1136
1137 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1138 {
1139         hist_field_fn_t fn = NULL;
1140
1141         switch (field_size) {
1142         case 8:
1143                 if (field_is_signed)
1144                         fn = hist_field_s64;
1145                 else
1146                         fn = hist_field_u64;
1147                 break;
1148         case 4:
1149                 if (field_is_signed)
1150                         fn = hist_field_s32;
1151                 else
1152                         fn = hist_field_u32;
1153                 break;
1154         case 2:
1155                 if (field_is_signed)
1156                         fn = hist_field_s16;
1157                 else
1158                         fn = hist_field_u16;
1159                 break;
1160         case 1:
1161                 if (field_is_signed)
1162                         fn = hist_field_s8;
1163                 else
1164                         fn = hist_field_u8;
1165                 break;
1166         }
1167
1168         return fn;
1169 }
1170
1171 static int parse_map_size(char *str)
1172 {
1173         unsigned long size, map_bits;
1174         int ret;
1175
1176         ret = kstrtoul(str, 0, &size);
1177         if (ret)
1178                 goto out;
1179
1180         map_bits = ilog2(roundup_pow_of_two(size));
1181         if (map_bits < TRACING_MAP_BITS_MIN ||
1182             map_bits > TRACING_MAP_BITS_MAX)
1183                 ret = -EINVAL;
1184         else
1185                 ret = map_bits;
1186  out:
1187         return ret;
1188 }
1189
1190 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1191 {
1192         unsigned int i;
1193
1194         if (!attrs)
1195                 return;
1196
1197         for (i = 0; i < attrs->n_assignments; i++)
1198                 kfree(attrs->assignment_str[i]);
1199
1200         for (i = 0; i < attrs->n_actions; i++)
1201                 kfree(attrs->action_str[i]);
1202
1203         kfree(attrs->name);
1204         kfree(attrs->sort_key_str);
1205         kfree(attrs->keys_str);
1206         kfree(attrs->vals_str);
1207         kfree(attrs->clock);
1208         kfree(attrs);
1209 }
1210
1211 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1212 {
1213         int ret = -EINVAL;
1214
1215         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1216                 return ret;
1217
1218         if ((str_has_prefix(str, "onmatch(")) ||
1219             (str_has_prefix(str, "onmax(")) ||
1220             (str_has_prefix(str, "onchange("))) {
1221                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1222                 if (!attrs->action_str[attrs->n_actions]) {
1223                         ret = -ENOMEM;
1224                         return ret;
1225                 }
1226                 attrs->n_actions++;
1227                 ret = 0;
1228         }
1229         return ret;
1230 }
1231
1232 static int parse_assignment(struct trace_array *tr,
1233                             char *str, struct hist_trigger_attrs *attrs)
1234 {
1235         int len, ret = 0;
1236
1237         if ((len = str_has_prefix(str, "key=")) ||
1238             (len = str_has_prefix(str, "keys="))) {
1239                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1240                 if (!attrs->keys_str) {
1241                         ret = -ENOMEM;
1242                         goto out;
1243                 }
1244         } else if ((len = str_has_prefix(str, "val=")) ||
1245                    (len = str_has_prefix(str, "vals=")) ||
1246                    (len = str_has_prefix(str, "values="))) {
1247                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1248                 if (!attrs->vals_str) {
1249                         ret = -ENOMEM;
1250                         goto out;
1251                 }
1252         } else if ((len = str_has_prefix(str, "sort="))) {
1253                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1254                 if (!attrs->sort_key_str) {
1255                         ret = -ENOMEM;
1256                         goto out;
1257                 }
1258         } else if (str_has_prefix(str, "name=")) {
1259                 attrs->name = kstrdup(str, GFP_KERNEL);
1260                 if (!attrs->name) {
1261                         ret = -ENOMEM;
1262                         goto out;
1263                 }
1264         } else if ((len = str_has_prefix(str, "clock="))) {
1265                 str += len;
1266
1267                 str = strstrip(str);
1268                 attrs->clock = kstrdup(str, GFP_KERNEL);
1269                 if (!attrs->clock) {
1270                         ret = -ENOMEM;
1271                         goto out;
1272                 }
1273         } else if ((len = str_has_prefix(str, "size="))) {
1274                 int map_bits = parse_map_size(str + len);
1275
1276                 if (map_bits < 0) {
1277                         ret = map_bits;
1278                         goto out;
1279                 }
1280                 attrs->map_bits = map_bits;
1281         } else {
1282                 char *assignment;
1283
1284                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1285                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1286                         ret = -EINVAL;
1287                         goto out;
1288                 }
1289
1290                 assignment = kstrdup(str, GFP_KERNEL);
1291                 if (!assignment) {
1292                         ret = -ENOMEM;
1293                         goto out;
1294                 }
1295
1296                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1297         }
1298  out:
1299         return ret;
1300 }
1301
1302 static struct hist_trigger_attrs *
1303 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1304 {
1305         struct hist_trigger_attrs *attrs;
1306         int ret = 0;
1307
1308         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1309         if (!attrs)
1310                 return ERR_PTR(-ENOMEM);
1311
1312         while (trigger_str) {
1313                 char *str = strsep(&trigger_str, ":");
1314                 char *rhs;
1315
1316                 rhs = strchr(str, '=');
1317                 if (rhs) {
1318                         if (!strlen(++rhs)) {
1319                                 ret = -EINVAL;
1320                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1321                                 goto free;
1322                         }
1323                         ret = parse_assignment(tr, str, attrs);
1324                         if (ret)
1325                                 goto free;
1326                 } else if (strcmp(str, "pause") == 0)
1327                         attrs->pause = true;
1328                 else if ((strcmp(str, "cont") == 0) ||
1329                          (strcmp(str, "continue") == 0))
1330                         attrs->cont = true;
1331                 else if (strcmp(str, "clear") == 0)
1332                         attrs->clear = true;
1333                 else {
1334                         ret = parse_action(str, attrs);
1335                         if (ret)
1336                                 goto free;
1337                 }
1338         }
1339
1340         if (!attrs->keys_str) {
1341                 ret = -EINVAL;
1342                 goto free;
1343         }
1344
1345         if (!attrs->clock) {
1346                 attrs->clock = kstrdup("global", GFP_KERNEL);
1347                 if (!attrs->clock) {
1348                         ret = -ENOMEM;
1349                         goto free;
1350                 }
1351         }
1352
1353         return attrs;
1354  free:
1355         destroy_hist_trigger_attrs(attrs);
1356
1357         return ERR_PTR(ret);
1358 }
1359
1360 static inline void save_comm(char *comm, struct task_struct *task)
1361 {
1362         if (!task->pid) {
1363                 strcpy(comm, "<idle>");
1364                 return;
1365         }
1366
1367         if (WARN_ON_ONCE(task->pid < 0)) {
1368                 strcpy(comm, "<XXX>");
1369                 return;
1370         }
1371
1372         strncpy(comm, task->comm, TASK_COMM_LEN);
1373 }
1374
1375 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1376 {
1377         unsigned int i;
1378
1379         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1380                 kfree(elt_data->field_var_str[i]);
1381
1382         kfree(elt_data->comm);
1383         kfree(elt_data);
1384 }
1385
1386 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1387 {
1388         struct hist_elt_data *elt_data = elt->private_data;
1389
1390         hist_elt_data_free(elt_data);
1391 }
1392
1393 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1394 {
1395         struct hist_trigger_data *hist_data = elt->map->private_data;
1396         unsigned int size = TASK_COMM_LEN;
1397         struct hist_elt_data *elt_data;
1398         struct hist_field *key_field;
1399         unsigned int i, n_str;
1400
1401         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1402         if (!elt_data)
1403                 return -ENOMEM;
1404
1405         for_each_hist_key_field(i, hist_data) {
1406                 key_field = hist_data->fields[i];
1407
1408                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1409                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1410                         if (!elt_data->comm) {
1411                                 kfree(elt_data);
1412                                 return -ENOMEM;
1413                         }
1414                         break;
1415                 }
1416         }
1417
1418         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1419                 hist_data->n_var_str;
1420         if (n_str > SYNTH_FIELDS_MAX) {
1421                 hist_elt_data_free(elt_data);
1422                 return -EINVAL;
1423         }
1424
1425         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1426
1427         size = STR_VAR_LEN_MAX;
1428
1429         for (i = 0; i < n_str; i++) {
1430                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1431                 if (!elt_data->field_var_str[i]) {
1432                         hist_elt_data_free(elt_data);
1433                         return -ENOMEM;
1434                 }
1435         }
1436
1437         elt->private_data = elt_data;
1438
1439         return 0;
1440 }
1441
1442 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1443 {
1444         struct hist_elt_data *elt_data = elt->private_data;
1445
1446         if (elt_data->comm)
1447                 save_comm(elt_data->comm, current);
1448 }
1449
1450 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1451         .elt_alloc      = hist_trigger_elt_data_alloc,
1452         .elt_free       = hist_trigger_elt_data_free,
1453         .elt_init       = hist_trigger_elt_data_init,
1454 };
1455
1456 static const char *get_hist_field_flags(struct hist_field *hist_field)
1457 {
1458         const char *flags_str = NULL;
1459
1460         if (hist_field->flags & HIST_FIELD_FL_HEX)
1461                 flags_str = "hex";
1462         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1463                 flags_str = "sym";
1464         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1465                 flags_str = "sym-offset";
1466         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1467                 flags_str = "execname";
1468         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1469                 flags_str = "syscall";
1470         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1471                 flags_str = "log2";
1472         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1473                 flags_str = "usecs";
1474
1475         return flags_str;
1476 }
1477
1478 static void expr_field_str(struct hist_field *field, char *expr)
1479 {
1480         if (field->flags & HIST_FIELD_FL_VAR_REF)
1481                 strcat(expr, "$");
1482
1483         strcat(expr, hist_field_name(field, 0));
1484
1485         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1486                 const char *flags_str = get_hist_field_flags(field);
1487
1488                 if (flags_str) {
1489                         strcat(expr, ".");
1490                         strcat(expr, flags_str);
1491                 }
1492         }
1493 }
1494
1495 static char *expr_str(struct hist_field *field, unsigned int level)
1496 {
1497         char *expr;
1498
1499         if (level > 1)
1500                 return NULL;
1501
1502         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1503         if (!expr)
1504                 return NULL;
1505
1506         if (!field->operands[0]) {
1507                 expr_field_str(field, expr);
1508                 return expr;
1509         }
1510
1511         if (field->operator == FIELD_OP_UNARY_MINUS) {
1512                 char *subexpr;
1513
1514                 strcat(expr, "-(");
1515                 subexpr = expr_str(field->operands[0], ++level);
1516                 if (!subexpr) {
1517                         kfree(expr);
1518                         return NULL;
1519                 }
1520                 strcat(expr, subexpr);
1521                 strcat(expr, ")");
1522
1523                 kfree(subexpr);
1524
1525                 return expr;
1526         }
1527
1528         expr_field_str(field->operands[0], expr);
1529
1530         switch (field->operator) {
1531         case FIELD_OP_MINUS:
1532                 strcat(expr, "-");
1533                 break;
1534         case FIELD_OP_PLUS:
1535                 strcat(expr, "+");
1536                 break;
1537         default:
1538                 kfree(expr);
1539                 return NULL;
1540         }
1541
1542         expr_field_str(field->operands[1], expr);
1543
1544         return expr;
1545 }
1546
1547 static int contains_operator(char *str)
1548 {
1549         enum field_op_id field_op = FIELD_OP_NONE;
1550         char *op;
1551
1552         op = strpbrk(str, "+-");
1553         if (!op)
1554                 return FIELD_OP_NONE;
1555
1556         switch (*op) {
1557         case '-':
1558                 if (*str == '-')
1559                         field_op = FIELD_OP_UNARY_MINUS;
1560                 else
1561                         field_op = FIELD_OP_MINUS;
1562                 break;
1563         case '+':
1564                 field_op = FIELD_OP_PLUS;
1565                 break;
1566         default:
1567                 break;
1568         }
1569
1570         return field_op;
1571 }
1572
1573 static void get_hist_field(struct hist_field *hist_field)
1574 {
1575         hist_field->ref++;
1576 }
1577
1578 static void __destroy_hist_field(struct hist_field *hist_field)
1579 {
1580         if (--hist_field->ref > 1)
1581                 return;
1582
1583         kfree(hist_field->var.name);
1584         kfree(hist_field->name);
1585         kfree(hist_field->type);
1586
1587         kfree(hist_field->system);
1588         kfree(hist_field->event_name);
1589
1590         kfree(hist_field);
1591 }
1592
1593 static void destroy_hist_field(struct hist_field *hist_field,
1594                                unsigned int level)
1595 {
1596         unsigned int i;
1597
1598         if (level > 3)
1599                 return;
1600
1601         if (!hist_field)
1602                 return;
1603
1604         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1605                 return; /* var refs will be destroyed separately */
1606
1607         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1608                 destroy_hist_field(hist_field->operands[i], level + 1);
1609
1610         __destroy_hist_field(hist_field);
1611 }
1612
1613 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1614                                             struct ftrace_event_field *field,
1615                                             unsigned long flags,
1616                                             char *var_name)
1617 {
1618         struct hist_field *hist_field;
1619
1620         if (field && is_function_field(field))
1621                 return NULL;
1622
1623         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1624         if (!hist_field)
1625                 return NULL;
1626
1627         hist_field->ref = 1;
1628
1629         hist_field->hist_data = hist_data;
1630
1631         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1632                 goto out; /* caller will populate */
1633
1634         if (flags & HIST_FIELD_FL_VAR_REF) {
1635                 hist_field->fn = hist_field_var_ref;
1636                 goto out;
1637         }
1638
1639         if (flags & HIST_FIELD_FL_HITCOUNT) {
1640                 hist_field->fn = hist_field_counter;
1641                 hist_field->size = sizeof(u64);
1642                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1643                 if (!hist_field->type)
1644                         goto free;
1645                 goto out;
1646         }
1647
1648         if (flags & HIST_FIELD_FL_STACKTRACE) {
1649                 hist_field->fn = hist_field_none;
1650                 goto out;
1651         }
1652
1653         if (flags & HIST_FIELD_FL_LOG2) {
1654                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1655                 hist_field->fn = hist_field_log2;
1656                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1657                 hist_field->size = hist_field->operands[0]->size;
1658                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1659                 if (!hist_field->type)
1660                         goto free;
1661                 goto out;
1662         }
1663
1664         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1665                 hist_field->fn = hist_field_timestamp;
1666                 hist_field->size = sizeof(u64);
1667                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1668                 if (!hist_field->type)
1669                         goto free;
1670                 goto out;
1671         }
1672
1673         if (flags & HIST_FIELD_FL_CPU) {
1674                 hist_field->fn = hist_field_cpu;
1675                 hist_field->size = sizeof(int);
1676                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1677                 if (!hist_field->type)
1678                         goto free;
1679                 goto out;
1680         }
1681
1682         if (WARN_ON_ONCE(!field))
1683                 goto out;
1684
1685         if (is_string_field(field)) {
1686                 flags |= HIST_FIELD_FL_STRING;
1687
1688                 hist_field->size = MAX_FILTER_STR_VAL;
1689                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1690                 if (!hist_field->type)
1691                         goto free;
1692
1693                 if (field->filter_type == FILTER_STATIC_STRING)
1694                         hist_field->fn = hist_field_string;
1695                 else if (field->filter_type == FILTER_DYN_STRING)
1696                         hist_field->fn = hist_field_dynstring;
1697                 else
1698                         hist_field->fn = hist_field_pstring;
1699         } else {
1700                 hist_field->size = field->size;
1701                 hist_field->is_signed = field->is_signed;
1702                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1703                 if (!hist_field->type)
1704                         goto free;
1705
1706                 hist_field->fn = select_value_fn(field->size,
1707                                                  field->is_signed);
1708                 if (!hist_field->fn) {
1709                         destroy_hist_field(hist_field, 0);
1710                         return NULL;
1711                 }
1712         }
1713  out:
1714         hist_field->field = field;
1715         hist_field->flags = flags;
1716
1717         if (var_name) {
1718                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1719                 if (!hist_field->var.name)
1720                         goto free;
1721         }
1722
1723         return hist_field;
1724  free:
1725         destroy_hist_field(hist_field, 0);
1726         return NULL;
1727 }
1728
1729 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1730 {
1731         unsigned int i;
1732
1733         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1734                 if (hist_data->fields[i]) {
1735                         destroy_hist_field(hist_data->fields[i], 0);
1736                         hist_data->fields[i] = NULL;
1737                 }
1738         }
1739
1740         for (i = 0; i < hist_data->n_var_refs; i++) {
1741                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1742                 __destroy_hist_field(hist_data->var_refs[i]);
1743                 hist_data->var_refs[i] = NULL;
1744         }
1745 }
1746
1747 static int init_var_ref(struct hist_field *ref_field,
1748                         struct hist_field *var_field,
1749                         char *system, char *event_name)
1750 {
1751         int err = 0;
1752
1753         ref_field->var.idx = var_field->var.idx;
1754         ref_field->var.hist_data = var_field->hist_data;
1755         ref_field->size = var_field->size;
1756         ref_field->is_signed = var_field->is_signed;
1757         ref_field->flags |= var_field->flags &
1758                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1759
1760         if (system) {
1761                 ref_field->system = kstrdup(system, GFP_KERNEL);
1762                 if (!ref_field->system)
1763                         return -ENOMEM;
1764         }
1765
1766         if (event_name) {
1767                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1768                 if (!ref_field->event_name) {
1769                         err = -ENOMEM;
1770                         goto free;
1771                 }
1772         }
1773
1774         if (var_field->var.name) {
1775                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1776                 if (!ref_field->name) {
1777                         err = -ENOMEM;
1778                         goto free;
1779                 }
1780         } else if (var_field->name) {
1781                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1782                 if (!ref_field->name) {
1783                         err = -ENOMEM;
1784                         goto free;
1785                 }
1786         }
1787
1788         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1789         if (!ref_field->type) {
1790                 err = -ENOMEM;
1791                 goto free;
1792         }
1793  out:
1794         return err;
1795  free:
1796         kfree(ref_field->system);
1797         kfree(ref_field->event_name);
1798         kfree(ref_field->name);
1799
1800         goto out;
1801 }
1802
1803 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1804                             struct hist_field *var_field)
1805 {
1806         struct hist_field *ref_field;
1807         int i;
1808
1809         for (i = 0; i < hist_data->n_var_refs; i++) {
1810                 ref_field = hist_data->var_refs[i];
1811                 if (ref_field->var.idx == var_field->var.idx &&
1812                     ref_field->var.hist_data == var_field->hist_data)
1813                         return i;
1814         }
1815
1816         return -ENOENT;
1817 }
1818
1819 /**
1820  * create_var_ref - Create a variable reference and attach it to trigger
1821  * @hist_data: The trigger that will be referencing the variable
1822  * @var_field: The VAR field to create a reference to
1823  * @system: The optional system string
1824  * @event_name: The optional event_name string
1825  *
1826  * Given a variable hist_field, create a VAR_REF hist_field that
1827  * represents a reference to it.
1828  *
1829  * This function also adds the reference to the trigger that
1830  * now references the variable.
1831  *
1832  * Return: The VAR_REF field if successful, NULL if not
1833  */
1834 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1835                                          struct hist_field *var_field,
1836                                          char *system, char *event_name)
1837 {
1838         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1839         struct hist_field *ref_field;
1840         int i;
1841
1842         /* Check if the variable already exists */
1843         for (i = 0; i < hist_data->n_var_refs; i++) {
1844                 ref_field = hist_data->var_refs[i];
1845                 if (ref_field->var.idx == var_field->var.idx &&
1846                     ref_field->var.hist_data == var_field->hist_data) {
1847                         get_hist_field(ref_field);
1848                         return ref_field;
1849                 }
1850         }
1851
1852         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1853         if (ref_field) {
1854                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1855                         destroy_hist_field(ref_field, 0);
1856                         return NULL;
1857                 }
1858
1859                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1860                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1861         }
1862
1863         return ref_field;
1864 }
1865
1866 static bool is_var_ref(char *var_name)
1867 {
1868         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1869                 return false;
1870
1871         return true;
1872 }
1873
1874 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1875                                  char *var_name)
1876 {
1877         char *name, *field;
1878         unsigned int i;
1879
1880         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1881                 name = hist_data->attrs->var_defs.name[i];
1882
1883                 if (strcmp(var_name, name) == 0) {
1884                         field = hist_data->attrs->var_defs.expr[i];
1885                         if (contains_operator(field) || is_var_ref(field))
1886                                 continue;
1887                         return field;
1888                 }
1889         }
1890
1891         return NULL;
1892 }
1893
1894 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1895                                  char *system, char *event_name,
1896                                  char *var_name)
1897 {
1898         struct trace_event_call *call;
1899
1900         if (system && event_name) {
1901                 call = hist_data->event_file->event_call;
1902
1903                 if (strcmp(system, call->class->system) != 0)
1904                         return NULL;
1905
1906                 if (strcmp(event_name, trace_event_name(call)) != 0)
1907                         return NULL;
1908         }
1909
1910         if (!!system != !!event_name)
1911                 return NULL;
1912
1913         if (!is_var_ref(var_name))
1914                 return NULL;
1915
1916         var_name++;
1917
1918         return field_name_from_var(hist_data, var_name);
1919 }
1920
1921 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1922                                         char *system, char *event_name,
1923                                         char *var_name)
1924 {
1925         struct hist_field *var_field = NULL, *ref_field = NULL;
1926         struct trace_array *tr = hist_data->event_file->tr;
1927
1928         if (!is_var_ref(var_name))
1929                 return NULL;
1930
1931         var_name++;
1932
1933         var_field = find_event_var(hist_data, system, event_name, var_name);
1934         if (var_field)
1935                 ref_field = create_var_ref(hist_data, var_field,
1936                                            system, event_name);
1937
1938         if (!ref_field)
1939                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1940
1941         return ref_field;
1942 }
1943
1944 static struct ftrace_event_field *
1945 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1946             char *field_str, unsigned long *flags)
1947 {
1948         struct ftrace_event_field *field = NULL;
1949         char *field_name, *modifier, *str;
1950         struct trace_array *tr = file->tr;
1951
1952         modifier = str = kstrdup(field_str, GFP_KERNEL);
1953         if (!modifier)
1954                 return ERR_PTR(-ENOMEM);
1955
1956         field_name = strsep(&modifier, ".");
1957         if (modifier) {
1958                 if (strcmp(modifier, "hex") == 0)
1959                         *flags |= HIST_FIELD_FL_HEX;
1960                 else if (strcmp(modifier, "sym") == 0)
1961                         *flags |= HIST_FIELD_FL_SYM;
1962                 else if (strcmp(modifier, "sym-offset") == 0)
1963                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1964                 else if ((strcmp(modifier, "execname") == 0) &&
1965                          (strcmp(field_name, "common_pid") == 0))
1966                         *flags |= HIST_FIELD_FL_EXECNAME;
1967                 else if (strcmp(modifier, "syscall") == 0)
1968                         *flags |= HIST_FIELD_FL_SYSCALL;
1969                 else if (strcmp(modifier, "log2") == 0)
1970                         *flags |= HIST_FIELD_FL_LOG2;
1971                 else if (strcmp(modifier, "usecs") == 0)
1972                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1973                 else {
1974                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1975                         field = ERR_PTR(-EINVAL);
1976                         goto out;
1977                 }
1978         }
1979
1980         if (strcmp(field_name, "common_timestamp") == 0) {
1981                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1982                 hist_data->enable_timestamps = true;
1983                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1984                         hist_data->attrs->ts_in_usecs = true;
1985         } else if (strcmp(field_name, "cpu") == 0)
1986                 *flags |= HIST_FIELD_FL_CPU;
1987         else {
1988                 field = trace_find_event_field(file->event_call, field_name);
1989                 if (!field || !field->size) {
1990                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1991                         field = ERR_PTR(-EINVAL);
1992                         goto out;
1993                 }
1994         }
1995  out:
1996         kfree(str);
1997
1998         return field;
1999 }
2000
2001 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2002                                        struct hist_field *var_ref,
2003                                        char *var_name)
2004 {
2005         struct hist_field *alias = NULL;
2006         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2007
2008         alias = create_hist_field(hist_data, NULL, flags, var_name);
2009         if (!alias)
2010                 return NULL;
2011
2012         alias->fn = var_ref->fn;
2013         alias->operands[0] = var_ref;
2014
2015         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2016                 destroy_hist_field(alias, 0);
2017                 return NULL;
2018         }
2019
2020         alias->var_ref_idx = var_ref->var_ref_idx;
2021
2022         return alias;
2023 }
2024
2025 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2026                                      struct trace_event_file *file, char *str,
2027                                      unsigned long *flags, char *var_name)
2028 {
2029         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2030         struct ftrace_event_field *field = NULL;
2031         struct hist_field *hist_field = NULL;
2032         int ret = 0;
2033
2034         s = strchr(str, '.');
2035         if (s) {
2036                 s = strchr(++s, '.');
2037                 if (s) {
2038                         ref_system = strsep(&str, ".");
2039                         if (!str) {
2040                                 ret = -EINVAL;
2041                                 goto out;
2042                         }
2043                         ref_event = strsep(&str, ".");
2044                         if (!str) {
2045                                 ret = -EINVAL;
2046                                 goto out;
2047                         }
2048                         ref_var = str;
2049                 }
2050         }
2051
2052         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2053         if (!s) {
2054                 hist_field = parse_var_ref(hist_data, ref_system,
2055                                            ref_event, ref_var);
2056                 if (hist_field) {
2057                         if (var_name) {
2058                                 hist_field = create_alias(hist_data, hist_field, var_name);
2059                                 if (!hist_field) {
2060                                         ret = -ENOMEM;
2061                                         goto out;
2062                                 }
2063                         }
2064                         return hist_field;
2065                 }
2066         } else
2067                 str = s;
2068
2069         field = parse_field(hist_data, file, str, flags);
2070         if (IS_ERR(field)) {
2071                 ret = PTR_ERR(field);
2072                 goto out;
2073         }
2074
2075         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2076         if (!hist_field) {
2077                 ret = -ENOMEM;
2078                 goto out;
2079         }
2080
2081         return hist_field;
2082  out:
2083         return ERR_PTR(ret);
2084 }
2085
2086 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2087                                      struct trace_event_file *file,
2088                                      char *str, unsigned long flags,
2089                                      char *var_name, unsigned int level);
2090
2091 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2092                                       struct trace_event_file *file,
2093                                       char *str, unsigned long flags,
2094                                       char *var_name, unsigned int level)
2095 {
2096         struct hist_field *operand1, *expr = NULL;
2097         unsigned long operand_flags;
2098         int ret = 0;
2099         char *s;
2100
2101         /* we support only -(xxx) i.e. explicit parens required */
2102
2103         if (level > 3) {
2104                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2105                 ret = -EINVAL;
2106                 goto free;
2107         }
2108
2109         str++; /* skip leading '-' */
2110
2111         s = strchr(str, '(');
2112         if (s)
2113                 str++;
2114         else {
2115                 ret = -EINVAL;
2116                 goto free;
2117         }
2118
2119         s = strrchr(str, ')');
2120         if (s)
2121                 *s = '\0';
2122         else {
2123                 ret = -EINVAL; /* no closing ')' */
2124                 goto free;
2125         }
2126
2127         flags |= HIST_FIELD_FL_EXPR;
2128         expr = create_hist_field(hist_data, NULL, flags, var_name);
2129         if (!expr) {
2130                 ret = -ENOMEM;
2131                 goto free;
2132         }
2133
2134         operand_flags = 0;
2135         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2136         if (IS_ERR(operand1)) {
2137                 ret = PTR_ERR(operand1);
2138                 goto free;
2139         }
2140
2141         expr->flags |= operand1->flags &
2142                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2143         expr->fn = hist_field_unary_minus;
2144         expr->operands[0] = operand1;
2145         expr->operator = FIELD_OP_UNARY_MINUS;
2146         expr->name = expr_str(expr, 0);
2147         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2148         if (!expr->type) {
2149                 ret = -ENOMEM;
2150                 goto free;
2151         }
2152
2153         return expr;
2154  free:
2155         destroy_hist_field(expr, 0);
2156         return ERR_PTR(ret);
2157 }
2158
2159 static int check_expr_operands(struct trace_array *tr,
2160                                struct hist_field *operand1,
2161                                struct hist_field *operand2)
2162 {
2163         unsigned long operand1_flags = operand1->flags;
2164         unsigned long operand2_flags = operand2->flags;
2165
2166         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2167             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2168                 struct hist_field *var;
2169
2170                 var = find_var_field(operand1->var.hist_data, operand1->name);
2171                 if (!var)
2172                         return -EINVAL;
2173                 operand1_flags = var->flags;
2174         }
2175
2176         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2177             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2178                 struct hist_field *var;
2179
2180                 var = find_var_field(operand2->var.hist_data, operand2->name);
2181                 if (!var)
2182                         return -EINVAL;
2183                 operand2_flags = var->flags;
2184         }
2185
2186         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2187             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2188                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2189                 return -EINVAL;
2190         }
2191
2192         return 0;
2193 }
2194
2195 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2196                                      struct trace_event_file *file,
2197                                      char *str, unsigned long flags,
2198                                      char *var_name, unsigned int level)
2199 {
2200         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2201         unsigned long operand_flags;
2202         int field_op, ret = -EINVAL;
2203         char *sep, *operand1_str;
2204
2205         if (level > 3) {
2206                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2207                 return ERR_PTR(-EINVAL);
2208         }
2209
2210         field_op = contains_operator(str);
2211
2212         if (field_op == FIELD_OP_NONE)
2213                 return parse_atom(hist_data, file, str, &flags, var_name);
2214
2215         if (field_op == FIELD_OP_UNARY_MINUS)
2216                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2217
2218         switch (field_op) {
2219         case FIELD_OP_MINUS:
2220                 sep = "-";
2221                 break;
2222         case FIELD_OP_PLUS:
2223                 sep = "+";
2224                 break;
2225         default:
2226                 goto free;
2227         }
2228
2229         operand1_str = strsep(&str, sep);
2230         if (!operand1_str || !str)
2231                 goto free;
2232
2233         operand_flags = 0;
2234         operand1 = parse_atom(hist_data, file, operand1_str,
2235                               &operand_flags, NULL);
2236         if (IS_ERR(operand1)) {
2237                 ret = PTR_ERR(operand1);
2238                 operand1 = NULL;
2239                 goto free;
2240         }
2241
2242         /* rest of string could be another expression e.g. b+c in a+b+c */
2243         operand_flags = 0;
2244         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2245         if (IS_ERR(operand2)) {
2246                 ret = PTR_ERR(operand2);
2247                 operand2 = NULL;
2248                 goto free;
2249         }
2250
2251         ret = check_expr_operands(file->tr, operand1, operand2);
2252         if (ret)
2253                 goto free;
2254
2255         flags |= HIST_FIELD_FL_EXPR;
2256
2257         flags |= operand1->flags &
2258                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2259
2260         expr = create_hist_field(hist_data, NULL, flags, var_name);
2261         if (!expr) {
2262                 ret = -ENOMEM;
2263                 goto free;
2264         }
2265
2266         operand1->read_once = true;
2267         operand2->read_once = true;
2268
2269         expr->operands[0] = operand1;
2270         expr->operands[1] = operand2;
2271         expr->operator = field_op;
2272         expr->name = expr_str(expr, 0);
2273         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2274         if (!expr->type) {
2275                 ret = -ENOMEM;
2276                 goto free;
2277         }
2278
2279         switch (field_op) {
2280         case FIELD_OP_MINUS:
2281                 expr->fn = hist_field_minus;
2282                 break;
2283         case FIELD_OP_PLUS:
2284                 expr->fn = hist_field_plus;
2285                 break;
2286         default:
2287                 ret = -EINVAL;
2288                 goto free;
2289         }
2290
2291         return expr;
2292  free:
2293         destroy_hist_field(operand1, 0);
2294         destroy_hist_field(operand2, 0);
2295         destroy_hist_field(expr, 0);
2296
2297         return ERR_PTR(ret);
2298 }
2299
2300 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2301                                  struct trace_event_file *file)
2302 {
2303         struct event_trigger_data *test;
2304
2305         lockdep_assert_held(&event_mutex);
2306
2307         list_for_each_entry(test, &file->triggers, list) {
2308                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2309                         if (test->private_data == hist_data)
2310                                 return test->filter_str;
2311                 }
2312         }
2313
2314         return NULL;
2315 }
2316
2317 static struct event_command trigger_hist_cmd;
2318 static int event_hist_trigger_func(struct event_command *cmd_ops,
2319                                    struct trace_event_file *file,
2320                                    char *glob, char *cmd, char *param);
2321
2322 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2323                             struct hist_trigger_data *hist_data,
2324                             unsigned int n_keys)
2325 {
2326         struct hist_field *target_hist_field, *hist_field;
2327         unsigned int n, i, j;
2328
2329         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2330                 return false;
2331
2332         i = hist_data->n_vals;
2333         j = target_hist_data->n_vals;
2334
2335         for (n = 0; n < n_keys; n++) {
2336                 hist_field = hist_data->fields[i + n];
2337                 target_hist_field = target_hist_data->fields[j + n];
2338
2339                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2340                         return false;
2341                 if (hist_field->size != target_hist_field->size)
2342                         return false;
2343                 if (hist_field->is_signed != target_hist_field->is_signed)
2344                         return false;
2345         }
2346
2347         return true;
2348 }
2349
2350 static struct hist_trigger_data *
2351 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2352                      struct trace_event_file *file)
2353 {
2354         struct hist_trigger_data *hist_data;
2355         struct event_trigger_data *test;
2356         unsigned int n_keys;
2357
2358         lockdep_assert_held(&event_mutex);
2359
2360         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2361
2362         list_for_each_entry(test, &file->triggers, list) {
2363                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2364                         hist_data = test->private_data;
2365
2366                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2367                                 return hist_data;
2368                 }
2369         }
2370
2371         return NULL;
2372 }
2373
2374 static struct trace_event_file *event_file(struct trace_array *tr,
2375                                            char *system, char *event_name)
2376 {
2377         struct trace_event_file *file;
2378
2379         file = __find_event_file(tr, system, event_name);
2380         if (!file)
2381                 return ERR_PTR(-EINVAL);
2382
2383         return file;
2384 }
2385
2386 static struct hist_field *
2387 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2388                          char *system, char *event_name, char *field_name)
2389 {
2390         struct hist_field *event_var;
2391         char *synthetic_name;
2392
2393         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2394         if (!synthetic_name)
2395                 return ERR_PTR(-ENOMEM);
2396
2397         strcpy(synthetic_name, "synthetic_");
2398         strcat(synthetic_name, field_name);
2399
2400         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2401
2402         kfree(synthetic_name);
2403
2404         return event_var;
2405 }
2406
2407 /**
2408  * create_field_var_hist - Automatically create a histogram and var for a field
2409  * @target_hist_data: The target hist trigger
2410  * @subsys_name: Optional subsystem name
2411  * @event_name: Optional event name
2412  * @field_name: The name of the field (and the resulting variable)
2413  *
2414  * Hist trigger actions fetch data from variables, not directly from
2415  * events.  However, for convenience, users are allowed to directly
2416  * specify an event field in an action, which will be automatically
2417  * converted into a variable on their behalf.
2418
2419  * If a user specifies a field on an event that isn't the event the
2420  * histogram currently being defined (the target event histogram), the
2421  * only way that can be accomplished is if a new hist trigger is
2422  * created and the field variable defined on that.
2423  *
2424  * This function creates a new histogram compatible with the target
2425  * event (meaning a histogram with the same key as the target
2426  * histogram), and creates a variable for the specified field, but
2427  * with 'synthetic_' prepended to the variable name in order to avoid
2428  * collision with normal field variables.
2429  *
2430  * Return: The variable created for the field.
2431  */
2432 static struct hist_field *
2433 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2434                       char *subsys_name, char *event_name, char *field_name)
2435 {
2436         struct trace_array *tr = target_hist_data->event_file->tr;
2437         struct hist_field *event_var = ERR_PTR(-EINVAL);
2438         struct hist_trigger_data *hist_data;
2439         unsigned int i, n, first = true;
2440         struct field_var_hist *var_hist;
2441         struct trace_event_file *file;
2442         struct hist_field *key_field;
2443         char *saved_filter;
2444         char *cmd;
2445         int ret;
2446
2447         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2448                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2449                 return ERR_PTR(-EINVAL);
2450         }
2451
2452         file = event_file(tr, subsys_name, event_name);
2453
2454         if (IS_ERR(file)) {
2455                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2456                 ret = PTR_ERR(file);
2457                 return ERR_PTR(ret);
2458         }
2459
2460         /*
2461          * Look for a histogram compatible with target.  We'll use the
2462          * found histogram specification to create a new matching
2463          * histogram with our variable on it.  target_hist_data is not
2464          * yet a registered histogram so we can't use that.
2465          */
2466         hist_data = find_compatible_hist(target_hist_data, file);
2467         if (!hist_data) {
2468                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2469                 return ERR_PTR(-EINVAL);
2470         }
2471
2472         /* See if a synthetic field variable has already been created */
2473         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2474                                              event_name, field_name);
2475         if (!IS_ERR_OR_NULL(event_var))
2476                 return event_var;
2477
2478         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2479         if (!var_hist)
2480                 return ERR_PTR(-ENOMEM);
2481
2482         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2483         if (!cmd) {
2484                 kfree(var_hist);
2485                 return ERR_PTR(-ENOMEM);
2486         }
2487
2488         /* Use the same keys as the compatible histogram */
2489         strcat(cmd, "keys=");
2490
2491         for_each_hist_key_field(i, hist_data) {
2492                 key_field = hist_data->fields[i];
2493                 if (!first)
2494                         strcat(cmd, ",");
2495                 strcat(cmd, key_field->field->name);
2496                 first = false;
2497         }
2498
2499         /* Create the synthetic field variable specification */
2500         strcat(cmd, ":synthetic_");
2501         strcat(cmd, field_name);
2502         strcat(cmd, "=");
2503         strcat(cmd, field_name);
2504
2505         /* Use the same filter as the compatible histogram */
2506         saved_filter = find_trigger_filter(hist_data, file);
2507         if (saved_filter) {
2508                 strcat(cmd, " if ");
2509                 strcat(cmd, saved_filter);
2510         }
2511
2512         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2513         if (!var_hist->cmd) {
2514                 kfree(cmd);
2515                 kfree(var_hist);
2516                 return ERR_PTR(-ENOMEM);
2517         }
2518
2519         /* Save the compatible histogram information */
2520         var_hist->hist_data = hist_data;
2521
2522         /* Create the new histogram with our variable */
2523         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2524                                       "", "hist", cmd);
2525         if (ret) {
2526                 kfree(cmd);
2527                 kfree(var_hist->cmd);
2528                 kfree(var_hist);
2529                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2530                 return ERR_PTR(ret);
2531         }
2532
2533         kfree(cmd);
2534
2535         /* If we can't find the variable, something went wrong */
2536         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2537                                              event_name, field_name);
2538         if (IS_ERR_OR_NULL(event_var)) {
2539                 kfree(var_hist->cmd);
2540                 kfree(var_hist);
2541                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2542                 return ERR_PTR(-EINVAL);
2543         }
2544
2545         n = target_hist_data->n_field_var_hists;
2546         target_hist_data->field_var_hists[n] = var_hist;
2547         target_hist_data->n_field_var_hists++;
2548
2549         return event_var;
2550 }
2551
2552 static struct hist_field *
2553 find_target_event_var(struct hist_trigger_data *hist_data,
2554                       char *subsys_name, char *event_name, char *var_name)
2555 {
2556         struct trace_event_file *file = hist_data->event_file;
2557         struct hist_field *hist_field = NULL;
2558
2559         if (subsys_name) {
2560                 struct trace_event_call *call;
2561
2562                 if (!event_name)
2563                         return NULL;
2564
2565                 call = file->event_call;
2566
2567                 if (strcmp(subsys_name, call->class->system) != 0)
2568                         return NULL;
2569
2570                 if (strcmp(event_name, trace_event_name(call)) != 0)
2571                         return NULL;
2572         }
2573
2574         hist_field = find_var_field(hist_data, var_name);
2575
2576         return hist_field;
2577 }
2578
2579 static inline void __update_field_vars(struct tracing_map_elt *elt,
2580                                        struct trace_buffer *buffer,
2581                                        struct ring_buffer_event *rbe,
2582                                        void *rec,
2583                                        struct field_var **field_vars,
2584                                        unsigned int n_field_vars,
2585                                        unsigned int field_var_str_start)
2586 {
2587         struct hist_elt_data *elt_data = elt->private_data;
2588         unsigned int i, j, var_idx;
2589         u64 var_val;
2590
2591         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2592                 struct field_var *field_var = field_vars[i];
2593                 struct hist_field *var = field_var->var;
2594                 struct hist_field *val = field_var->val;
2595
2596                 var_val = val->fn(val, elt, buffer, rbe, rec);
2597                 var_idx = var->var.idx;
2598
2599                 if (val->flags & HIST_FIELD_FL_STRING) {
2600                         char *str = elt_data->field_var_str[j++];
2601                         char *val_str = (char *)(uintptr_t)var_val;
2602
2603                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2604                         var_val = (u64)(uintptr_t)str;
2605                 }
2606                 tracing_map_set_var(elt, var_idx, var_val);
2607         }
2608 }
2609
2610 static void update_field_vars(struct hist_trigger_data *hist_data,
2611                               struct tracing_map_elt *elt,
2612                               struct trace_buffer *buffer,
2613                               struct ring_buffer_event *rbe,
2614                               void *rec)
2615 {
2616         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2617                             hist_data->n_field_vars, 0);
2618 }
2619
2620 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2621                                  struct tracing_map_elt *elt,
2622                                  struct trace_buffer *buffer,  void *rec,
2623                                  struct ring_buffer_event *rbe, void *key,
2624                                  struct action_data *data, u64 *var_ref_vals)
2625 {
2626         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2627                             hist_data->n_save_vars, hist_data->n_field_var_str);
2628 }
2629
2630 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2631                                      struct trace_event_file *file,
2632                                      char *name, int size, const char *type)
2633 {
2634         struct hist_field *var;
2635         int idx;
2636
2637         if (find_var(hist_data, file, name) && !hist_data->remove) {
2638                 var = ERR_PTR(-EINVAL);
2639                 goto out;
2640         }
2641
2642         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2643         if (!var) {
2644                 var = ERR_PTR(-ENOMEM);
2645                 goto out;
2646         }
2647
2648         idx = tracing_map_add_var(hist_data->map);
2649         if (idx < 0) {
2650                 kfree(var);
2651                 var = ERR_PTR(-EINVAL);
2652                 goto out;
2653         }
2654
2655         var->ref = 1;
2656         var->flags = HIST_FIELD_FL_VAR;
2657         var->var.idx = idx;
2658         var->var.hist_data = var->hist_data = hist_data;
2659         var->size = size;
2660         var->var.name = kstrdup(name, GFP_KERNEL);
2661         var->type = kstrdup(type, GFP_KERNEL);
2662         if (!var->var.name || !var->type) {
2663                 kfree(var->var.name);
2664                 kfree(var->type);
2665                 kfree(var);
2666                 var = ERR_PTR(-ENOMEM);
2667         }
2668  out:
2669         return var;
2670 }
2671
2672 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2673                                           struct trace_event_file *file,
2674                                           char *field_name)
2675 {
2676         struct hist_field *val = NULL, *var = NULL;
2677         unsigned long flags = HIST_FIELD_FL_VAR;
2678         struct trace_array *tr = file->tr;
2679         struct field_var *field_var;
2680         int ret = 0;
2681
2682         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2683                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2684                 ret = -EINVAL;
2685                 goto err;
2686         }
2687
2688         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2689         if (IS_ERR(val)) {
2690                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2691                 ret = PTR_ERR(val);
2692                 goto err;
2693         }
2694
2695         var = create_var(hist_data, file, field_name, val->size, val->type);
2696         if (IS_ERR(var)) {
2697                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2698                 kfree(val);
2699                 ret = PTR_ERR(var);
2700                 goto err;
2701         }
2702
2703         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2704         if (!field_var) {
2705                 kfree(val);
2706                 kfree(var);
2707                 ret =  -ENOMEM;
2708                 goto err;
2709         }
2710
2711         field_var->var = var;
2712         field_var->val = val;
2713  out:
2714         return field_var;
2715  err:
2716         field_var = ERR_PTR(ret);
2717         goto out;
2718 }
2719
2720 /**
2721  * create_target_field_var - Automatically create a variable for a field
2722  * @target_hist_data: The target hist trigger
2723  * @subsys_name: Optional subsystem name
2724  * @event_name: Optional event name
2725  * @var_name: The name of the field (and the resulting variable)
2726  *
2727  * Hist trigger actions fetch data from variables, not directly from
2728  * events.  However, for convenience, users are allowed to directly
2729  * specify an event field in an action, which will be automatically
2730  * converted into a variable on their behalf.
2731
2732  * This function creates a field variable with the name var_name on
2733  * the hist trigger currently being defined on the target event.  If
2734  * subsys_name and event_name are specified, this function simply
2735  * verifies that they do in fact match the target event subsystem and
2736  * event name.
2737  *
2738  * Return: The variable created for the field.
2739  */
2740 static struct field_var *
2741 create_target_field_var(struct hist_trigger_data *target_hist_data,
2742                         char *subsys_name, char *event_name, char *var_name)
2743 {
2744         struct trace_event_file *file = target_hist_data->event_file;
2745
2746         if (subsys_name) {
2747                 struct trace_event_call *call;
2748
2749                 if (!event_name)
2750                         return NULL;
2751
2752                 call = file->event_call;
2753
2754                 if (strcmp(subsys_name, call->class->system) != 0)
2755                         return NULL;
2756
2757                 if (strcmp(event_name, trace_event_name(call)) != 0)
2758                         return NULL;
2759         }
2760
2761         return create_field_var(target_hist_data, file, var_name);
2762 }
2763
2764 static bool check_track_val_max(u64 track_val, u64 var_val)
2765 {
2766         if (var_val <= track_val)
2767                 return false;
2768
2769         return true;
2770 }
2771
2772 static bool check_track_val_changed(u64 track_val, u64 var_val)
2773 {
2774         if (var_val == track_val)
2775                 return false;
2776
2777         return true;
2778 }
2779
2780 static u64 get_track_val(struct hist_trigger_data *hist_data,
2781                          struct tracing_map_elt *elt,
2782                          struct action_data *data)
2783 {
2784         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2785         u64 track_val;
2786
2787         track_val = tracing_map_read_var(elt, track_var_idx);
2788
2789         return track_val;
2790 }
2791
2792 static void save_track_val(struct hist_trigger_data *hist_data,
2793                            struct tracing_map_elt *elt,
2794                            struct action_data *data, u64 var_val)
2795 {
2796         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2797
2798         tracing_map_set_var(elt, track_var_idx, var_val);
2799 }
2800
2801 static void save_track_data(struct hist_trigger_data *hist_data,
2802                             struct tracing_map_elt *elt,
2803                             struct trace_buffer *buffer, void *rec,
2804                             struct ring_buffer_event *rbe, void *key,
2805                             struct action_data *data, u64 *var_ref_vals)
2806 {
2807         if (data->track_data.save_data)
2808                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2809                                            key, data, var_ref_vals);
2810 }
2811
2812 static bool check_track_val(struct tracing_map_elt *elt,
2813                             struct action_data *data,
2814                             u64 var_val)
2815 {
2816         struct hist_trigger_data *hist_data;
2817         u64 track_val;
2818
2819         hist_data = data->track_data.track_var->hist_data;
2820         track_val = get_track_val(hist_data, elt, data);
2821
2822         return data->track_data.check_val(track_val, var_val);
2823 }
2824
2825 #ifdef CONFIG_TRACER_SNAPSHOT
2826 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2827 {
2828         /* called with tr->max_lock held */
2829         struct track_data *track_data = tr->cond_snapshot->cond_data;
2830         struct hist_elt_data *elt_data, *track_elt_data;
2831         struct snapshot_context *context = cond_data;
2832         struct action_data *action;
2833         u64 track_val;
2834
2835         if (!track_data)
2836                 return false;
2837
2838         action = track_data->action_data;
2839
2840         track_val = get_track_val(track_data->hist_data, context->elt,
2841                                   track_data->action_data);
2842
2843         if (!action->track_data.check_val(track_data->track_val, track_val))
2844                 return false;
2845
2846         track_data->track_val = track_val;
2847         memcpy(track_data->key, context->key, track_data->key_len);
2848
2849         elt_data = context->elt->private_data;
2850         track_elt_data = track_data->elt.private_data;
2851         if (elt_data->comm)
2852                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2853
2854         track_data->updated = true;
2855
2856         return true;
2857 }
2858
2859 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2860                                      struct tracing_map_elt *elt,
2861                                      struct trace_buffer *buffer, void *rec,
2862                                      struct ring_buffer_event *rbe, void *key,
2863                                      struct action_data *data,
2864                                      u64 *var_ref_vals)
2865 {
2866         struct trace_event_file *file = hist_data->event_file;
2867         struct snapshot_context context;
2868
2869         context.elt = elt;
2870         context.key = key;
2871
2872         tracing_snapshot_cond(file->tr, &context);
2873 }
2874
2875 static void hist_trigger_print_key(struct seq_file *m,
2876                                    struct hist_trigger_data *hist_data,
2877                                    void *key,
2878                                    struct tracing_map_elt *elt);
2879
2880 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2881 {
2882         unsigned int i;
2883
2884         if (!hist_data->n_actions)
2885                 return NULL;
2886
2887         for (i = 0; i < hist_data->n_actions; i++) {
2888                 struct action_data *data = hist_data->actions[i];
2889
2890                 if (data->action == ACTION_SNAPSHOT)
2891                         return data;
2892         }
2893
2894         return NULL;
2895 }
2896
2897 static void track_data_snapshot_print(struct seq_file *m,
2898                                       struct hist_trigger_data *hist_data)
2899 {
2900         struct trace_event_file *file = hist_data->event_file;
2901         struct track_data *track_data;
2902         struct action_data *action;
2903
2904         track_data = tracing_cond_snapshot_data(file->tr);
2905         if (!track_data)
2906                 return;
2907
2908         if (!track_data->updated)
2909                 return;
2910
2911         action = snapshot_action(hist_data);
2912         if (!action)
2913                 return;
2914
2915         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2916         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2917                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2918                    action->track_data.var_str, track_data->track_val);
2919
2920         seq_puts(m, "\ttriggered by event with key: ");
2921         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2922         seq_putc(m, '\n');
2923 }
2924 #else
2925 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2926 {
2927         return false;
2928 }
2929 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2930                                      struct tracing_map_elt *elt,
2931                                      struct trace_buffer *buffer, void *rec,
2932                                      struct ring_buffer_event *rbe, void *key,
2933                                      struct action_data *data,
2934                                      u64 *var_ref_vals) {}
2935 static void track_data_snapshot_print(struct seq_file *m,
2936                                       struct hist_trigger_data *hist_data) {}
2937 #endif /* CONFIG_TRACER_SNAPSHOT */
2938
2939 static void track_data_print(struct seq_file *m,
2940                              struct hist_trigger_data *hist_data,
2941                              struct tracing_map_elt *elt,
2942                              struct action_data *data)
2943 {
2944         u64 track_val = get_track_val(hist_data, elt, data);
2945         unsigned int i, save_var_idx;
2946
2947         if (data->handler == HANDLER_ONMAX)
2948                 seq_printf(m, "\n\tmax: %10llu", track_val);
2949         else if (data->handler == HANDLER_ONCHANGE)
2950                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2951
2952         if (data->action == ACTION_SNAPSHOT)
2953                 return;
2954
2955         for (i = 0; i < hist_data->n_save_vars; i++) {
2956                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2957                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2958                 u64 val;
2959
2960                 save_var_idx = save_var->var.idx;
2961
2962                 val = tracing_map_read_var(elt, save_var_idx);
2963
2964                 if (save_val->flags & HIST_FIELD_FL_STRING) {
2965                         seq_printf(m, "  %s: %-32s", save_var->var.name,
2966                                    (char *)(uintptr_t)(val));
2967                 } else
2968                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2969         }
2970 }
2971
2972 static void ontrack_action(struct hist_trigger_data *hist_data,
2973                            struct tracing_map_elt *elt,
2974                            struct trace_buffer *buffer, void *rec,
2975                            struct ring_buffer_event *rbe, void *key,
2976                            struct action_data *data, u64 *var_ref_vals)
2977 {
2978         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2979
2980         if (check_track_val(elt, data, var_val)) {
2981                 save_track_val(hist_data, elt, data, var_val);
2982                 save_track_data(hist_data, elt, buffer, rec, rbe,
2983                                 key, data, var_ref_vals);
2984         }
2985 }
2986
2987 static void action_data_destroy(struct action_data *data)
2988 {
2989         unsigned int i;
2990
2991         lockdep_assert_held(&event_mutex);
2992
2993         kfree(data->action_name);
2994
2995         for (i = 0; i < data->n_params; i++)
2996                 kfree(data->params[i]);
2997
2998         if (data->synth_event)
2999                 data->synth_event->ref--;
3000
3001         kfree(data->synth_event_name);
3002
3003         kfree(data);
3004 }
3005
3006 static void track_data_destroy(struct hist_trigger_data *hist_data,
3007                                struct action_data *data)
3008 {
3009         struct trace_event_file *file = hist_data->event_file;
3010
3011         destroy_hist_field(data->track_data.track_var, 0);
3012
3013         if (data->action == ACTION_SNAPSHOT) {
3014                 struct track_data *track_data;
3015
3016                 track_data = tracing_cond_snapshot_data(file->tr);
3017                 if (track_data && track_data->hist_data == hist_data) {
3018                         tracing_snapshot_cond_disable(file->tr);
3019                         track_data_free(track_data);
3020                 }
3021         }
3022
3023         kfree(data->track_data.var_str);
3024
3025         action_data_destroy(data);
3026 }
3027
3028 static int action_create(struct hist_trigger_data *hist_data,
3029                          struct action_data *data);
3030
3031 static int track_data_create(struct hist_trigger_data *hist_data,
3032                              struct action_data *data)
3033 {
3034         struct hist_field *var_field, *ref_field, *track_var = NULL;
3035         struct trace_event_file *file = hist_data->event_file;
3036         struct trace_array *tr = file->tr;
3037         char *track_data_var_str;
3038         int ret = 0;
3039
3040         track_data_var_str = data->track_data.var_str;
3041         if (track_data_var_str[0] != '$') {
3042                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3043                 return -EINVAL;
3044         }
3045         track_data_var_str++;
3046
3047         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3048         if (!var_field) {
3049                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3050                 return -EINVAL;
3051         }
3052
3053         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3054         if (!ref_field)
3055                 return -ENOMEM;
3056
3057         data->track_data.var_ref = ref_field;
3058
3059         if (data->handler == HANDLER_ONMAX)
3060                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3061         if (IS_ERR(track_var)) {
3062                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3063                 ret = PTR_ERR(track_var);
3064                 goto out;
3065         }
3066
3067         if (data->handler == HANDLER_ONCHANGE)
3068                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3069         if (IS_ERR(track_var)) {
3070                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3071                 ret = PTR_ERR(track_var);
3072                 goto out;
3073         }
3074         data->track_data.track_var = track_var;
3075
3076         ret = action_create(hist_data, data);
3077  out:
3078         return ret;
3079 }
3080
3081 static int parse_action_params(struct trace_array *tr, char *params,
3082                                struct action_data *data)
3083 {
3084         char *param, *saved_param;
3085         bool first_param = true;
3086         int ret = 0;
3087
3088         while (params) {
3089                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3090                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3091                         goto out;
3092                 }
3093
3094                 param = strsep(&params, ",");
3095                 if (!param) {
3096                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3097                         ret = -EINVAL;
3098                         goto out;
3099                 }
3100
3101                 param = strstrip(param);
3102                 if (strlen(param) < 2) {
3103                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3104                         ret = -EINVAL;
3105                         goto out;
3106                 }
3107
3108                 saved_param = kstrdup(param, GFP_KERNEL);
3109                 if (!saved_param) {
3110                         ret = -ENOMEM;
3111                         goto out;
3112                 }
3113
3114                 if (first_param && data->use_trace_keyword) {
3115                         data->synth_event_name = saved_param;
3116                         first_param = false;
3117                         continue;
3118                 }
3119                 first_param = false;
3120
3121                 data->params[data->n_params++] = saved_param;
3122         }
3123  out:
3124         return ret;
3125 }
3126
3127 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3128                         enum handler_id handler)
3129 {
3130         char *action_name;
3131         int ret = 0;
3132
3133         strsep(&str, ".");
3134         if (!str) {
3135                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3136                 ret = -EINVAL;
3137                 goto out;
3138         }
3139
3140         action_name = strsep(&str, "(");
3141         if (!action_name || !str) {
3142                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3143                 ret = -EINVAL;
3144                 goto out;
3145         }
3146
3147         if (str_has_prefix(action_name, "save")) {
3148                 char *params = strsep(&str, ")");
3149
3150                 if (!params) {
3151                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3152                         ret = -EINVAL;
3153                         goto out;
3154                 }
3155
3156                 ret = parse_action_params(tr, params, data);
3157                 if (ret)
3158                         goto out;
3159
3160                 if (handler == HANDLER_ONMAX)
3161                         data->track_data.check_val = check_track_val_max;
3162                 else if (handler == HANDLER_ONCHANGE)
3163                         data->track_data.check_val = check_track_val_changed;
3164                 else {
3165                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3166                         ret = -EINVAL;
3167                         goto out;
3168                 }
3169
3170                 data->track_data.save_data = save_track_data_vars;
3171                 data->fn = ontrack_action;
3172                 data->action = ACTION_SAVE;
3173         } else if (str_has_prefix(action_name, "snapshot")) {
3174                 char *params = strsep(&str, ")");
3175
3176                 if (!str) {
3177                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3178                         ret = -EINVAL;
3179                         goto out;
3180                 }
3181
3182                 if (handler == HANDLER_ONMAX)
3183                         data->track_data.check_val = check_track_val_max;
3184                 else if (handler == HANDLER_ONCHANGE)
3185                         data->track_data.check_val = check_track_val_changed;
3186                 else {
3187                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3188                         ret = -EINVAL;
3189                         goto out;
3190                 }
3191
3192                 data->track_data.save_data = save_track_data_snapshot;
3193                 data->fn = ontrack_action;
3194                 data->action = ACTION_SNAPSHOT;
3195         } else {
3196                 char *params = strsep(&str, ")");
3197
3198                 if (str_has_prefix(action_name, "trace"))
3199                         data->use_trace_keyword = true;
3200
3201                 if (params) {
3202                         ret = parse_action_params(tr, params, data);
3203                         if (ret)
3204                                 goto out;
3205                 }
3206
3207                 if (handler == HANDLER_ONMAX)
3208                         data->track_data.check_val = check_track_val_max;
3209                 else if (handler == HANDLER_ONCHANGE)
3210                         data->track_data.check_val = check_track_val_changed;
3211
3212                 if (handler != HANDLER_ONMATCH) {
3213                         data->track_data.save_data = action_trace;
3214                         data->fn = ontrack_action;
3215                 } else
3216                         data->fn = action_trace;
3217
3218                 data->action = ACTION_TRACE;
3219         }
3220
3221         data->action_name = kstrdup(action_name, GFP_KERNEL);
3222         if (!data->action_name) {
3223                 ret = -ENOMEM;
3224                 goto out;
3225         }
3226
3227         data->handler = handler;
3228  out:
3229         return ret;
3230 }
3231
3232 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3233                                             char *str, enum handler_id handler)
3234 {
3235         struct action_data *data;
3236         int ret = -EINVAL;
3237         char *var_str;
3238
3239         data = kzalloc(sizeof(*data), GFP_KERNEL);
3240         if (!data)
3241                 return ERR_PTR(-ENOMEM);
3242
3243         var_str = strsep(&str, ")");
3244         if (!var_str || !str) {
3245                 ret = -EINVAL;
3246                 goto free;
3247         }
3248
3249         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3250         if (!data->track_data.var_str) {
3251                 ret = -ENOMEM;
3252                 goto free;
3253         }
3254
3255         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3256         if (ret)
3257                 goto free;
3258  out:
3259         return data;
3260  free:
3261         track_data_destroy(hist_data, data);
3262         data = ERR_PTR(ret);
3263         goto out;
3264 }
3265
3266 static void onmatch_destroy(struct action_data *data)
3267 {
3268         kfree(data->match_data.event);
3269         kfree(data->match_data.event_system);
3270
3271         action_data_destroy(data);
3272 }
3273
3274 static void destroy_field_var(struct field_var *field_var)
3275 {
3276         if (!field_var)
3277                 return;
3278
3279         destroy_hist_field(field_var->var, 0);
3280         destroy_hist_field(field_var->val, 0);
3281
3282         kfree(field_var);
3283 }
3284
3285 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3286 {
3287         unsigned int i;
3288
3289         for (i = 0; i < hist_data->n_field_vars; i++)
3290                 destroy_field_var(hist_data->field_vars[i]);
3291
3292         for (i = 0; i < hist_data->n_save_vars; i++)
3293                 destroy_field_var(hist_data->save_vars[i]);
3294 }
3295
3296 static void save_field_var(struct hist_trigger_data *hist_data,
3297                            struct field_var *field_var)
3298 {
3299         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3300
3301         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3302                 hist_data->n_field_var_str++;
3303 }
3304
3305
3306 static int check_synth_field(struct synth_event *event,
3307                              struct hist_field *hist_field,
3308                              unsigned int field_pos)
3309 {
3310         struct synth_field *field;
3311
3312         if (field_pos >= event->n_fields)
3313                 return -EINVAL;
3314
3315         field = event->fields[field_pos];
3316
3317         /*
3318          * A dynamic string synth field can accept static or
3319          * dynamic. A static string synth field can only accept a
3320          * same-sized static string, which is checked for later.
3321          */
3322         if (strstr(hist_field->type, "char[") && field->is_string
3323             && field->is_dynamic)
3324                 return 0;
3325
3326         if (strcmp(field->type, hist_field->type) != 0) {
3327                 if (field->size != hist_field->size ||
3328                     field->is_signed != hist_field->is_signed)
3329                         return -EINVAL;
3330         }
3331
3332         return 0;
3333 }
3334
3335 static struct hist_field *
3336 trace_action_find_var(struct hist_trigger_data *hist_data,
3337                       struct action_data *data,
3338                       char *system, char *event, char *var)
3339 {
3340         struct trace_array *tr = hist_data->event_file->tr;
3341         struct hist_field *hist_field;
3342
3343         var++; /* skip '$' */
3344
3345         hist_field = find_target_event_var(hist_data, system, event, var);
3346         if (!hist_field) {
3347                 if (!system && data->handler == HANDLER_ONMATCH) {
3348                         system = data->match_data.event_system;
3349                         event = data->match_data.event;
3350                 }
3351
3352                 hist_field = find_event_var(hist_data, system, event, var);
3353         }
3354
3355         if (!hist_field)
3356                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3357
3358         return hist_field;
3359 }
3360
3361 static struct hist_field *
3362 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3363                               struct action_data *data, char *system,
3364                               char *event, char *var)
3365 {
3366         struct hist_field *hist_field = NULL;
3367         struct field_var *field_var;
3368
3369         /*
3370          * First try to create a field var on the target event (the
3371          * currently being defined).  This will create a variable for
3372          * unqualified fields on the target event, or if qualified,
3373          * target fields that have qualified names matching the target.
3374          */
3375         field_var = create_target_field_var(hist_data, system, event, var);
3376
3377         if (field_var && !IS_ERR(field_var)) {
3378                 save_field_var(hist_data, field_var);
3379                 hist_field = field_var->var;
3380         } else {
3381                 field_var = NULL;
3382                 /*
3383                  * If no explicit system.event is specified, default to
3384                  * looking for fields on the onmatch(system.event.xxx)
3385                  * event.
3386                  */
3387                 if (!system && data->handler == HANDLER_ONMATCH) {
3388                         system = data->match_data.event_system;
3389                         event = data->match_data.event;
3390                 }
3391
3392                 /*
3393                  * At this point, we're looking at a field on another
3394                  * event.  Because we can't modify a hist trigger on
3395                  * another event to add a variable for a field, we need
3396                  * to create a new trigger on that event and create the
3397                  * variable at the same time.
3398                  */
3399                 hist_field = create_field_var_hist(hist_data, system, event, var);
3400                 if (IS_ERR(hist_field))
3401                         goto free;
3402         }
3403  out:
3404         return hist_field;
3405  free:
3406         destroy_field_var(field_var);
3407         hist_field = NULL;
3408         goto out;
3409 }
3410
3411 static int trace_action_create(struct hist_trigger_data *hist_data,
3412                                struct action_data *data)
3413 {
3414         struct trace_array *tr = hist_data->event_file->tr;
3415         char *event_name, *param, *system = NULL;
3416         struct hist_field *hist_field, *var_ref;
3417         unsigned int i;
3418         unsigned int field_pos = 0;
3419         struct synth_event *event;
3420         char *synth_event_name;
3421         int var_ref_idx, ret = 0;
3422
3423         lockdep_assert_held(&event_mutex);
3424
3425         if (data->use_trace_keyword)
3426                 synth_event_name = data->synth_event_name;
3427         else
3428                 synth_event_name = data->action_name;
3429
3430         event = find_synth_event(synth_event_name);
3431         if (!event) {
3432                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3433                 return -EINVAL;
3434         }
3435
3436         event->ref++;
3437
3438         for (i = 0; i < data->n_params; i++) {
3439                 char *p;
3440
3441                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3442                 if (!param) {
3443                         ret = -ENOMEM;
3444                         goto err;
3445                 }
3446
3447                 system = strsep(&param, ".");
3448                 if (!param) {
3449                         param = (char *)system;
3450                         system = event_name = NULL;
3451                 } else {
3452                         event_name = strsep(&param, ".");
3453                         if (!param) {
3454                                 kfree(p);
3455                                 ret = -EINVAL;
3456                                 goto err;
3457                         }
3458                 }
3459
3460                 if (param[0] == '$')
3461                         hist_field = trace_action_find_var(hist_data, data,
3462                                                            system, event_name,
3463                                                            param);
3464                 else
3465                         hist_field = trace_action_create_field_var(hist_data,
3466                                                                    data,
3467                                                                    system,
3468                                                                    event_name,
3469                                                                    param);
3470
3471                 if (!hist_field) {
3472                         kfree(p);
3473                         ret = -EINVAL;
3474                         goto err;
3475                 }
3476
3477                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3478                         var_ref = create_var_ref(hist_data, hist_field,
3479                                                  system, event_name);
3480                         if (!var_ref) {
3481                                 kfree(p);
3482                                 ret = -ENOMEM;
3483                                 goto err;
3484                         }
3485
3486                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3487                         if (WARN_ON(var_ref_idx < 0)) {
3488                                 ret = var_ref_idx;
3489                                 goto err;
3490                         }
3491
3492                         data->var_ref_idx[i] = var_ref_idx;
3493
3494                         field_pos++;
3495                         kfree(p);
3496                         continue;
3497                 }
3498
3499                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3500                 kfree(p);
3501                 ret = -EINVAL;
3502                 goto err;
3503         }
3504
3505         if (field_pos != event->n_fields) {
3506                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3507                 ret = -EINVAL;
3508                 goto err;
3509         }
3510
3511         data->synth_event = event;
3512  out:
3513         return ret;
3514  err:
3515         event->ref--;
3516
3517         goto out;
3518 }
3519
3520 static int action_create(struct hist_trigger_data *hist_data,
3521                          struct action_data *data)
3522 {
3523         struct trace_event_file *file = hist_data->event_file;
3524         struct trace_array *tr = file->tr;
3525         struct track_data *track_data;
3526         struct field_var *field_var;
3527         unsigned int i;
3528         char *param;
3529         int ret = 0;
3530
3531         if (data->action == ACTION_TRACE)
3532                 return trace_action_create(hist_data, data);
3533
3534         if (data->action == ACTION_SNAPSHOT) {
3535                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3536                 if (IS_ERR(track_data)) {
3537                         ret = PTR_ERR(track_data);
3538                         goto out;
3539                 }
3540
3541                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3542                                                    cond_snapshot_update);
3543                 if (ret)
3544                         track_data_free(track_data);
3545
3546                 goto out;
3547         }
3548
3549         if (data->action == ACTION_SAVE) {
3550                 if (hist_data->n_save_vars) {
3551                         ret = -EEXIST;
3552                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3553                         goto out;
3554                 }
3555
3556                 for (i = 0; i < data->n_params; i++) {
3557                         param = kstrdup(data->params[i], GFP_KERNEL);
3558                         if (!param) {
3559                                 ret = -ENOMEM;
3560                                 goto out;
3561                         }
3562
3563                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3564                         if (IS_ERR(field_var)) {
3565                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3566                                          errpos(param));
3567                                 ret = PTR_ERR(field_var);
3568                                 kfree(param);
3569                                 goto out;
3570                         }
3571
3572                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3573                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3574                                 hist_data->n_save_var_str++;
3575                         kfree(param);
3576                 }
3577         }
3578  out:
3579         return ret;
3580 }
3581
3582 static int onmatch_create(struct hist_trigger_data *hist_data,
3583                           struct action_data *data)
3584 {
3585         return action_create(hist_data, data);
3586 }
3587
3588 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3589 {
3590         char *match_event, *match_event_system;
3591         struct action_data *data;
3592         int ret = -EINVAL;
3593
3594         data = kzalloc(sizeof(*data), GFP_KERNEL);
3595         if (!data)
3596                 return ERR_PTR(-ENOMEM);
3597
3598         match_event = strsep(&str, ")");
3599         if (!match_event || !str) {
3600                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3601                 goto free;
3602         }
3603
3604         match_event_system = strsep(&match_event, ".");
3605         if (!match_event) {
3606                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3607                 goto free;
3608         }
3609
3610         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3611                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3612                 goto free;
3613         }
3614
3615         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3616         if (!data->match_data.event) {
3617                 ret = -ENOMEM;
3618                 goto free;
3619         }
3620
3621         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3622         if (!data->match_data.event_system) {
3623                 ret = -ENOMEM;
3624                 goto free;
3625         }
3626
3627         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3628         if (ret)
3629                 goto free;
3630  out:
3631         return data;
3632  free:
3633         onmatch_destroy(data);
3634         data = ERR_PTR(ret);
3635         goto out;
3636 }
3637
3638 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3639 {
3640         hist_data->fields[HITCOUNT_IDX] =
3641                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3642         if (!hist_data->fields[HITCOUNT_IDX])
3643                 return -ENOMEM;
3644
3645         hist_data->n_vals++;
3646         hist_data->n_fields++;
3647
3648         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3649                 return -EINVAL;
3650
3651         return 0;
3652 }
3653
3654 static int __create_val_field(struct hist_trigger_data *hist_data,
3655                               unsigned int val_idx,
3656                               struct trace_event_file *file,
3657                               char *var_name, char *field_str,
3658                               unsigned long flags)
3659 {
3660         struct hist_field *hist_field;
3661         int ret = 0;
3662
3663         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3664         if (IS_ERR(hist_field)) {
3665                 ret = PTR_ERR(hist_field);
3666                 goto out;
3667         }
3668
3669         hist_data->fields[val_idx] = hist_field;
3670
3671         ++hist_data->n_vals;
3672         ++hist_data->n_fields;
3673
3674         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3675                 ret = -EINVAL;
3676  out:
3677         return ret;
3678 }
3679
3680 static int create_val_field(struct hist_trigger_data *hist_data,
3681                             unsigned int val_idx,
3682                             struct trace_event_file *file,
3683                             char *field_str)
3684 {
3685         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3686                 return -EINVAL;
3687
3688         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3689 }
3690
3691 static int create_var_field(struct hist_trigger_data *hist_data,
3692                             unsigned int val_idx,
3693                             struct trace_event_file *file,
3694                             char *var_name, char *expr_str)
3695 {
3696         struct trace_array *tr = hist_data->event_file->tr;
3697         unsigned long flags = 0;
3698         int ret;
3699
3700         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3701                 return -EINVAL;
3702
3703         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3704                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3705                 return -EINVAL;
3706         }
3707
3708         flags |= HIST_FIELD_FL_VAR;
3709         hist_data->n_vars++;
3710         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3711                 return -EINVAL;
3712
3713         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3714
3715         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3716                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3717
3718         return ret;
3719 }
3720
3721 static int create_val_fields(struct hist_trigger_data *hist_data,
3722                              struct trace_event_file *file)
3723 {
3724         char *fields_str, *field_str;
3725         unsigned int i, j = 1;
3726         int ret;
3727
3728         ret = create_hitcount_val(hist_data);
3729         if (ret)
3730                 goto out;
3731
3732         fields_str = hist_data->attrs->vals_str;
3733         if (!fields_str)
3734                 goto out;
3735
3736         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3737                      j < TRACING_MAP_VALS_MAX; i++) {
3738                 field_str = strsep(&fields_str, ",");
3739                 if (!field_str)
3740                         break;
3741
3742                 if (strcmp(field_str, "hitcount") == 0)
3743                         continue;
3744
3745                 ret = create_val_field(hist_data, j++, file, field_str);
3746                 if (ret)
3747                         goto out;
3748         }
3749
3750         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3751                 ret = -EINVAL;
3752  out:
3753         return ret;
3754 }
3755
3756 static int create_key_field(struct hist_trigger_data *hist_data,
3757                             unsigned int key_idx,
3758                             unsigned int key_offset,
3759                             struct trace_event_file *file,
3760                             char *field_str)
3761 {
3762         struct trace_array *tr = hist_data->event_file->tr;
3763         struct hist_field *hist_field = NULL;
3764         unsigned long flags = 0;
3765         unsigned int key_size;
3766         int ret = 0;
3767
3768         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3769                 return -EINVAL;
3770
3771         flags |= HIST_FIELD_FL_KEY;
3772
3773         if (strcmp(field_str, "stacktrace") == 0) {
3774                 flags |= HIST_FIELD_FL_STACKTRACE;
3775                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3776                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3777         } else {
3778                 hist_field = parse_expr(hist_data, file, field_str, flags,
3779                                         NULL, 0);
3780                 if (IS_ERR(hist_field)) {
3781                         ret = PTR_ERR(hist_field);
3782                         goto out;
3783                 }
3784
3785                 if (field_has_hist_vars(hist_field, 0)) {
3786                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3787                         destroy_hist_field(hist_field, 0);
3788                         ret = -EINVAL;
3789                         goto out;
3790                 }
3791
3792                 key_size = hist_field->size;
3793         }
3794
3795         hist_data->fields[key_idx] = hist_field;
3796
3797         key_size = ALIGN(key_size, sizeof(u64));
3798         hist_data->fields[key_idx]->size = key_size;
3799         hist_data->fields[key_idx]->offset = key_offset;
3800
3801         hist_data->key_size += key_size;
3802
3803         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3804                 ret = -EINVAL;
3805                 goto out;
3806         }
3807
3808         hist_data->n_keys++;
3809         hist_data->n_fields++;
3810
3811         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3812                 return -EINVAL;
3813
3814         ret = key_size;
3815  out:
3816         return ret;
3817 }
3818
3819 static int create_key_fields(struct hist_trigger_data *hist_data,
3820                              struct trace_event_file *file)
3821 {
3822         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3823         char *fields_str, *field_str;
3824         int ret = -EINVAL;
3825
3826         fields_str = hist_data->attrs->keys_str;
3827         if (!fields_str)
3828                 goto out;
3829
3830         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3831                 field_str = strsep(&fields_str, ",");
3832                 if (!field_str)
3833                         break;
3834                 ret = create_key_field(hist_data, i, key_offset,
3835                                        file, field_str);
3836                 if (ret < 0)
3837                         goto out;
3838                 key_offset += ret;
3839         }
3840         if (fields_str) {
3841                 ret = -EINVAL;
3842                 goto out;
3843         }
3844         ret = 0;
3845  out:
3846         return ret;
3847 }
3848
3849 static int create_var_fields(struct hist_trigger_data *hist_data,
3850                              struct trace_event_file *file)
3851 {
3852         unsigned int i, j = hist_data->n_vals;
3853         int ret = 0;
3854
3855         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3856
3857         for (i = 0; i < n_vars; i++) {
3858                 char *var_name = hist_data->attrs->var_defs.name[i];
3859                 char *expr = hist_data->attrs->var_defs.expr[i];
3860
3861                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3862                 if (ret)
3863                         goto out;
3864         }
3865  out:
3866         return ret;
3867 }
3868
3869 static void free_var_defs(struct hist_trigger_data *hist_data)
3870 {
3871         unsigned int i;
3872
3873         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3874                 kfree(hist_data->attrs->var_defs.name[i]);
3875                 kfree(hist_data->attrs->var_defs.expr[i]);
3876         }
3877
3878         hist_data->attrs->var_defs.n_vars = 0;
3879 }
3880
3881 static int parse_var_defs(struct hist_trigger_data *hist_data)
3882 {
3883         struct trace_array *tr = hist_data->event_file->tr;
3884         char *s, *str, *var_name, *field_str;
3885         unsigned int i, j, n_vars = 0;
3886         int ret = 0;
3887
3888         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3889                 str = hist_data->attrs->assignment_str[i];
3890                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3891                         field_str = strsep(&str, ",");
3892                         if (!field_str)
3893                                 break;
3894
3895                         var_name = strsep(&field_str, "=");
3896                         if (!var_name || !field_str) {
3897                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3898                                          errpos(var_name));
3899                                 ret = -EINVAL;
3900                                 goto free;
3901                         }
3902
3903                         if (n_vars == TRACING_MAP_VARS_MAX) {
3904                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3905                                 ret = -EINVAL;
3906                                 goto free;
3907                         }
3908
3909                         s = kstrdup(var_name, GFP_KERNEL);
3910                         if (!s) {
3911                                 ret = -ENOMEM;
3912                                 goto free;
3913                         }
3914                         hist_data->attrs->var_defs.name[n_vars] = s;
3915
3916                         s = kstrdup(field_str, GFP_KERNEL);
3917                         if (!s) {
3918                                 ret = -ENOMEM;
3919                                 goto free;
3920                         }
3921                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3922
3923                         hist_data->attrs->var_defs.n_vars = n_vars;
3924                 }
3925         }
3926
3927         return ret;
3928  free:
3929         free_var_defs(hist_data);
3930
3931         return ret;
3932 }
3933
3934 static int create_hist_fields(struct hist_trigger_data *hist_data,
3935                               struct trace_event_file *file)
3936 {
3937         int ret;
3938
3939         ret = parse_var_defs(hist_data);
3940         if (ret)
3941                 goto out;
3942
3943         ret = create_val_fields(hist_data, file);
3944         if (ret)
3945                 goto out;
3946
3947         ret = create_var_fields(hist_data, file);
3948         if (ret)
3949                 goto out;
3950
3951         ret = create_key_fields(hist_data, file);
3952         if (ret)
3953                 goto out;
3954  out:
3955         free_var_defs(hist_data);
3956
3957         return ret;
3958 }
3959
3960 static int is_descending(struct trace_array *tr, const char *str)
3961 {
3962         if (!str)
3963                 return 0;
3964
3965         if (strcmp(str, "descending") == 0)
3966                 return 1;
3967
3968         if (strcmp(str, "ascending") == 0)
3969                 return 0;
3970
3971         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3972
3973         return -EINVAL;
3974 }
3975
3976 static int create_sort_keys(struct hist_trigger_data *hist_data)
3977 {
3978         struct trace_array *tr = hist_data->event_file->tr;
3979         char *fields_str = hist_data->attrs->sort_key_str;
3980         struct tracing_map_sort_key *sort_key;
3981         int descending, ret = 0;
3982         unsigned int i, j, k;
3983
3984         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3985
3986         if (!fields_str)
3987                 goto out;
3988
3989         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3990                 struct hist_field *hist_field;
3991                 char *field_str, *field_name;
3992                 const char *test_name;
3993
3994                 sort_key = &hist_data->sort_keys[i];
3995
3996                 field_str = strsep(&fields_str, ",");
3997                 if (!field_str)
3998                         break;
3999
4000                 if (!*field_str) {
4001                         ret = -EINVAL;
4002                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4003                         break;
4004                 }
4005
4006                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4007                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4008                         ret = -EINVAL;
4009                         break;
4010                 }
4011
4012                 field_name = strsep(&field_str, ".");
4013                 if (!field_name || !*field_name) {
4014                         ret = -EINVAL;
4015                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4016                         break;
4017                 }
4018
4019                 if (strcmp(field_name, "hitcount") == 0) {
4020                         descending = is_descending(tr, field_str);
4021                         if (descending < 0) {
4022                                 ret = descending;
4023                                 break;
4024                         }
4025                         sort_key->descending = descending;
4026                         continue;
4027                 }
4028
4029                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4030                         unsigned int idx;
4031
4032                         hist_field = hist_data->fields[j];
4033                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4034                                 continue;
4035
4036                         idx = k++;
4037
4038                         test_name = hist_field_name(hist_field, 0);
4039
4040                         if (strcmp(field_name, test_name) == 0) {
4041                                 sort_key->field_idx = idx;
4042                                 descending = is_descending(tr, field_str);
4043                                 if (descending < 0) {
4044                                         ret = descending;
4045                                         goto out;
4046                                 }
4047                                 sort_key->descending = descending;
4048                                 break;
4049                         }
4050                 }
4051                 if (j == hist_data->n_fields) {
4052                         ret = -EINVAL;
4053                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4054                         break;
4055                 }
4056         }
4057
4058         hist_data->n_sort_keys = i;
4059  out:
4060         return ret;
4061 }
4062
4063 static void destroy_actions(struct hist_trigger_data *hist_data)
4064 {
4065         unsigned int i;
4066
4067         for (i = 0; i < hist_data->n_actions; i++) {
4068                 struct action_data *data = hist_data->actions[i];
4069
4070                 if (data->handler == HANDLER_ONMATCH)
4071                         onmatch_destroy(data);
4072                 else if (data->handler == HANDLER_ONMAX ||
4073                          data->handler == HANDLER_ONCHANGE)
4074                         track_data_destroy(hist_data, data);
4075                 else
4076                         kfree(data);
4077         }
4078 }
4079
4080 static int parse_actions(struct hist_trigger_data *hist_data)
4081 {
4082         struct trace_array *tr = hist_data->event_file->tr;
4083         struct action_data *data;
4084         unsigned int i;
4085         int ret = 0;
4086         char *str;
4087         int len;
4088
4089         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4090                 str = hist_data->attrs->action_str[i];
4091
4092                 if ((len = str_has_prefix(str, "onmatch("))) {
4093                         char *action_str = str + len;
4094
4095                         data = onmatch_parse(tr, action_str);
4096                         if (IS_ERR(data)) {
4097                                 ret = PTR_ERR(data);
4098                                 break;
4099                         }
4100                 } else if ((len = str_has_prefix(str, "onmax("))) {
4101                         char *action_str = str + len;
4102
4103                         data = track_data_parse(hist_data, action_str,
4104                                                 HANDLER_ONMAX);
4105                         if (IS_ERR(data)) {
4106                                 ret = PTR_ERR(data);
4107                                 break;
4108                         }
4109                 } else if ((len = str_has_prefix(str, "onchange("))) {
4110                         char *action_str = str + len;
4111
4112                         data = track_data_parse(hist_data, action_str,
4113                                                 HANDLER_ONCHANGE);
4114                         if (IS_ERR(data)) {
4115                                 ret = PTR_ERR(data);
4116                                 break;
4117                         }
4118                 } else {
4119                         ret = -EINVAL;
4120                         break;
4121                 }
4122
4123                 hist_data->actions[hist_data->n_actions++] = data;
4124         }
4125
4126         return ret;
4127 }
4128
4129 static int create_actions(struct hist_trigger_data *hist_data)
4130 {
4131         struct action_data *data;
4132         unsigned int i;
4133         int ret = 0;
4134
4135         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4136                 data = hist_data->actions[i];
4137
4138                 if (data->handler == HANDLER_ONMATCH) {
4139                         ret = onmatch_create(hist_data, data);
4140                         if (ret)
4141                                 break;
4142                 } else if (data->handler == HANDLER_ONMAX ||
4143                            data->handler == HANDLER_ONCHANGE) {
4144                         ret = track_data_create(hist_data, data);
4145                         if (ret)
4146                                 break;
4147                 } else {
4148                         ret = -EINVAL;
4149                         break;
4150                 }
4151         }
4152
4153         return ret;
4154 }
4155
4156 static void print_actions(struct seq_file *m,
4157                           struct hist_trigger_data *hist_data,
4158                           struct tracing_map_elt *elt)
4159 {
4160         unsigned int i;
4161
4162         for (i = 0; i < hist_data->n_actions; i++) {
4163                 struct action_data *data = hist_data->actions[i];
4164
4165                 if (data->action == ACTION_SNAPSHOT)
4166                         continue;
4167
4168                 if (data->handler == HANDLER_ONMAX ||
4169                     data->handler == HANDLER_ONCHANGE)
4170                         track_data_print(m, hist_data, elt, data);
4171         }
4172 }
4173
4174 static void print_action_spec(struct seq_file *m,
4175                               struct hist_trigger_data *hist_data,
4176                               struct action_data *data)
4177 {
4178         unsigned int i;
4179
4180         if (data->action == ACTION_SAVE) {
4181                 for (i = 0; i < hist_data->n_save_vars; i++) {
4182                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4183                         if (i < hist_data->n_save_vars - 1)
4184                                 seq_puts(m, ",");
4185                 }
4186         } else if (data->action == ACTION_TRACE) {
4187                 if (data->use_trace_keyword)
4188                         seq_printf(m, "%s", data->synth_event_name);
4189                 for (i = 0; i < data->n_params; i++) {
4190                         if (i || data->use_trace_keyword)
4191                                 seq_puts(m, ",");
4192                         seq_printf(m, "%s", data->params[i]);
4193                 }
4194         }
4195 }
4196
4197 static void print_track_data_spec(struct seq_file *m,
4198                                   struct hist_trigger_data *hist_data,
4199                                   struct action_data *data)
4200 {
4201         if (data->handler == HANDLER_ONMAX)
4202                 seq_puts(m, ":onmax(");
4203         else if (data->handler == HANDLER_ONCHANGE)
4204                 seq_puts(m, ":onchange(");
4205         seq_printf(m, "%s", data->track_data.var_str);
4206         seq_printf(m, ").%s(", data->action_name);
4207
4208         print_action_spec(m, hist_data, data);
4209
4210         seq_puts(m, ")");
4211 }
4212
4213 static void print_onmatch_spec(struct seq_file *m,
4214                                struct hist_trigger_data *hist_data,
4215                                struct action_data *data)
4216 {
4217         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4218                    data->match_data.event);
4219
4220         seq_printf(m, "%s(", data->action_name);
4221
4222         print_action_spec(m, hist_data, data);
4223
4224         seq_puts(m, ")");
4225 }
4226
4227 static bool actions_match(struct hist_trigger_data *hist_data,
4228                           struct hist_trigger_data *hist_data_test)
4229 {
4230         unsigned int i, j;
4231
4232         if (hist_data->n_actions != hist_data_test->n_actions)
4233                 return false;
4234
4235         for (i = 0; i < hist_data->n_actions; i++) {
4236                 struct action_data *data = hist_data->actions[i];
4237                 struct action_data *data_test = hist_data_test->actions[i];
4238                 char *action_name, *action_name_test;
4239
4240                 if (data->handler != data_test->handler)
4241                         return false;
4242                 if (data->action != data_test->action)
4243                         return false;
4244
4245                 if (data->n_params != data_test->n_params)
4246                         return false;
4247
4248                 for (j = 0; j < data->n_params; j++) {
4249                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4250                                 return false;
4251                 }
4252
4253                 if (data->use_trace_keyword)
4254                         action_name = data->synth_event_name;
4255                 else
4256                         action_name = data->action_name;
4257
4258                 if (data_test->use_trace_keyword)
4259                         action_name_test = data_test->synth_event_name;
4260                 else
4261                         action_name_test = data_test->action_name;
4262
4263                 if (strcmp(action_name, action_name_test) != 0)
4264                         return false;
4265
4266                 if (data->handler == HANDLER_ONMATCH) {
4267                         if (strcmp(data->match_data.event_system,
4268                                    data_test->match_data.event_system) != 0)
4269                                 return false;
4270                         if (strcmp(data->match_data.event,
4271                                    data_test->match_data.event) != 0)
4272                                 return false;
4273                 } else if (data->handler == HANDLER_ONMAX ||
4274                            data->handler == HANDLER_ONCHANGE) {
4275                         if (strcmp(data->track_data.var_str,
4276                                    data_test->track_data.var_str) != 0)
4277                                 return false;
4278                 }
4279         }
4280
4281         return true;
4282 }
4283
4284
4285 static void print_actions_spec(struct seq_file *m,
4286                                struct hist_trigger_data *hist_data)
4287 {
4288         unsigned int i;
4289
4290         for (i = 0; i < hist_data->n_actions; i++) {
4291                 struct action_data *data = hist_data->actions[i];
4292
4293                 if (data->handler == HANDLER_ONMATCH)
4294                         print_onmatch_spec(m, hist_data, data);
4295                 else if (data->handler == HANDLER_ONMAX ||
4296                          data->handler == HANDLER_ONCHANGE)
4297                         print_track_data_spec(m, hist_data, data);
4298         }
4299 }
4300
4301 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4302 {
4303         unsigned int i;
4304
4305         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4306                 kfree(hist_data->field_var_hists[i]->cmd);
4307                 kfree(hist_data->field_var_hists[i]);
4308         }
4309 }
4310
4311 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4312 {
4313         if (!hist_data)
4314                 return;
4315
4316         destroy_hist_trigger_attrs(hist_data->attrs);
4317         destroy_hist_fields(hist_data);
4318         tracing_map_destroy(hist_data->map);
4319
4320         destroy_actions(hist_data);
4321         destroy_field_vars(hist_data);
4322         destroy_field_var_hists(hist_data);
4323
4324         kfree(hist_data);
4325 }
4326
4327 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4328 {
4329         struct tracing_map *map = hist_data->map;
4330         struct ftrace_event_field *field;
4331         struct hist_field *hist_field;
4332         int i, idx = 0;
4333
4334         for_each_hist_field(i, hist_data) {
4335                 hist_field = hist_data->fields[i];
4336                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4337                         tracing_map_cmp_fn_t cmp_fn;
4338
4339                         field = hist_field->field;
4340
4341                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4342                                 cmp_fn = tracing_map_cmp_none;
4343                         else if (!field)
4344                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4345                                                              hist_field->is_signed);
4346                         else if (is_string_field(field))
4347                                 cmp_fn = tracing_map_cmp_string;
4348                         else
4349                                 cmp_fn = tracing_map_cmp_num(field->size,
4350                                                              field->is_signed);
4351                         idx = tracing_map_add_key_field(map,
4352                                                         hist_field->offset,
4353                                                         cmp_fn);
4354                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4355                         idx = tracing_map_add_sum_field(map);
4356
4357                 if (idx < 0)
4358                         return idx;
4359
4360                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4361                         idx = tracing_map_add_var(map);
4362                         if (idx < 0)
4363                                 return idx;
4364                         hist_field->var.idx = idx;
4365                         hist_field->var.hist_data = hist_data;
4366                 }
4367         }
4368
4369         return 0;
4370 }
4371
4372 static struct hist_trigger_data *
4373 create_hist_data(unsigned int map_bits,
4374                  struct hist_trigger_attrs *attrs,
4375                  struct trace_event_file *file,
4376                  bool remove)
4377 {
4378         const struct tracing_map_ops *map_ops = NULL;
4379         struct hist_trigger_data *hist_data;
4380         int ret = 0;
4381
4382         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4383         if (!hist_data)
4384                 return ERR_PTR(-ENOMEM);
4385
4386         hist_data->attrs = attrs;
4387         hist_data->remove = remove;
4388         hist_data->event_file = file;
4389
4390         ret = parse_actions(hist_data);
4391         if (ret)
4392                 goto free;
4393
4394         ret = create_hist_fields(hist_data, file);
4395         if (ret)
4396                 goto free;
4397
4398         ret = create_sort_keys(hist_data);
4399         if (ret)
4400                 goto free;
4401
4402         map_ops = &hist_trigger_elt_data_ops;
4403
4404         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4405                                             map_ops, hist_data);
4406         if (IS_ERR(hist_data->map)) {
4407                 ret = PTR_ERR(hist_data->map);
4408                 hist_data->map = NULL;
4409                 goto free;
4410         }
4411
4412         ret = create_tracing_map_fields(hist_data);
4413         if (ret)
4414                 goto free;
4415  out:
4416         return hist_data;
4417  free:
4418         hist_data->attrs = NULL;
4419
4420         destroy_hist_data(hist_data);
4421
4422         hist_data = ERR_PTR(ret);
4423
4424         goto out;
4425 }
4426
4427 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4428                                     struct tracing_map_elt *elt,
4429                                     struct trace_buffer *buffer, void *rec,
4430                                     struct ring_buffer_event *rbe,
4431                                     u64 *var_ref_vals)
4432 {
4433         struct hist_elt_data *elt_data;
4434         struct hist_field *hist_field;
4435         unsigned int i, var_idx;
4436         u64 hist_val;
4437
4438         elt_data = elt->private_data;
4439         elt_data->var_ref_vals = var_ref_vals;
4440
4441         for_each_hist_val_field(i, hist_data) {
4442                 hist_field = hist_data->fields[i];
4443                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4444                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4445                         var_idx = hist_field->var.idx;
4446
4447                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4448                                 unsigned int str_start, var_str_idx, idx;
4449                                 char *str, *val_str;
4450
4451                                 str_start = hist_data->n_field_var_str +
4452                                         hist_data->n_save_var_str;
4453                                 var_str_idx = hist_field->var_str_idx;
4454                                 idx = str_start + var_str_idx;
4455
4456                                 str = elt_data->field_var_str[idx];
4457                                 val_str = (char *)(uintptr_t)hist_val;
4458                                 strscpy(str, val_str, STR_VAR_LEN_MAX);
4459
4460                                 hist_val = (u64)(uintptr_t)str;
4461                         }
4462                         tracing_map_set_var(elt, var_idx, hist_val);
4463                         continue;
4464                 }
4465                 tracing_map_update_sum(elt, i, hist_val);
4466         }
4467
4468         for_each_hist_key_field(i, hist_data) {
4469                 hist_field = hist_data->fields[i];
4470                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4471                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4472                         var_idx = hist_field->var.idx;
4473                         tracing_map_set_var(elt, var_idx, hist_val);
4474                 }
4475         }
4476
4477         update_field_vars(hist_data, elt, buffer, rbe, rec);
4478 }
4479
4480 static inline void add_to_key(char *compound_key, void *key,
4481                               struct hist_field *key_field, void *rec)
4482 {
4483         size_t size = key_field->size;
4484
4485         if (key_field->flags & HIST_FIELD_FL_STRING) {
4486                 struct ftrace_event_field *field;
4487
4488                 field = key_field->field;
4489                 if (field->filter_type == FILTER_DYN_STRING)
4490                         size = *(u32 *)(rec + field->offset) >> 16;
4491                 else if (field->filter_type == FILTER_PTR_STRING)
4492                         size = strlen(key);
4493                 else if (field->filter_type == FILTER_STATIC_STRING)
4494                         size = field->size;
4495
4496                 /* ensure NULL-termination */
4497                 if (size > key_field->size - 1)
4498                         size = key_field->size - 1;
4499
4500                 strncpy(compound_key + key_field->offset, (char *)key, size);
4501         } else
4502                 memcpy(compound_key + key_field->offset, key, size);
4503 }
4504
4505 static void
4506 hist_trigger_actions(struct hist_trigger_data *hist_data,
4507                      struct tracing_map_elt *elt,
4508                      struct trace_buffer *buffer, void *rec,
4509                      struct ring_buffer_event *rbe, void *key,
4510                      u64 *var_ref_vals)
4511 {
4512         struct action_data *data;
4513         unsigned int i;
4514
4515         for (i = 0; i < hist_data->n_actions; i++) {
4516                 data = hist_data->actions[i];
4517                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4518         }
4519 }
4520
4521 static void event_hist_trigger(struct event_trigger_data *data,
4522                                struct trace_buffer *buffer, void *rec,
4523                                struct ring_buffer_event *rbe)
4524 {
4525         struct hist_trigger_data *hist_data = data->private_data;
4526         bool use_compound_key = (hist_data->n_keys > 1);
4527         unsigned long entries[HIST_STACKTRACE_DEPTH];
4528         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4529         char compound_key[HIST_KEY_SIZE_MAX];
4530         struct tracing_map_elt *elt = NULL;
4531         struct hist_field *key_field;
4532         u64 field_contents;
4533         void *key = NULL;
4534         unsigned int i;
4535
4536         memset(compound_key, 0, hist_data->key_size);
4537
4538         for_each_hist_key_field(i, hist_data) {
4539                 key_field = hist_data->fields[i];
4540
4541                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4542                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4543                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4544                                          HIST_STACKTRACE_SKIP);
4545                         key = entries;
4546                 } else {
4547                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4548                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4549                                 key = (void *)(unsigned long)field_contents;
4550                                 use_compound_key = true;
4551                         } else
4552                                 key = (void *)&field_contents;
4553                 }
4554
4555                 if (use_compound_key)
4556                         add_to_key(compound_key, key, key_field, rec);
4557         }
4558
4559         if (use_compound_key)
4560                 key = compound_key;
4561
4562         if (hist_data->n_var_refs &&
4563             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4564                 return;
4565
4566         elt = tracing_map_insert(hist_data->map, key);
4567         if (!elt)
4568                 return;
4569
4570         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4571
4572         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4573                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4574 }
4575
4576 static void hist_trigger_stacktrace_print(struct seq_file *m,
4577                                           unsigned long *stacktrace_entries,
4578                                           unsigned int max_entries)
4579 {
4580         char str[KSYM_SYMBOL_LEN];
4581         unsigned int spaces = 8;
4582         unsigned int i;
4583
4584         for (i = 0; i < max_entries; i++) {
4585                 if (!stacktrace_entries[i])
4586                         return;
4587
4588                 seq_printf(m, "%*c", 1 + spaces, ' ');
4589                 sprint_symbol(str, stacktrace_entries[i]);
4590                 seq_printf(m, "%s\n", str);
4591         }
4592 }
4593
4594 static void hist_trigger_print_key(struct seq_file *m,
4595                                    struct hist_trigger_data *hist_data,
4596                                    void *key,
4597                                    struct tracing_map_elt *elt)
4598 {
4599         struct hist_field *key_field;
4600         char str[KSYM_SYMBOL_LEN];
4601         bool multiline = false;
4602         const char *field_name;
4603         unsigned int i;
4604         u64 uval;
4605
4606         seq_puts(m, "{ ");
4607
4608         for_each_hist_key_field(i, hist_data) {
4609                 key_field = hist_data->fields[i];
4610
4611                 if (i > hist_data->n_vals)
4612                         seq_puts(m, ", ");
4613
4614                 field_name = hist_field_name(key_field, 0);
4615
4616                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4617                         uval = *(u64 *)(key + key_field->offset);
4618                         seq_printf(m, "%s: %llx", field_name, uval);
4619                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4620                         uval = *(u64 *)(key + key_field->offset);
4621                         sprint_symbol_no_offset(str, uval);
4622                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4623                                    uval, str);
4624                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4625                         uval = *(u64 *)(key + key_field->offset);
4626                         sprint_symbol(str, uval);
4627                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4628                                    uval, str);
4629                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4630                         struct hist_elt_data *elt_data = elt->private_data;
4631                         char *comm;
4632
4633                         if (WARN_ON_ONCE(!elt_data))
4634                                 return;
4635
4636                         comm = elt_data->comm;
4637
4638                         uval = *(u64 *)(key + key_field->offset);
4639                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4640                                    comm, uval);
4641                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4642                         const char *syscall_name;
4643
4644                         uval = *(u64 *)(key + key_field->offset);
4645                         syscall_name = get_syscall_name(uval);
4646                         if (!syscall_name)
4647                                 syscall_name = "unknown_syscall";
4648
4649                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4650                                    syscall_name, uval);
4651                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4652                         seq_puts(m, "stacktrace:\n");
4653                         hist_trigger_stacktrace_print(m,
4654                                                       key + key_field->offset,
4655                                                       HIST_STACKTRACE_DEPTH);
4656                         multiline = true;
4657                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4658                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4659                                    *(u64 *)(key + key_field->offset));
4660                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4661                         seq_printf(m, "%s: %-50s", field_name,
4662                                    (char *)(key + key_field->offset));
4663                 } else {
4664                         uval = *(u64 *)(key + key_field->offset);
4665                         seq_printf(m, "%s: %10llu", field_name, uval);
4666                 }
4667         }
4668
4669         if (!multiline)
4670                 seq_puts(m, " ");
4671
4672         seq_puts(m, "}");
4673 }
4674
4675 static void hist_trigger_entry_print(struct seq_file *m,
4676                                      struct hist_trigger_data *hist_data,
4677                                      void *key,
4678                                      struct tracing_map_elt *elt)
4679 {
4680         const char *field_name;
4681         unsigned int i;
4682
4683         hist_trigger_print_key(m, hist_data, key, elt);
4684
4685         seq_printf(m, " hitcount: %10llu",
4686                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4687
4688         for (i = 1; i < hist_data->n_vals; i++) {
4689                 field_name = hist_field_name(hist_data->fields[i], 0);
4690
4691                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4692                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4693                         continue;
4694
4695                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4696                         seq_printf(m, "  %s: %10llx", field_name,
4697                                    tracing_map_read_sum(elt, i));
4698                 } else {
4699                         seq_printf(m, "  %s: %10llu", field_name,
4700                                    tracing_map_read_sum(elt, i));
4701                 }
4702         }
4703
4704         print_actions(m, hist_data, elt);
4705
4706         seq_puts(m, "\n");
4707 }
4708
4709 static int print_entries(struct seq_file *m,
4710                          struct hist_trigger_data *hist_data)
4711 {
4712         struct tracing_map_sort_entry **sort_entries = NULL;
4713         struct tracing_map *map = hist_data->map;
4714         int i, n_entries;
4715
4716         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4717                                              hist_data->n_sort_keys,
4718                                              &sort_entries);
4719         if (n_entries < 0)
4720                 return n_entries;
4721
4722         for (i = 0; i < n_entries; i++)
4723                 hist_trigger_entry_print(m, hist_data,
4724                                          sort_entries[i]->key,
4725                                          sort_entries[i]->elt);
4726
4727         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4728
4729         return n_entries;
4730 }
4731
4732 static void hist_trigger_show(struct seq_file *m,
4733                               struct event_trigger_data *data, int n)
4734 {
4735         struct hist_trigger_data *hist_data;
4736         int n_entries;
4737
4738         if (n > 0)
4739                 seq_puts(m, "\n\n");
4740
4741         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4742         data->ops->print(m, data->ops, data);
4743         seq_puts(m, "#\n\n");
4744
4745         hist_data = data->private_data;
4746         n_entries = print_entries(m, hist_data);
4747         if (n_entries < 0)
4748                 n_entries = 0;
4749
4750         track_data_snapshot_print(m, hist_data);
4751
4752         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4753                    (u64)atomic64_read(&hist_data->map->hits),
4754                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4755 }
4756
4757 static int hist_show(struct seq_file *m, void *v)
4758 {
4759         struct event_trigger_data *data;
4760         struct trace_event_file *event_file;
4761         int n = 0, ret = 0;
4762
4763         mutex_lock(&event_mutex);
4764
4765         event_file = event_file_data(m->private);
4766         if (unlikely(!event_file)) {
4767                 ret = -ENODEV;
4768                 goto out_unlock;
4769         }
4770
4771         list_for_each_entry(data, &event_file->triggers, list) {
4772                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4773                         hist_trigger_show(m, data, n++);
4774         }
4775
4776  out_unlock:
4777         mutex_unlock(&event_mutex);
4778
4779         return ret;
4780 }
4781
4782 static int event_hist_open(struct inode *inode, struct file *file)
4783 {
4784         int ret;
4785
4786         ret = security_locked_down(LOCKDOWN_TRACEFS);
4787         if (ret)
4788                 return ret;
4789
4790         return single_open(file, hist_show, file);
4791 }
4792
4793 const struct file_operations event_hist_fops = {
4794         .open = event_hist_open,
4795         .read = seq_read,
4796         .llseek = seq_lseek,
4797         .release = single_release,
4798 };
4799
4800 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4801 static void hist_field_debug_show_flags(struct seq_file *m,
4802                                         unsigned long flags)
4803 {
4804         seq_puts(m, "      flags:\n");
4805
4806         if (flags & HIST_FIELD_FL_KEY)
4807                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4808         else if (flags & HIST_FIELD_FL_HITCOUNT)
4809                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4810         else if (flags & HIST_FIELD_FL_VAR)
4811                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4812         else if (flags & HIST_FIELD_FL_VAR_REF)
4813                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4814         else
4815                 seq_puts(m, "        VAL: normal u64 value\n");
4816
4817         if (flags & HIST_FIELD_FL_ALIAS)
4818                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4819 }
4820
4821 static int hist_field_debug_show(struct seq_file *m,
4822                                  struct hist_field *field, unsigned long flags)
4823 {
4824         if ((field->flags & flags) != flags) {
4825                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4826                 return -EINVAL;
4827         }
4828
4829         hist_field_debug_show_flags(m, field->flags);
4830         if (field->field)
4831                 seq_printf(m, "      ftrace_event_field name: %s\n",
4832                            field->field->name);
4833
4834         if (field->flags & HIST_FIELD_FL_VAR) {
4835                 seq_printf(m, "      var.name: %s\n", field->var.name);
4836                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4837                            field->var.idx);
4838         }
4839
4840         if (field->flags & HIST_FIELD_FL_ALIAS)
4841                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4842                            field->var_ref_idx);
4843
4844         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4845                 seq_printf(m, "      name: %s\n", field->name);
4846                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4847                            field->var.idx);
4848                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4849                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4850                            field->var_ref_idx);
4851                 if (field->system)
4852                         seq_printf(m, "      system: %s\n", field->system);
4853                 if (field->event_name)
4854                         seq_printf(m, "      event_name: %s\n", field->event_name);
4855         }
4856
4857         seq_printf(m, "      type: %s\n", field->type);
4858         seq_printf(m, "      size: %u\n", field->size);
4859         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4860
4861         return 0;
4862 }
4863
4864 static int field_var_debug_show(struct seq_file *m,
4865                                 struct field_var *field_var, unsigned int i,
4866                                 bool save_vars)
4867 {
4868         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4869         struct hist_field *field;
4870         int ret = 0;
4871
4872         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4873
4874         field = field_var->var;
4875
4876         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4877
4878         hist_field_debug_show_flags(m, field->flags);
4879         seq_printf(m, "      var.name: %s\n", field->var.name);
4880         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4881                    field->var.idx);
4882
4883         field = field_var->val;
4884
4885         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4886         if (field->field)
4887                 seq_printf(m, "      ftrace_event_field name: %s\n",
4888                            field->field->name);
4889         else {
4890                 ret = -EINVAL;
4891                 goto out;
4892         }
4893
4894         seq_printf(m, "      type: %s\n", field->type);
4895         seq_printf(m, "      size: %u\n", field->size);
4896         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4897 out:
4898         return ret;
4899 }
4900
4901 static int hist_action_debug_show(struct seq_file *m,
4902                                   struct action_data *data, int i)
4903 {
4904         int ret = 0;
4905
4906         if (data->handler == HANDLER_ONMAX ||
4907             data->handler == HANDLER_ONCHANGE) {
4908                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4909                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4910                                             HIST_FIELD_FL_VAR_REF);
4911                 if (ret)
4912                         goto out;
4913
4914                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4915                 ret = hist_field_debug_show(m, data->track_data.track_var,
4916                                             HIST_FIELD_FL_VAR);
4917                 if (ret)
4918                         goto out;
4919         }
4920
4921         if (data->handler == HANDLER_ONMATCH) {
4922                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4923                            i, data->match_data.event_system);
4924                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4925                            i, data->match_data.event);
4926         }
4927 out:
4928         return ret;
4929 }
4930
4931 static int hist_actions_debug_show(struct seq_file *m,
4932                                    struct hist_trigger_data *hist_data)
4933 {
4934         int i, ret = 0;
4935
4936         if (hist_data->n_actions)
4937                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4938
4939         for (i = 0; i < hist_data->n_actions; i++) {
4940                 struct action_data *action = hist_data->actions[i];
4941
4942                 ret = hist_action_debug_show(m, action, i);
4943                 if (ret)
4944                         goto out;
4945         }
4946
4947         if (hist_data->n_save_vars)
4948                 seq_puts(m, "\n  save action variables (save() params):\n");
4949
4950         for (i = 0; i < hist_data->n_save_vars; i++) {
4951                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4952                 if (ret)
4953                         goto out;
4954         }
4955 out:
4956         return ret;
4957 }
4958
4959 static void hist_trigger_debug_show(struct seq_file *m,
4960                                     struct event_trigger_data *data, int n)
4961 {
4962         struct hist_trigger_data *hist_data;
4963         int i, ret;
4964
4965         if (n > 0)
4966                 seq_puts(m, "\n\n");
4967
4968         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4969         data->ops->print(m, data->ops, data);
4970         seq_puts(m, "#\n\n");
4971
4972         hist_data = data->private_data;
4973
4974         seq_printf(m, "hist_data: %p\n\n", hist_data);
4975         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4976         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4977         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4978
4979         seq_puts(m, "\n  val fields:\n\n");
4980
4981         seq_puts(m, "    hist_data->fields[0]:\n");
4982         ret = hist_field_debug_show(m, hist_data->fields[0],
4983                                     HIST_FIELD_FL_HITCOUNT);
4984         if (ret)
4985                 return;
4986
4987         for (i = 1; i < hist_data->n_vals; i++) {
4988                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4989                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4990                 if (ret)
4991                         return;
4992         }
4993
4994         seq_puts(m, "\n  key fields:\n");
4995
4996         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
4997                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4998                 ret = hist_field_debug_show(m, hist_data->fields[i],
4999                                             HIST_FIELD_FL_KEY);
5000                 if (ret)
5001                         return;
5002         }
5003
5004         if (hist_data->n_var_refs)
5005                 seq_puts(m, "\n  variable reference fields:\n");
5006
5007         for (i = 0; i < hist_data->n_var_refs; i++) {
5008                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5009                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5010                                             HIST_FIELD_FL_VAR_REF);
5011                 if (ret)
5012                         return;
5013         }
5014
5015         if (hist_data->n_field_vars)
5016                 seq_puts(m, "\n  field variables:\n");
5017
5018         for (i = 0; i < hist_data->n_field_vars; i++) {
5019                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5020                 if (ret)
5021                         return;
5022         }
5023
5024         ret = hist_actions_debug_show(m, hist_data);
5025         if (ret)
5026                 return;
5027 }
5028
5029 static int hist_debug_show(struct seq_file *m, void *v)
5030 {
5031         struct event_trigger_data *data;
5032         struct trace_event_file *event_file;
5033         int n = 0, ret = 0;
5034
5035         mutex_lock(&event_mutex);
5036
5037         event_file = event_file_data(m->private);
5038         if (unlikely(!event_file)) {
5039                 ret = -ENODEV;
5040                 goto out_unlock;
5041         }
5042
5043         list_for_each_entry(data, &event_file->triggers, list) {
5044                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5045                         hist_trigger_debug_show(m, data, n++);
5046         }
5047
5048  out_unlock:
5049         mutex_unlock(&event_mutex);
5050
5051         return ret;
5052 }
5053
5054 static int event_hist_debug_open(struct inode *inode, struct file *file)
5055 {
5056         int ret;
5057
5058         ret = security_locked_down(LOCKDOWN_TRACEFS);
5059         if (ret)
5060                 return ret;
5061
5062         return single_open(file, hist_debug_show, file);
5063 }
5064
5065 const struct file_operations event_hist_debug_fops = {
5066         .open = event_hist_debug_open,
5067         .read = seq_read,
5068         .llseek = seq_lseek,
5069         .release = single_release,
5070 };
5071 #endif
5072
5073 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5074 {
5075         const char *field_name = hist_field_name(hist_field, 0);
5076
5077         if (hist_field->var.name)
5078                 seq_printf(m, "%s=", hist_field->var.name);
5079
5080         if (hist_field->flags & HIST_FIELD_FL_CPU)
5081                 seq_puts(m, "cpu");
5082         else if (field_name) {
5083                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5084                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5085                         seq_putc(m, '$');
5086                 seq_printf(m, "%s", field_name);
5087         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5088                 seq_puts(m, "common_timestamp");
5089
5090         if (hist_field->flags) {
5091                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5092                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5093                         const char *flags = get_hist_field_flags(hist_field);
5094
5095                         if (flags)
5096                                 seq_printf(m, ".%s", flags);
5097                 }
5098         }
5099 }
5100
5101 static int event_hist_trigger_print(struct seq_file *m,
5102                                     struct event_trigger_ops *ops,
5103                                     struct event_trigger_data *data)
5104 {
5105         struct hist_trigger_data *hist_data = data->private_data;
5106         struct hist_field *field;
5107         bool have_var = false;
5108         unsigned int i;
5109
5110         seq_puts(m, "hist:");
5111
5112         if (data->name)
5113                 seq_printf(m, "%s:", data->name);
5114
5115         seq_puts(m, "keys=");
5116
5117         for_each_hist_key_field(i, hist_data) {
5118                 field = hist_data->fields[i];
5119
5120                 if (i > hist_data->n_vals)
5121                         seq_puts(m, ",");
5122
5123                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5124                         seq_puts(m, "stacktrace");
5125                 else
5126                         hist_field_print(m, field);
5127         }
5128
5129         seq_puts(m, ":vals=");
5130
5131         for_each_hist_val_field(i, hist_data) {
5132                 field = hist_data->fields[i];
5133                 if (field->flags & HIST_FIELD_FL_VAR) {
5134                         have_var = true;
5135                         continue;
5136                 }
5137
5138                 if (i == HITCOUNT_IDX)
5139                         seq_puts(m, "hitcount");
5140                 else {
5141                         seq_puts(m, ",");
5142                         hist_field_print(m, field);
5143                 }
5144         }
5145
5146         if (have_var) {
5147                 unsigned int n = 0;
5148
5149                 seq_puts(m, ":");
5150
5151                 for_each_hist_val_field(i, hist_data) {
5152                         field = hist_data->fields[i];
5153
5154                         if (field->flags & HIST_FIELD_FL_VAR) {
5155                                 if (n++)
5156                                         seq_puts(m, ",");
5157                                 hist_field_print(m, field);
5158                         }
5159                 }
5160         }
5161
5162         seq_puts(m, ":sort=");
5163
5164         for (i = 0; i < hist_data->n_sort_keys; i++) {
5165                 struct tracing_map_sort_key *sort_key;
5166                 unsigned int idx, first_key_idx;
5167
5168                 /* skip VAR vals */
5169                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5170
5171                 sort_key = &hist_data->sort_keys[i];
5172                 idx = sort_key->field_idx;
5173
5174                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5175                         return -EINVAL;
5176
5177                 if (i > 0)
5178                         seq_puts(m, ",");
5179
5180                 if (idx == HITCOUNT_IDX)
5181                         seq_puts(m, "hitcount");
5182                 else {
5183                         if (idx >= first_key_idx)
5184                                 idx += hist_data->n_vars;
5185                         hist_field_print(m, hist_data->fields[idx]);
5186                 }
5187
5188                 if (sort_key->descending)
5189                         seq_puts(m, ".descending");
5190         }
5191         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5192         if (hist_data->enable_timestamps)
5193                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5194
5195         print_actions_spec(m, hist_data);
5196
5197         if (data->filter_str)
5198                 seq_printf(m, " if %s", data->filter_str);
5199
5200         if (data->paused)
5201                 seq_puts(m, " [paused]");
5202         else
5203                 seq_puts(m, " [active]");
5204
5205         seq_putc(m, '\n');
5206
5207         return 0;
5208 }
5209
5210 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5211                                    struct event_trigger_data *data)
5212 {
5213         struct hist_trigger_data *hist_data = data->private_data;
5214
5215         if (!data->ref && hist_data->attrs->name)
5216                 save_named_trigger(hist_data->attrs->name, data);
5217
5218         data->ref++;
5219
5220         return 0;
5221 }
5222
5223 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5224 {
5225         struct trace_event_file *file;
5226         unsigned int i;
5227         char *cmd;
5228         int ret;
5229
5230         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5231                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5232                 cmd = hist_data->field_var_hists[i]->cmd;
5233                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5234                                               "!hist", "hist", cmd);
5235         }
5236 }
5237
5238 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5239                                     struct event_trigger_data *data)
5240 {
5241         struct hist_trigger_data *hist_data = data->private_data;
5242
5243         if (WARN_ON_ONCE(data->ref <= 0))
5244                 return;
5245
5246         data->ref--;
5247         if (!data->ref) {
5248                 if (data->name)
5249                         del_named_trigger(data);
5250
5251                 trigger_data_free(data);
5252
5253                 remove_hist_vars(hist_data);
5254
5255                 unregister_field_var_hists(hist_data);
5256
5257                 destroy_hist_data(hist_data);
5258         }
5259 }
5260
5261 static struct event_trigger_ops event_hist_trigger_ops = {
5262         .func                   = event_hist_trigger,
5263         .print                  = event_hist_trigger_print,
5264         .init                   = event_hist_trigger_init,
5265         .free                   = event_hist_trigger_free,
5266 };
5267
5268 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5269                                          struct event_trigger_data *data)
5270 {
5271         data->ref++;
5272
5273         save_named_trigger(data->named_data->name, data);
5274
5275         event_hist_trigger_init(ops, data->named_data);
5276
5277         return 0;
5278 }
5279
5280 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5281                                           struct event_trigger_data *data)
5282 {
5283         if (WARN_ON_ONCE(data->ref <= 0))
5284                 return;
5285
5286         event_hist_trigger_free(ops, data->named_data);
5287
5288         data->ref--;
5289         if (!data->ref) {
5290                 del_named_trigger(data);
5291                 trigger_data_free(data);
5292         }
5293 }
5294
5295 static struct event_trigger_ops event_hist_trigger_named_ops = {
5296         .func                   = event_hist_trigger,
5297         .print                  = event_hist_trigger_print,
5298         .init                   = event_hist_trigger_named_init,
5299         .free                   = event_hist_trigger_named_free,
5300 };
5301
5302 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5303                                                             char *param)
5304 {
5305         return &event_hist_trigger_ops;
5306 }
5307
5308 static void hist_clear(struct event_trigger_data *data)
5309 {
5310         struct hist_trigger_data *hist_data = data->private_data;
5311
5312         if (data->name)
5313                 pause_named_trigger(data);
5314
5315         tracepoint_synchronize_unregister();
5316
5317         tracing_map_clear(hist_data->map);
5318
5319         if (data->name)
5320                 unpause_named_trigger(data);
5321 }
5322
5323 static bool compatible_field(struct ftrace_event_field *field,
5324                              struct ftrace_event_field *test_field)
5325 {
5326         if (field == test_field)
5327                 return true;
5328         if (field == NULL || test_field == NULL)
5329                 return false;
5330         if (strcmp(field->name, test_field->name) != 0)
5331                 return false;
5332         if (strcmp(field->type, test_field->type) != 0)
5333                 return false;
5334         if (field->size != test_field->size)
5335                 return false;
5336         if (field->is_signed != test_field->is_signed)
5337                 return false;
5338
5339         return true;
5340 }
5341
5342 static bool hist_trigger_match(struct event_trigger_data *data,
5343                                struct event_trigger_data *data_test,
5344                                struct event_trigger_data *named_data,
5345                                bool ignore_filter)
5346 {
5347         struct tracing_map_sort_key *sort_key, *sort_key_test;
5348         struct hist_trigger_data *hist_data, *hist_data_test;
5349         struct hist_field *key_field, *key_field_test;
5350         unsigned int i;
5351
5352         if (named_data && (named_data != data_test) &&
5353             (named_data != data_test->named_data))
5354                 return false;
5355
5356         if (!named_data && is_named_trigger(data_test))
5357                 return false;
5358
5359         hist_data = data->private_data;
5360         hist_data_test = data_test->private_data;
5361
5362         if (hist_data->n_vals != hist_data_test->n_vals ||
5363             hist_data->n_fields != hist_data_test->n_fields ||
5364             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5365                 return false;
5366
5367         if (!ignore_filter) {
5368                 if ((data->filter_str && !data_test->filter_str) ||
5369                    (!data->filter_str && data_test->filter_str))
5370                         return false;
5371         }
5372
5373         for_each_hist_field(i, hist_data) {
5374                 key_field = hist_data->fields[i];
5375                 key_field_test = hist_data_test->fields[i];
5376
5377                 if (key_field->flags != key_field_test->flags)
5378                         return false;
5379                 if (!compatible_field(key_field->field, key_field_test->field))
5380                         return false;
5381                 if (key_field->offset != key_field_test->offset)
5382                         return false;
5383                 if (key_field->size != key_field_test->size)
5384                         return false;
5385                 if (key_field->is_signed != key_field_test->is_signed)
5386                         return false;
5387                 if (!!key_field->var.name != !!key_field_test->var.name)
5388                         return false;
5389                 if (key_field->var.name &&
5390                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5391                         return false;
5392         }
5393
5394         for (i = 0; i < hist_data->n_sort_keys; i++) {
5395                 sort_key = &hist_data->sort_keys[i];
5396                 sort_key_test = &hist_data_test->sort_keys[i];
5397
5398                 if (sort_key->field_idx != sort_key_test->field_idx ||
5399                     sort_key->descending != sort_key_test->descending)
5400                         return false;
5401         }
5402
5403         if (!ignore_filter && data->filter_str &&
5404             (strcmp(data->filter_str, data_test->filter_str) != 0))
5405                 return false;
5406
5407         if (!actions_match(hist_data, hist_data_test))
5408                 return false;
5409
5410         return true;
5411 }
5412
5413 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5414                                  struct event_trigger_data *data,
5415                                  struct trace_event_file *file)
5416 {
5417         struct hist_trigger_data *hist_data = data->private_data;
5418         struct event_trigger_data *test, *named_data = NULL;
5419         struct trace_array *tr = file->tr;
5420         int ret = 0;
5421
5422         if (hist_data->attrs->name) {
5423                 named_data = find_named_trigger(hist_data->attrs->name);
5424                 if (named_data) {
5425                         if (!hist_trigger_match(data, named_data, named_data,
5426                                                 true)) {
5427                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5428                                 ret = -EINVAL;
5429                                 goto out;
5430                         }
5431                 }
5432         }
5433
5434         if (hist_data->attrs->name && !named_data)
5435                 goto new;
5436
5437         lockdep_assert_held(&event_mutex);
5438
5439         list_for_each_entry(test, &file->triggers, list) {
5440                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5441                         if (!hist_trigger_match(data, test, named_data, false))
5442                                 continue;
5443                         if (hist_data->attrs->pause)
5444                                 test->paused = true;
5445                         else if (hist_data->attrs->cont)
5446                                 test->paused = false;
5447                         else if (hist_data->attrs->clear)
5448                                 hist_clear(test);
5449                         else {
5450                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5451                                 ret = -EEXIST;
5452                         }
5453                         goto out;
5454                 }
5455         }
5456  new:
5457         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5458                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5459                 ret = -ENOENT;
5460                 goto out;
5461         }
5462
5463         if (hist_data->attrs->pause)
5464                 data->paused = true;
5465
5466         if (named_data) {
5467                 data->private_data = named_data->private_data;
5468                 set_named_trigger_data(data, named_data);
5469                 data->ops = &event_hist_trigger_named_ops;
5470         }
5471
5472         if (data->ops->init) {
5473                 ret = data->ops->init(data->ops, data);
5474                 if (ret < 0)
5475                         goto out;
5476         }
5477
5478         if (hist_data->enable_timestamps) {
5479                 char *clock = hist_data->attrs->clock;
5480
5481                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5482                 if (ret) {
5483                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5484                         goto out;
5485                 }
5486
5487                 tracing_set_filter_buffering(file->tr, true);
5488         }
5489
5490         if (named_data)
5491                 destroy_hist_data(hist_data);
5492
5493         ret++;
5494  out:
5495         return ret;
5496 }
5497
5498 static int hist_trigger_enable(struct event_trigger_data *data,
5499                                struct trace_event_file *file)
5500 {
5501         int ret = 0;
5502
5503         list_add_tail_rcu(&data->list, &file->triggers);
5504
5505         update_cond_flag(file);
5506
5507         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5508                 list_del_rcu(&data->list);
5509                 update_cond_flag(file);
5510                 ret--;
5511         }
5512
5513         return ret;
5514 }
5515
5516 static bool have_hist_trigger_match(struct event_trigger_data *data,
5517                                     struct trace_event_file *file)
5518 {
5519         struct hist_trigger_data *hist_data = data->private_data;
5520         struct event_trigger_data *test, *named_data = NULL;
5521         bool match = false;
5522
5523         lockdep_assert_held(&event_mutex);
5524
5525         if (hist_data->attrs->name)
5526                 named_data = find_named_trigger(hist_data->attrs->name);
5527
5528         list_for_each_entry(test, &file->triggers, list) {
5529                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5530                         if (hist_trigger_match(data, test, named_data, false)) {
5531                                 match = true;
5532                                 break;
5533                         }
5534                 }
5535         }
5536
5537         return match;
5538 }
5539
5540 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5541                                     struct trace_event_file *file)
5542 {
5543         struct hist_trigger_data *hist_data = data->private_data;
5544         struct event_trigger_data *test, *named_data = NULL;
5545
5546         lockdep_assert_held(&event_mutex);
5547
5548         if (hist_data->attrs->name)
5549                 named_data = find_named_trigger(hist_data->attrs->name);
5550
5551         list_for_each_entry(test, &file->triggers, list) {
5552                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5553                         if (!hist_trigger_match(data, test, named_data, false))
5554                                 continue;
5555                         hist_data = test->private_data;
5556                         if (check_var_refs(hist_data))
5557                                 return true;
5558                         break;
5559                 }
5560         }
5561
5562         return false;
5563 }
5564
5565 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5566                                     struct event_trigger_data *data,
5567                                     struct trace_event_file *file)
5568 {
5569         struct hist_trigger_data *hist_data = data->private_data;
5570         struct event_trigger_data *test, *named_data = NULL;
5571         bool unregistered = false;
5572
5573         lockdep_assert_held(&event_mutex);
5574
5575         if (hist_data->attrs->name)
5576                 named_data = find_named_trigger(hist_data->attrs->name);
5577
5578         list_for_each_entry(test, &file->triggers, list) {
5579                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5580                         if (!hist_trigger_match(data, test, named_data, false))
5581                                 continue;
5582                         unregistered = true;
5583                         list_del_rcu(&test->list);
5584                         trace_event_trigger_enable_disable(file, 0);
5585                         update_cond_flag(file);
5586                         break;
5587                 }
5588         }
5589
5590         if (unregistered && test->ops->free)
5591                 test->ops->free(test->ops, test);
5592
5593         if (hist_data->enable_timestamps) {
5594                 if (!hist_data->remove || unregistered)
5595                         tracing_set_filter_buffering(file->tr, false);
5596         }
5597 }
5598
5599 static bool hist_file_check_refs(struct trace_event_file *file)
5600 {
5601         struct hist_trigger_data *hist_data;
5602         struct event_trigger_data *test;
5603
5604         lockdep_assert_held(&event_mutex);
5605
5606         list_for_each_entry(test, &file->triggers, list) {
5607                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5608                         hist_data = test->private_data;
5609                         if (check_var_refs(hist_data))
5610                                 return true;
5611                 }
5612         }
5613
5614         return false;
5615 }
5616
5617 static void hist_unreg_all(struct trace_event_file *file)
5618 {
5619         struct event_trigger_data *test, *n;
5620         struct hist_trigger_data *hist_data;
5621         struct synth_event *se;
5622         const char *se_name;
5623
5624         lockdep_assert_held(&event_mutex);
5625
5626         if (hist_file_check_refs(file))
5627                 return;
5628
5629         list_for_each_entry_safe(test, n, &file->triggers, list) {
5630                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5631                         hist_data = test->private_data;
5632                         list_del_rcu(&test->list);
5633                         trace_event_trigger_enable_disable(file, 0);
5634
5635                         se_name = trace_event_name(file->event_call);
5636                         se = find_synth_event(se_name);
5637                         if (se)
5638                                 se->ref--;
5639
5640                         update_cond_flag(file);
5641                         if (hist_data->enable_timestamps)
5642                                 tracing_set_filter_buffering(file->tr, false);
5643                         if (test->ops->free)
5644                                 test->ops->free(test->ops, test);
5645                 }
5646         }
5647 }
5648
5649 static int event_hist_trigger_func(struct event_command *cmd_ops,
5650                                    struct trace_event_file *file,
5651                                    char *glob, char *cmd, char *param)
5652 {
5653         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5654         struct event_trigger_data *trigger_data;
5655         struct hist_trigger_attrs *attrs;
5656         struct event_trigger_ops *trigger_ops;
5657         struct hist_trigger_data *hist_data;
5658         struct synth_event *se;
5659         const char *se_name;
5660         bool remove = false;
5661         char *trigger, *p;
5662         int ret = 0;
5663
5664         lockdep_assert_held(&event_mutex);
5665
5666         if (glob && strlen(glob)) {
5667                 hist_err_clear();
5668                 last_cmd_set(file, param);
5669         }
5670
5671         if (!param)
5672                 return -EINVAL;
5673
5674         if (glob[0] == '!')
5675                 remove = true;
5676
5677         /*
5678          * separate the trigger from the filter (k:v [if filter])
5679          * allowing for whitespace in the trigger
5680          */
5681         p = trigger = param;
5682         do {
5683                 p = strstr(p, "if");
5684                 if (!p)
5685                         break;
5686                 if (p == param)
5687                         return -EINVAL;
5688                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5689                         p++;
5690                         continue;
5691                 }
5692                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5693                         return -EINVAL;
5694                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5695                         p++;
5696                         continue;
5697                 }
5698                 break;
5699         } while (p);
5700
5701         if (!p)
5702                 param = NULL;
5703         else {
5704                 *(p - 1) = '\0';
5705                 param = strstrip(p);
5706                 trigger = strstrip(trigger);
5707         }
5708
5709         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5710         if (IS_ERR(attrs))
5711                 return PTR_ERR(attrs);
5712
5713         if (attrs->map_bits)
5714                 hist_trigger_bits = attrs->map_bits;
5715
5716         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5717         if (IS_ERR(hist_data)) {
5718                 destroy_hist_trigger_attrs(attrs);
5719                 return PTR_ERR(hist_data);
5720         }
5721
5722         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5723
5724         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5725         if (!trigger_data) {
5726                 ret = -ENOMEM;
5727                 goto out_free;
5728         }
5729
5730         trigger_data->count = -1;
5731         trigger_data->ops = trigger_ops;
5732         trigger_data->cmd_ops = cmd_ops;
5733
5734         INIT_LIST_HEAD(&trigger_data->list);
5735         RCU_INIT_POINTER(trigger_data->filter, NULL);
5736
5737         trigger_data->private_data = hist_data;
5738
5739         /* if param is non-empty, it's supposed to be a filter */
5740         if (param && cmd_ops->set_filter) {
5741                 ret = cmd_ops->set_filter(param, trigger_data, file);
5742                 if (ret < 0)
5743                         goto out_free;
5744         }
5745
5746         if (remove) {
5747                 if (!have_hist_trigger_match(trigger_data, file))
5748                         goto out_free;
5749
5750                 if (hist_trigger_check_refs(trigger_data, file)) {
5751                         ret = -EBUSY;
5752                         goto out_free;
5753                 }
5754
5755                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5756                 se_name = trace_event_name(file->event_call);
5757                 se = find_synth_event(se_name);
5758                 if (se)
5759                         se->ref--;
5760                 ret = 0;
5761                 goto out_free;
5762         }
5763
5764         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5765         /*
5766          * The above returns on success the # of triggers registered,
5767          * but if it didn't register any it returns zero.  Consider no
5768          * triggers registered a failure too.
5769          */
5770         if (!ret) {
5771                 if (!(attrs->pause || attrs->cont || attrs->clear))
5772                         ret = -ENOENT;
5773                 goto out_free;
5774         } else if (ret < 0)
5775                 goto out_free;
5776
5777         if (get_named_trigger_data(trigger_data))
5778                 goto enable;
5779
5780         if (has_hist_vars(hist_data))
5781                 save_hist_vars(hist_data);
5782
5783         ret = create_actions(hist_data);
5784         if (ret)
5785                 goto out_unreg;
5786
5787         ret = tracing_map_init(hist_data->map);
5788         if (ret)
5789                 goto out_unreg;
5790 enable:
5791         ret = hist_trigger_enable(trigger_data, file);
5792         if (ret)
5793                 goto out_unreg;
5794
5795         se_name = trace_event_name(file->event_call);
5796         se = find_synth_event(se_name);
5797         if (se)
5798                 se->ref++;
5799         /* Just return zero, not the number of registered triggers */
5800         ret = 0;
5801  out:
5802         if (ret == 0)
5803                 hist_err_clear();
5804
5805         return ret;
5806  out_unreg:
5807         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5808  out_free:
5809         if (cmd_ops->set_filter)
5810                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5811
5812         remove_hist_vars(hist_data);
5813
5814         kfree(trigger_data);
5815
5816         destroy_hist_data(hist_data);
5817         goto out;
5818 }
5819
5820 static struct event_command trigger_hist_cmd = {
5821         .name                   = "hist",
5822         .trigger_type           = ETT_EVENT_HIST,
5823         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5824         .func                   = event_hist_trigger_func,
5825         .reg                    = hist_register_trigger,
5826         .unreg                  = hist_unregister_trigger,
5827         .unreg_all              = hist_unreg_all,
5828         .get_trigger_ops        = event_hist_get_trigger_ops,
5829         .set_filter             = set_trigger_filter,
5830 };
5831
5832 __init int register_trigger_hist_cmd(void)
5833 {
5834         int ret;
5835
5836         ret = register_event_command(&trigger_hist_cmd);
5837         WARN_ON(ret < 0);
5838
5839         return ret;
5840 }
5841
5842 static void
5843 hist_enable_trigger(struct event_trigger_data *data,
5844                     struct trace_buffer *buffer,  void *rec,
5845                     struct ring_buffer_event *event)
5846 {
5847         struct enable_trigger_data *enable_data = data->private_data;
5848         struct event_trigger_data *test;
5849
5850         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5851                                 lockdep_is_held(&event_mutex)) {
5852                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5853                         if (enable_data->enable)
5854                                 test->paused = false;
5855                         else
5856                                 test->paused = true;
5857                 }
5858         }
5859 }
5860
5861 static void
5862 hist_enable_count_trigger(struct event_trigger_data *data,
5863                           struct trace_buffer *buffer,  void *rec,
5864                           struct ring_buffer_event *event)
5865 {
5866         if (!data->count)
5867                 return;
5868
5869         if (data->count != -1)
5870                 (data->count)--;
5871
5872         hist_enable_trigger(data, buffer, rec, event);
5873 }
5874
5875 static struct event_trigger_ops hist_enable_trigger_ops = {
5876         .func                   = hist_enable_trigger,
5877         .print                  = event_enable_trigger_print,
5878         .init                   = event_trigger_init,
5879         .free                   = event_enable_trigger_free,
5880 };
5881
5882 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5883         .func                   = hist_enable_count_trigger,
5884         .print                  = event_enable_trigger_print,
5885         .init                   = event_trigger_init,
5886         .free                   = event_enable_trigger_free,
5887 };
5888
5889 static struct event_trigger_ops hist_disable_trigger_ops = {
5890         .func                   = hist_enable_trigger,
5891         .print                  = event_enable_trigger_print,
5892         .init                   = event_trigger_init,
5893         .free                   = event_enable_trigger_free,
5894 };
5895
5896 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5897         .func                   = hist_enable_count_trigger,
5898         .print                  = event_enable_trigger_print,
5899         .init                   = event_trigger_init,
5900         .free                   = event_enable_trigger_free,
5901 };
5902
5903 static struct event_trigger_ops *
5904 hist_enable_get_trigger_ops(char *cmd, char *param)
5905 {
5906         struct event_trigger_ops *ops;
5907         bool enable;
5908
5909         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5910
5911         if (enable)
5912                 ops = param ? &hist_enable_count_trigger_ops :
5913                         &hist_enable_trigger_ops;
5914         else
5915                 ops = param ? &hist_disable_count_trigger_ops :
5916                         &hist_disable_trigger_ops;
5917
5918         return ops;
5919 }
5920
5921 static void hist_enable_unreg_all(struct trace_event_file *file)
5922 {
5923         struct event_trigger_data *test, *n;
5924
5925         list_for_each_entry_safe(test, n, &file->triggers, list) {
5926                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5927                         list_del_rcu(&test->list);
5928                         update_cond_flag(file);
5929                         trace_event_trigger_enable_disable(file, 0);
5930                         if (test->ops->free)
5931                                 test->ops->free(test->ops, test);
5932                 }
5933         }
5934 }
5935
5936 static struct event_command trigger_hist_enable_cmd = {
5937         .name                   = ENABLE_HIST_STR,
5938         .trigger_type           = ETT_HIST_ENABLE,
5939         .func                   = event_enable_trigger_func,
5940         .reg                    = event_enable_register_trigger,
5941         .unreg                  = event_enable_unregister_trigger,
5942         .unreg_all              = hist_enable_unreg_all,
5943         .get_trigger_ops        = hist_enable_get_trigger_ops,
5944         .set_filter             = set_trigger_filter,
5945 };
5946
5947 static struct event_command trigger_hist_disable_cmd = {
5948         .name                   = DISABLE_HIST_STR,
5949         .trigger_type           = ETT_HIST_ENABLE,
5950         .func                   = event_enable_trigger_func,
5951         .reg                    = event_enable_register_trigger,
5952         .unreg                  = event_enable_unregister_trigger,
5953         .unreg_all              = hist_enable_unreg_all,
5954         .get_trigger_ops        = hist_enable_get_trigger_ops,
5955         .set_filter             = set_trigger_filter,
5956 };
5957
5958 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5959 {
5960         unregister_event_command(&trigger_hist_enable_cmd);
5961         unregister_event_command(&trigger_hist_disable_cmd);
5962 }
5963
5964 __init int register_trigger_hist_enable_disable_cmds(void)
5965 {
5966         int ret;
5967
5968         ret = register_event_command(&trigger_hist_enable_cmd);
5969         if (WARN_ON(ret < 0))
5970                 return ret;
5971         ret = register_event_command(&trigger_hist_disable_cmd);
5972         if (WARN_ON(ret < 0))
5973                 unregister_trigger_hist_enable_disable_cmds();
5974
5975         return ret;
5976 }