kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"),
  70
  71 #undef C
  72 #define C(a, b)         HIST_ERR_##a
  73
  74 enum { ERRORS };
  75
  76 #undef C
  77 #define C(a, b)         b
  78
  79 static const char *err_text[] = { ERRORS };
  80
  81 struct hist_field;
  82
  83 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84                                 struct tracing_map_elt *elt,
  85                                 struct trace_buffer *buffer,
  86                                 struct ring_buffer_event *rbe,
  87                                 void *event);
  88
  89 #define HIST_FIELD_OPERANDS_MAX 2
  90 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  91 #define HIST_ACTIONS_MAX        8
  92
  93 enum field_op_id {
  94         FIELD_OP_NONE,
  95         FIELD_OP_PLUS,
  96         FIELD_OP_MINUS,
  97         FIELD_OP_UNARY_MINUS,
  98 };
  99
 100 /*
 101  * A hist_var (histogram variable) contains variable information for
 102  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 103  * flag set.  A hist_var has a variable name e.g. ts0, and is
 104  * associated with a given histogram trigger, as specified by
 105  * hist_data.  The hist_var idx is the unique index assigned to the
 106  * variable by the hist trigger's tracing_map.  The idx is what is
 107  * used to set a variable's value and, by a variable reference, to
 108  * retrieve it.
 109  */
 110 struct hist_var {
 111         char                            *name;
 112         struct hist_trigger_data        *hist_data;
 113         unsigned int                    idx;
 114 };
 115
 116 struct hist_field {
 117         struct ftrace_event_field       *field;
 118         unsigned long                   flags;
 119         hist_field_fn_t                 fn;
 120         unsigned int                    ref;
 121         unsigned int                    size;
 122         unsigned int                    offset;
 123         unsigned int                    is_signed;
 124         const char                      *type;
 125         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 126         struct hist_trigger_data        *hist_data;
 127
 128         /*
 129          * Variable fields contain variable-specific info in var.
 130          */
 131         struct hist_var                 var;
 132         enum field_op_id                operator;
 133         char                            *system;
 134         char                            *event_name;
 135
 136         /*
 137          * The name field is used for EXPR and VAR_REF fields.  VAR
 138          * fields contain the variable name in var.name.
 139          */
 140         char                            *name;
 141
 142         /*
 143          * When a histogram trigger is hit, if it has any references
 144          * to variables, the values of those variables are collected
 145          * into a var_ref_vals array by resolve_var_refs().  The
 146          * current value of each variable is read from the tracing_map
 147          * using the hist field's hist_var.idx and entered into the
 148          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 149          */
 150         unsigned int                    var_ref_idx;
 151         bool                            read_once;
 152
 153         unsigned int                    var_str_idx;
 154 };
 155
 156 static u64 hist_field_none(struct hist_field *field,
 157                            struct tracing_map_elt *elt,
 158                            struct trace_buffer *buffer,
 159                            struct ring_buffer_event *rbe,
 160                            void *event)
 161 {
 162         return 0;
 163 }
 164
 165 static u64 hist_field_counter(struct hist_field *field,
 166                               struct tracing_map_elt *elt,
 167                               struct trace_buffer *buffer,
 168                               struct ring_buffer_event *rbe,
 169                               void *event)
 170 {
 171         return 1;
 172 }
 173
 174 static u64 hist_field_string(struct hist_field *hist_field,
 175                              struct tracing_map_elt *elt,
 176                              struct trace_buffer *buffer,
 177                              struct ring_buffer_event *rbe,
 178                              void *event)
 179 {
 180         char *addr = (char *)(event + hist_field->field->offset);
 181
 182         return (u64)(unsigned long)addr;
 183 }
 184
 185 static u64 hist_field_dynstring(struct hist_field *hist_field,
 186                                 struct tracing_map_elt *elt,
 187                                 struct trace_buffer *buffer,
 188                                 struct ring_buffer_event *rbe,
 189                                 void *event)
 190 {
 191         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 192         int str_loc = str_item & 0xffff;
 193         char *addr = (char *)(event + str_loc);
 194
 195         return (u64)(unsigned long)addr;
 196 }
 197
 198 static u64 hist_field_pstring(struct hist_field *hist_field,
 199                               struct tracing_map_elt *elt,
 200                               struct trace_buffer *buffer,
 201                               struct ring_buffer_event *rbe,
 202                               void *event)
 203 {
 204         char **addr = (char **)(event + hist_field->field->offset);
 205
 206         return (u64)(unsigned long)*addr;
 207 }
 208
 209 static u64 hist_field_log2(struct hist_field *hist_field,
 210                            struct tracing_map_elt *elt,
 211                            struct trace_buffer *buffer,
 212                            struct ring_buffer_event *rbe,
 213                            void *event)
 214 {
 215         struct hist_field *operand = hist_field->operands[0];
 216
 217         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 218
 219         return (u64) ilog2(roundup_pow_of_two(val));
 220 }
 221
 222 static u64 hist_field_plus(struct hist_field *hist_field,
 223                            struct tracing_map_elt *elt,
 224                            struct trace_buffer *buffer,
 225                            struct ring_buffer_event *rbe,
 226                            void *event)
 227 {
 228         struct hist_field *operand1 = hist_field->operands[0];
 229         struct hist_field *operand2 = hist_field->operands[1];
 230
 231         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 232         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 233
 234         return val1 + val2;
 235 }
 236
 237 static u64 hist_field_minus(struct hist_field *hist_field,
 238                             struct tracing_map_elt *elt,
 239                             struct trace_buffer *buffer,
 240                             struct ring_buffer_event *rbe,
 241                             void *event)
 242 {
 243         struct hist_field *operand1 = hist_field->operands[0];
 244         struct hist_field *operand2 = hist_field->operands[1];
 245
 246         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 247         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 248
 249         return val1 - val2;
 250 }
 251
 252 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 253                                   struct tracing_map_elt *elt,
 254                                   struct trace_buffer *buffer,
 255                                   struct ring_buffer_event *rbe,
 256                                   void *event)
 257 {
 258         struct hist_field *operand = hist_field->operands[0];
 259
 260         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 261         u64 val = (u64)-sval;
 262
 263         return val;
 264 }
 265
 266 #define DEFINE_HIST_FIELD_FN(type)                                      \
 267         static u64 hist_field_##type(struct hist_field *hist_field,     \
 268                                      struct tracing_map_elt *elt,       \
 269                                      struct trace_buffer *buffer,       \
 270                                      struct ring_buffer_event *rbe,     \
 271                                      void *event)                       \
 272 {                                                                       \
 273         type *addr = (type *)(event + hist_field->field->offset);       \
 274                                                                         \
 275         return (u64)(unsigned long)*addr;                               \
 276 }
 277
 278 DEFINE_HIST_FIELD_FN(s64);
 279 DEFINE_HIST_FIELD_FN(u64);
 280 DEFINE_HIST_FIELD_FN(s32);
 281 DEFINE_HIST_FIELD_FN(u32);
 282 DEFINE_HIST_FIELD_FN(s16);
 283 DEFINE_HIST_FIELD_FN(u16);
 284 DEFINE_HIST_FIELD_FN(s8);
 285 DEFINE_HIST_FIELD_FN(u8);
 286
 287 #define for_each_hist_field(i, hist_data)       \
 288         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 289
 290 #define for_each_hist_val_field(i, hist_data)   \
 291         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 292
 293 #define for_each_hist_key_field(i, hist_data)   \
 294         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 295
 296 #define HIST_STACKTRACE_DEPTH   16
 297 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 298 #define HIST_STACKTRACE_SKIP    5
 299
 300 #define HITCOUNT_IDX            0
 301 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 302
 303 enum hist_field_flags {
 304         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 305         HIST_FIELD_FL_KEY               = 1 << 1,
 306         HIST_FIELD_FL_STRING            = 1 << 2,
 307         HIST_FIELD_FL_HEX               = 1 << 3,
 308         HIST_FIELD_FL_SYM               = 1 << 4,
 309         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 310         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 311         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 312         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 313         HIST_FIELD_FL_LOG2              = 1 << 9,
 314         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 315         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 316         HIST_FIELD_FL_VAR               = 1 << 12,
 317         HIST_FIELD_FL_EXPR              = 1 << 13,
 318         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 319         HIST_FIELD_FL_CPU               = 1 << 15,
 320         HIST_FIELD_FL_ALIAS             = 1 << 16,
 321 };
 322
 323 struct var_defs {
 324         unsigned int    n_vars;
 325         char            *name[TRACING_MAP_VARS_MAX];
 326         char            *expr[TRACING_MAP_VARS_MAX];
 327 };
 328
 329 struct hist_trigger_attrs {
 330         char            *keys_str;
 331         char            *vals_str;
 332         char            *sort_key_str;
 333         char            *name;
 334         char            *clock;
 335         bool            pause;
 336         bool            cont;
 337         bool            clear;
 338         bool            ts_in_usecs;
 339         unsigned int    map_bits;
 340
 341         char            *assignment_str[TRACING_MAP_VARS_MAX];
 342         unsigned int    n_assignments;
 343
 344         char            *action_str[HIST_ACTIONS_MAX];
 345         unsigned int    n_actions;
 346
 347         struct var_defs var_defs;
 348 };
 349
 350 struct field_var {
 351         struct hist_field       *var;
 352         struct hist_field       *val;
 353 };
 354
 355 struct field_var_hist {
 356         struct hist_trigger_data        *hist_data;
 357         char                            *cmd;
 358 };
 359
 360 struct hist_trigger_data {
 361         struct hist_field               *fields[HIST_FIELDS_MAX];
 362         unsigned int                    n_vals;
 363         unsigned int                    n_keys;
 364         unsigned int                    n_fields;
 365         unsigned int                    n_vars;
 366         unsigned int                    n_var_str;
 367         unsigned int                    key_size;
 368         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 369         unsigned int                    n_sort_keys;
 370         struct trace_event_file         *event_file;
 371         struct hist_trigger_attrs       *attrs;
 372         struct tracing_map              *map;
 373         bool                            enable_timestamps;
 374         bool                            remove;
 375         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 376         unsigned int                    n_var_refs;
 377
 378         struct action_data              *actions[HIST_ACTIONS_MAX];
 379         unsigned int                    n_actions;
 380
 381         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 382         unsigned int                    n_field_vars;
 383         unsigned int                    n_field_var_str;
 384         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 385         unsigned int                    n_field_var_hists;
 386
 387         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 388         unsigned int                    n_save_vars;
 389         unsigned int                    n_save_var_str;
 390 };
 391
 392 struct action_data;
 393
 394 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 395                              struct tracing_map_elt *elt,
 396                              struct trace_buffer *buffer, void *rec,
 397                              struct ring_buffer_event *rbe, void *key,
 398                              struct action_data *data, u64 *var_ref_vals);
 399
 400 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 401
 402 enum handler_id {
 403         HANDLER_ONMATCH = 1,
 404         HANDLER_ONMAX,
 405         HANDLER_ONCHANGE,
 406 };
 407
 408 enum action_id {
 409         ACTION_SAVE = 1,
 410         ACTION_TRACE,
 411         ACTION_SNAPSHOT,
 412 };
 413
 414 struct action_data {
 415         enum handler_id         handler;
 416         enum action_id          action;
 417         char                    *action_name;
 418         action_fn_t             fn;
 419
 420         unsigned int            n_params;
 421         char                    *params[SYNTH_FIELDS_MAX];
 422
 423         /*
 424          * When a histogram trigger is hit, the values of any
 425          * references to variables, including variables being passed
 426          * as parameters to synthetic events, are collected into a
 427          * var_ref_vals array.  This var_ref_idx array is an array of
 428          * indices into the var_ref_vals array, one for each synthetic
 429          * event param, and is passed to the synthetic event
 430          * invocation.
 431          */
 432         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 433         struct synth_event      *synth_event;
 434         bool                    use_trace_keyword;
 435         char                    *synth_event_name;
 436
 437         union {
 438                 struct {
 439                         char                    *event;
 440                         char                    *event_system;
 441                 } match_data;
 442
 443                 struct {
 444                         /*
 445                          * var_str contains the $-unstripped variable
 446                          * name referenced by var_ref, and used when
 447                          * printing the action.  Because var_ref
 448                          * creation is deferred to create_actions(),
 449                          * we need a per-action way to save it until
 450                          * then, thus var_str.
 451                          */
 452                         char                    *var_str;
 453
 454                         /*
 455                          * var_ref refers to the variable being
 456                          * tracked e.g onmax($var).
 457                          */
 458                         struct hist_field       *var_ref;
 459
 460                         /*
 461                          * track_var contains the 'invisible' tracking
 462                          * variable created to keep the current
 463                          * e.g. max value.
 464                          */
 465                         struct hist_field       *track_var;
 466
 467                         check_track_val_fn_t    check_val;
 468                         action_fn_t             save_data;
 469                 } track_data;
 470         };
 471 };
 472
 473 struct track_data {
 474         u64                             track_val;
 475         bool                            updated;
 476
 477         unsigned int                    key_len;
 478         void                            *key;
 479         struct tracing_map_elt          elt;
 480
 481         struct action_data              *action_data;
 482         struct hist_trigger_data        *hist_data;
 483 };
 484
 485 struct hist_elt_data {
 486         char *comm;
 487         u64 *var_ref_vals;
 488         char *field_var_str[SYNTH_FIELDS_MAX];
 489 };
 490
 491 struct snapshot_context {
 492         struct tracing_map_elt  *elt;
 493         void                    *key;
 494 };
 495
 496 static void track_data_free(struct track_data *track_data)
 497 {
 498         struct hist_elt_data *elt_data;
 499
 500         if (!track_data)
 501                 return;
 502
 503         kfree(track_data->key);
 504
 505         elt_data = track_data->elt.private_data;
 506         if (elt_data) {
 507                 kfree(elt_data->comm);
 508                 kfree(elt_data);
 509         }
 510
 511         kfree(track_data);
 512 }
 513
 514 static struct track_data *track_data_alloc(unsigned int key_len,
 515                                            struct action_data *action_data,
 516                                            struct hist_trigger_data *hist_data)
 517 {
 518         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 519         struct hist_elt_data *elt_data;
 520
 521         if (!data)
 522                 return ERR_PTR(-ENOMEM);
 523
 524         data->key = kzalloc(key_len, GFP_KERNEL);
 525         if (!data->key) {
 526                 track_data_free(data);
 527                 return ERR_PTR(-ENOMEM);
 528         }
 529
 530         data->key_len = key_len;
 531         data->action_data = action_data;
 532         data->hist_data = hist_data;
 533
 534         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 535         if (!elt_data) {
 536                 track_data_free(data);
 537                 return ERR_PTR(-ENOMEM);
 538         }
 539
 540         data->elt.private_data = elt_data;
 541
 542         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 543         if (!elt_data->comm) {
 544                 track_data_free(data);
 545                 return ERR_PTR(-ENOMEM);
 546         }
 547
 548         return data;
 549 }
 550
 551 static char last_cmd[MAX_FILTER_STR_VAL];
 552 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 553
 554 static int errpos(char *str)
 555 {
 556         return err_pos(last_cmd, str);
 557 }
 558
 559 static void last_cmd_set(struct trace_event_file *file, char *str)
 560 {
 561         const char *system = NULL, *name = NULL;
 562         struct trace_event_call *call;
 563
 564         if (!str)
 565                 return;
 566
 567         strcpy(last_cmd, "hist:");
 568         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 569
 570         if (file) {
 571                 call = file->event_call;
 572                 system = call->class->system;
 573                 if (system) {
 574                         name = trace_event_name(call);
 575                         if (!name)
 576                                 system = NULL;
 577                 }
 578         }
 579
 580         if (system)
 581                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 582 }
 583
 584 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 585 {
 586         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 587                         err_type, err_pos);
 588 }
 589
 590 static void hist_err_clear(void)
 591 {
 592         last_cmd[0] = '\0';
 593         last_cmd_loc[0] = '\0';
 594 }
 595
 596 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 597                                     unsigned int *var_ref_idx);
 598
 599 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 600                                unsigned int *var_ref_idx)
 601 {
 602         struct tracepoint *tp = event->tp;
 603
 604         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 605                 struct tracepoint_func *probe_func_ptr;
 606                 synth_probe_func_t probe_func;
 607                 void *__data;
 608
 609                 if (!(cpu_online(raw_smp_processor_id())))
 610                         return;
 611
 612                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 613                 if (probe_func_ptr) {
 614                         do {
 615                                 probe_func = probe_func_ptr->func;
 616                                 __data = probe_func_ptr->data;
 617                                 probe_func(__data, var_ref_vals, var_ref_idx);
 618                         } while ((++probe_func_ptr)->func);
 619                 }
 620         }
 621 }
 622
 623 static void action_trace(struct hist_trigger_data *hist_data,
 624                          struct tracing_map_elt *elt,
 625                          struct trace_buffer *buffer, void *rec,
 626                          struct ring_buffer_event *rbe, void *key,
 627                          struct action_data *data, u64 *var_ref_vals)
 628 {
 629         struct synth_event *event = data->synth_event;
 630
 631         trace_synth(event, var_ref_vals, data->var_ref_idx);
 632 }
 633
 634 struct hist_var_data {
 635         struct list_head list;
 636         struct hist_trigger_data *hist_data;
 637 };
 638
 639 static u64 hist_field_timestamp(struct hist_field *hist_field,
 640                                 struct tracing_map_elt *elt,
 641                                 struct trace_buffer *buffer,
 642                                 struct ring_buffer_event *rbe,
 643                                 void *event)
 644 {
 645         struct hist_trigger_data *hist_data = hist_field->hist_data;
 646         struct trace_array *tr = hist_data->event_file->tr;
 647
 648         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 649
 650         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 651                 ts = ns2usecs(ts);
 652
 653         return ts;
 654 }
 655
 656 static u64 hist_field_cpu(struct hist_field *hist_field,
 657                           struct tracing_map_elt *elt,
 658                           struct trace_buffer *buffer,
 659                           struct ring_buffer_event *rbe,
 660                           void *event)
 661 {
 662         int cpu = smp_processor_id();
 663
 664         return cpu;
 665 }
 666
 667 /**
 668  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 669  * @hist_field: The VAR_REF field to check
 670  * @var_data: The hist trigger that owns the variable
 671  * @var_idx: The trigger variable identifier
 672  *
 673  * Check the given VAR_REF field to see whether or not it references
 674  * the given variable associated with the given trigger.
 675  *
 676  * Return: The VAR_REF field if it does reference the variable, NULL if not
 677  */
 678 static struct hist_field *
 679 check_field_for_var_ref(struct hist_field *hist_field,
 680                         struct hist_trigger_data *var_data,
 681                         unsigned int var_idx)
 682 {
 683         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 684
 685         if (hist_field && hist_field->var.idx == var_idx &&
 686             hist_field->var.hist_data == var_data)
 687                 return hist_field;
 688
 689         return NULL;
 690 }
 691
 692 /**
 693  * find_var_ref - Check if a trigger has a reference to a trigger variable
 694  * @hist_data: The hist trigger that might have a reference to the variable
 695  * @var_data: The hist trigger that owns the variable
 696  * @var_idx: The trigger variable identifier
 697  *
 698  * Check the list of var_refs[] on the first hist trigger to see
 699  * whether any of them are references to the variable on the second
 700  * trigger.
 701  *
 702  * Return: The VAR_REF field referencing the variable if so, NULL if not
 703  */
 704 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 705                                        struct hist_trigger_data *var_data,
 706                                        unsigned int var_idx)
 707 {
 708         struct hist_field *hist_field;
 709         unsigned int i;
 710
 711         for (i = 0; i < hist_data->n_var_refs; i++) {
 712                 hist_field = hist_data->var_refs[i];
 713                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 714                         return hist_field;
 715         }
 716
 717         return NULL;
 718 }
 719
 720 /**
 721  * find_any_var_ref - Check if there is a reference to a given trigger variable
 722  * @hist_data: The hist trigger
 723  * @var_idx: The trigger variable identifier
 724  *
 725  * Check to see whether the given variable is currently referenced by
 726  * any other trigger.
 727  *
 728  * The trigger the variable is defined on is explicitly excluded - the
 729  * assumption being that a self-reference doesn't prevent a trigger
 730  * from being removed.
 731  *
 732  * Return: The VAR_REF field referencing the variable if so, NULL if not
 733  */
 734 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 735                                            unsigned int var_idx)
 736 {
 737         struct trace_array *tr = hist_data->event_file->tr;
 738         struct hist_field *found = NULL;
 739         struct hist_var_data *var_data;
 740
 741         list_for_each_entry(var_data, &tr->hist_vars, list) {
 742                 if (var_data->hist_data == hist_data)
 743                         continue;
 744                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 745                 if (found)
 746                         break;
 747         }
 748
 749         return found;
 750 }
 751
 752 /**
 753  * check_var_refs - Check if there is a reference to any of trigger's variables
 754  * @hist_data: The hist trigger
 755  *
 756  * A trigger can define one or more variables.  If any one of them is
 757  * currently referenced by any other trigger, this function will
 758  * determine that.
 759
 760  * Typically used to determine whether or not a trigger can be removed
 761  * - if there are any references to a trigger's variables, it cannot.
 762  *
 763  * Return: True if there is a reference to any of trigger's variables
 764  */
 765 static bool check_var_refs(struct hist_trigger_data *hist_data)
 766 {
 767         struct hist_field *field;
 768         bool found = false;
 769         int i;
 770
 771         for_each_hist_field(i, hist_data) {
 772                 field = hist_data->fields[i];
 773                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 774                         if (find_any_var_ref(hist_data, field->var.idx)) {
 775                                 found = true;
 776                                 break;
 777                         }
 778                 }
 779         }
 780
 781         return found;
 782 }
 783
 784 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 785 {
 786         struct trace_array *tr = hist_data->event_file->tr;
 787         struct hist_var_data *var_data, *found = NULL;
 788
 789         list_for_each_entry(var_data, &tr->hist_vars, list) {
 790                 if (var_data->hist_data == hist_data) {
 791                         found = var_data;
 792                         break;
 793                 }
 794         }
 795
 796         return found;
 797 }
 798
 799 static bool field_has_hist_vars(struct hist_field *hist_field,
 800                                 unsigned int level)
 801 {
 802         int i;
 803
 804         if (level > 3)
 805                 return false;
 806
 807         if (!hist_field)
 808                 return false;
 809
 810         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 811             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 812                 return true;
 813
 814         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 815                 struct hist_field *operand;
 816
 817                 operand = hist_field->operands[i];
 818                 if (field_has_hist_vars(operand, level + 1))
 819                         return true;
 820         }
 821
 822         return false;
 823 }
 824
 825 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 826 {
 827         struct hist_field *hist_field;
 828         int i;
 829
 830         for_each_hist_field(i, hist_data) {
 831                 hist_field = hist_data->fields[i];
 832                 if (field_has_hist_vars(hist_field, 0))
 833                         return true;
 834         }
 835
 836         return false;
 837 }
 838
 839 static int save_hist_vars(struct hist_trigger_data *hist_data)
 840 {
 841         struct trace_array *tr = hist_data->event_file->tr;
 842         struct hist_var_data *var_data;
 843
 844         var_data = find_hist_vars(hist_data);
 845         if (var_data)
 846                 return 0;
 847
 848         if (tracing_check_open_get_tr(tr))
 849                 return -ENODEV;
 850
 851         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 852         if (!var_data) {
 853                 trace_array_put(tr);
 854                 return -ENOMEM;
 855         }
 856
 857         var_data->hist_data = hist_data;
 858         list_add(&var_data->list, &tr->hist_vars);
 859
 860         return 0;
 861 }
 862
 863 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 864 {
 865         struct trace_array *tr = hist_data->event_file->tr;
 866         struct hist_var_data *var_data;
 867
 868         var_data = find_hist_vars(hist_data);
 869         if (!var_data)
 870                 return;
 871
 872         if (WARN_ON(check_var_refs(hist_data)))
 873                 return;
 874
 875         list_del(&var_data->list);
 876
 877         kfree(var_data);
 878
 879         trace_array_put(tr);
 880 }
 881
 882 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 883                                          const char *var_name)
 884 {
 885         struct hist_field *hist_field, *found = NULL;
 886         int i;
 887
 888         for_each_hist_field(i, hist_data) {
 889                 hist_field = hist_data->fields[i];
 890                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 891                     strcmp(hist_field->var.name, var_name) == 0) {
 892                         found = hist_field;
 893                         break;
 894                 }
 895         }
 896
 897         return found;
 898 }
 899
 900 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 901                                    struct trace_event_file *file,
 902                                    const char *var_name)
 903 {
 904         struct hist_trigger_data *test_data;
 905         struct event_trigger_data *test;
 906         struct hist_field *hist_field;
 907
 908         lockdep_assert_held(&event_mutex);
 909
 910         hist_field = find_var_field(hist_data, var_name);
 911         if (hist_field)
 912                 return hist_field;
 913
 914         list_for_each_entry(test, &file->triggers, list) {
 915                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 916                         test_data = test->private_data;
 917                         hist_field = find_var_field(test_data, var_name);
 918                         if (hist_field)
 919                                 return hist_field;
 920                 }
 921         }
 922
 923         return NULL;
 924 }
 925
 926 static struct trace_event_file *find_var_file(struct trace_array *tr,
 927                                               char *system,
 928                                               char *event_name,
 929                                               char *var_name)
 930 {
 931         struct hist_trigger_data *var_hist_data;
 932         struct hist_var_data *var_data;
 933         struct trace_event_file *file, *found = NULL;
 934
 935         if (system)
 936                 return find_event_file(tr, system, event_name);
 937
 938         list_for_each_entry(var_data, &tr->hist_vars, list) {
 939                 var_hist_data = var_data->hist_data;
 940                 file = var_hist_data->event_file;
 941                 if (file == found)
 942                         continue;
 943
 944                 if (find_var_field(var_hist_data, var_name)) {
 945                         if (found) {
 946                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 947                                 return NULL;
 948                         }
 949
 950                         found = file;
 951                 }
 952         }
 953
 954         return found;
 955 }
 956
 957 static struct hist_field *find_file_var(struct trace_event_file *file,
 958                                         const char *var_name)
 959 {
 960         struct hist_trigger_data *test_data;
 961         struct event_trigger_data *test;
 962         struct hist_field *hist_field;
 963
 964         lockdep_assert_held(&event_mutex);
 965
 966         list_for_each_entry(test, &file->triggers, list) {
 967                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 968                         test_data = test->private_data;
 969                         hist_field = find_var_field(test_data, var_name);
 970                         if (hist_field)
 971                                 return hist_field;
 972                 }
 973         }
 974
 975         return NULL;
 976 }
 977
 978 static struct hist_field *
 979 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 980 {
 981         struct trace_array *tr = hist_data->event_file->tr;
 982         struct hist_field *hist_field, *found = NULL;
 983         struct trace_event_file *file;
 984         unsigned int i;
 985
 986         for (i = 0; i < hist_data->n_actions; i++) {
 987                 struct action_data *data = hist_data->actions[i];
 988
 989                 if (data->handler == HANDLER_ONMATCH) {
 990                         char *system = data->match_data.event_system;
 991                         char *event_name = data->match_data.event;
 992
 993                         file = find_var_file(tr, system, event_name, var_name);
 994                         if (!file)
 995                                 continue;
 996                         hist_field = find_file_var(file, var_name);
 997                         if (hist_field) {
 998                                 if (found) {
 999                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1000                                                  errpos(var_name));
1001                                         return ERR_PTR(-EINVAL);
1002                                 }
1003
1004                                 found = hist_field;
1005                         }
1006                 }
1007         }
1008         return found;
1009 }
1010
1011 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1012                                          char *system,
1013                                          char *event_name,
1014                                          char *var_name)
1015 {
1016         struct trace_array *tr = hist_data->event_file->tr;
1017         struct hist_field *hist_field = NULL;
1018         struct trace_event_file *file;
1019
1020         if (!system || !event_name) {
1021                 hist_field = find_match_var(hist_data, var_name);
1022                 if (IS_ERR(hist_field))
1023                         return NULL;
1024                 if (hist_field)
1025                         return hist_field;
1026         }
1027
1028         file = find_var_file(tr, system, event_name, var_name);
1029         if (!file)
1030                 return NULL;
1031
1032         hist_field = find_file_var(file, var_name);
1033
1034         return hist_field;
1035 }
1036
1037 static u64 hist_field_var_ref(struct hist_field *hist_field,
1038                               struct tracing_map_elt *elt,
1039                               struct trace_buffer *buffer,
1040                               struct ring_buffer_event *rbe,
1041                               void *event)
1042 {
1043         struct hist_elt_data *elt_data;
1044         u64 var_val = 0;
1045
1046         if (WARN_ON_ONCE(!elt))
1047                 return var_val;
1048
1049         elt_data = elt->private_data;
1050         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1051
1052         return var_val;
1053 }
1054
1055 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1056                              u64 *var_ref_vals, bool self)
1057 {
1058         struct hist_trigger_data *var_data;
1059         struct tracing_map_elt *var_elt;
1060         struct hist_field *hist_field;
1061         unsigned int i, var_idx;
1062         bool resolved = true;
1063         u64 var_val = 0;
1064
1065         for (i = 0; i < hist_data->n_var_refs; i++) {
1066                 hist_field = hist_data->var_refs[i];
1067                 var_idx = hist_field->var.idx;
1068                 var_data = hist_field->var.hist_data;
1069
1070                 if (var_data == NULL) {
1071                         resolved = false;
1072                         break;
1073                 }
1074
1075                 if ((self && var_data != hist_data) ||
1076                     (!self && var_data == hist_data))
1077                         continue;
1078
1079                 var_elt = tracing_map_lookup(var_data->map, key);
1080                 if (!var_elt) {
1081                         resolved = false;
1082                         break;
1083                 }
1084
1085                 if (!tracing_map_var_set(var_elt, var_idx)) {
1086                         resolved = false;
1087                         break;
1088                 }
1089
1090                 if (self || !hist_field->read_once)
1091                         var_val = tracing_map_read_var(var_elt, var_idx);
1092                 else
1093                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1094
1095                 var_ref_vals[i] = var_val;
1096         }
1097
1098         return resolved;
1099 }
1100
1101 static const char *hist_field_name(struct hist_field *field,
1102                                    unsigned int level)
1103 {
1104         const char *field_name = "";
1105
1106         if (level > 1)
1107                 return field_name;
1108
1109         if (field->field)
1110                 field_name = field->field->name;
1111         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1112                  field->flags & HIST_FIELD_FL_ALIAS)
1113                 field_name = hist_field_name(field->operands[0], ++level);
1114         else if (field->flags & HIST_FIELD_FL_CPU)
1115                 field_name = "common_cpu";
1116         else if (field->flags & HIST_FIELD_FL_EXPR ||
1117                  field->flags & HIST_FIELD_FL_VAR_REF) {
1118                 if (field->system) {
1119                         static char full_name[MAX_FILTER_STR_VAL];
1120
1121                         strcat(full_name, field->system);
1122                         strcat(full_name, ".");
1123                         strcat(full_name, field->event_name);
1124                         strcat(full_name, ".");
1125                         strcat(full_name, field->name);
1126                         field_name = full_name;
1127                 } else
1128                         field_name = field->name;
1129         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1130                 field_name = "common_timestamp";
1131
1132         if (field_name == NULL)
1133                 field_name = "";
1134
1135         return field_name;
1136 }
1137
1138 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1139 {
1140         hist_field_fn_t fn = NULL;
1141
1142         switch (field_size) {
1143         case 8:
1144                 if (field_is_signed)
1145                         fn = hist_field_s64;
1146                 else
1147                         fn = hist_field_u64;
1148                 break;
1149         case 4:
1150                 if (field_is_signed)
1151                         fn = hist_field_s32;
1152                 else
1153                         fn = hist_field_u32;
1154                 break;
1155         case 2:
1156                 if (field_is_signed)
1157                         fn = hist_field_s16;
1158                 else
1159                         fn = hist_field_u16;
1160                 break;
1161         case 1:
1162                 if (field_is_signed)
1163                         fn = hist_field_s8;
1164                 else
1165                         fn = hist_field_u8;
1166                 break;
1167         }
1168
1169         return fn;
1170 }
1171
1172 static int parse_map_size(char *str)
1173 {
1174         unsigned long size, map_bits;
1175         int ret;
1176
1177         ret = kstrtoul(str, 0, &size);
1178         if (ret)
1179                 goto out;
1180
1181         map_bits = ilog2(roundup_pow_of_two(size));
1182         if (map_bits < TRACING_MAP_BITS_MIN ||
1183             map_bits > TRACING_MAP_BITS_MAX)
1184                 ret = -EINVAL;
1185         else
1186                 ret = map_bits;
1187  out:
1188         return ret;
1189 }
1190
1191 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1192 {
1193         unsigned int i;
1194
1195         if (!attrs)
1196                 return;
1197
1198         for (i = 0; i < attrs->n_assignments; i++)
1199                 kfree(attrs->assignment_str[i]);
1200
1201         for (i = 0; i < attrs->n_actions; i++)
1202                 kfree(attrs->action_str[i]);
1203
1204         kfree(attrs->name);
1205         kfree(attrs->sort_key_str);
1206         kfree(attrs->keys_str);
1207         kfree(attrs->vals_str);
1208         kfree(attrs->clock);
1209         kfree(attrs);
1210 }
1211
1212 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1213 {
1214         int ret = -EINVAL;
1215
1216         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1217                 return ret;
1218
1219         if ((str_has_prefix(str, "onmatch(")) ||
1220             (str_has_prefix(str, "onmax(")) ||
1221             (str_has_prefix(str, "onchange("))) {
1222                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1223                 if (!attrs->action_str[attrs->n_actions]) {
1224                         ret = -ENOMEM;
1225                         return ret;
1226                 }
1227                 attrs->n_actions++;
1228                 ret = 0;
1229         }
1230         return ret;
1231 }
1232
1233 static int parse_assignment(struct trace_array *tr,
1234                             char *str, struct hist_trigger_attrs *attrs)
1235 {
1236         int len, ret = 0;
1237
1238         if ((len = str_has_prefix(str, "key=")) ||
1239             (len = str_has_prefix(str, "keys="))) {
1240                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1241                 if (!attrs->keys_str) {
1242                         ret = -ENOMEM;
1243                         goto out;
1244                 }
1245         } else if ((len = str_has_prefix(str, "val=")) ||
1246                    (len = str_has_prefix(str, "vals=")) ||
1247                    (len = str_has_prefix(str, "values="))) {
1248                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1249                 if (!attrs->vals_str) {
1250                         ret = -ENOMEM;
1251                         goto out;
1252                 }
1253         } else if ((len = str_has_prefix(str, "sort="))) {
1254                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1255                 if (!attrs->sort_key_str) {
1256                         ret = -ENOMEM;
1257                         goto out;
1258                 }
1259         } else if (str_has_prefix(str, "name=")) {
1260                 attrs->name = kstrdup(str, GFP_KERNEL);
1261                 if (!attrs->name) {
1262                         ret = -ENOMEM;
1263                         goto out;
1264                 }
1265         } else if ((len = str_has_prefix(str, "clock="))) {
1266                 str += len;
1267
1268                 str = strstrip(str);
1269                 attrs->clock = kstrdup(str, GFP_KERNEL);
1270                 if (!attrs->clock) {
1271                         ret = -ENOMEM;
1272                         goto out;
1273                 }
1274         } else if ((len = str_has_prefix(str, "size="))) {
1275                 int map_bits = parse_map_size(str + len);
1276
1277                 if (map_bits < 0) {
1278                         ret = map_bits;
1279                         goto out;
1280                 }
1281                 attrs->map_bits = map_bits;
1282         } else {
1283                 char *assignment;
1284
1285                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1286                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1287                         ret = -EINVAL;
1288                         goto out;
1289                 }
1290
1291                 assignment = kstrdup(str, GFP_KERNEL);
1292                 if (!assignment) {
1293                         ret = -ENOMEM;
1294                         goto out;
1295                 }
1296
1297                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1298         }
1299  out:
1300         return ret;
1301 }
1302
1303 static struct hist_trigger_attrs *
1304 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1305 {
1306         struct hist_trigger_attrs *attrs;
1307         int ret = 0;
1308
1309         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1310         if (!attrs)
1311                 return ERR_PTR(-ENOMEM);
1312
1313         while (trigger_str) {
1314                 char *str = strsep(&trigger_str, ":");
1315                 char *rhs;
1316
1317                 rhs = strchr(str, '=');
1318                 if (rhs) {
1319                         if (!strlen(++rhs)) {
1320                                 ret = -EINVAL;
1321                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1322                                 goto free;
1323                         }
1324                         ret = parse_assignment(tr, str, attrs);
1325                         if (ret)
1326                                 goto free;
1327                 } else if (strcmp(str, "pause") == 0)
1328                         attrs->pause = true;
1329                 else if ((strcmp(str, "cont") == 0) ||
1330                          (strcmp(str, "continue") == 0))
1331                         attrs->cont = true;
1332                 else if (strcmp(str, "clear") == 0)
1333                         attrs->clear = true;
1334                 else {
1335                         ret = parse_action(str, attrs);
1336                         if (ret)
1337                                 goto free;
1338                 }
1339         }
1340
1341         if (!attrs->keys_str) {
1342                 ret = -EINVAL;
1343                 goto free;
1344         }
1345
1346         if (!attrs->clock) {
1347                 attrs->clock = kstrdup("global", GFP_KERNEL);
1348                 if (!attrs->clock) {
1349                         ret = -ENOMEM;
1350                         goto free;
1351                 }
1352         }
1353
1354         return attrs;
1355  free:
1356         destroy_hist_trigger_attrs(attrs);
1357
1358         return ERR_PTR(ret);
1359 }
1360
1361 static inline void save_comm(char *comm, struct task_struct *task)
1362 {
1363         if (!task->pid) {
1364                 strcpy(comm, "<idle>");
1365                 return;
1366         }
1367
1368         if (WARN_ON_ONCE(task->pid < 0)) {
1369                 strcpy(comm, "<XXX>");
1370                 return;
1371         }
1372
1373         strncpy(comm, task->comm, TASK_COMM_LEN);
1374 }
1375
1376 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1377 {
1378         unsigned int i;
1379
1380         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1381                 kfree(elt_data->field_var_str[i]);
1382
1383         kfree(elt_data->comm);
1384         kfree(elt_data);
1385 }
1386
1387 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1388 {
1389         struct hist_elt_data *elt_data = elt->private_data;
1390
1391         hist_elt_data_free(elt_data);
1392 }
1393
1394 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1395 {
1396         struct hist_trigger_data *hist_data = elt->map->private_data;
1397         unsigned int size = TASK_COMM_LEN;
1398         struct hist_elt_data *elt_data;
1399         struct hist_field *key_field;
1400         unsigned int i, n_str;
1401
1402         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1403         if (!elt_data)
1404                 return -ENOMEM;
1405
1406         for_each_hist_key_field(i, hist_data) {
1407                 key_field = hist_data->fields[i];
1408
1409                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1410                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1411                         if (!elt_data->comm) {
1412                                 kfree(elt_data);
1413                                 return -ENOMEM;
1414                         }
1415                         break;
1416                 }
1417         }
1418
1419         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1420                 hist_data->n_var_str;
1421         if (n_str > SYNTH_FIELDS_MAX) {
1422                 hist_elt_data_free(elt_data);
1423                 return -EINVAL;
1424         }
1425
1426         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1427
1428         size = STR_VAR_LEN_MAX;
1429
1430         for (i = 0; i < n_str; i++) {
1431                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1432                 if (!elt_data->field_var_str[i]) {
1433                         hist_elt_data_free(elt_data);
1434                         return -ENOMEM;
1435                 }
1436         }
1437
1438         elt->private_data = elt_data;
1439
1440         return 0;
1441 }
1442
1443 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1444 {
1445         struct hist_elt_data *elt_data = elt->private_data;
1446
1447         if (elt_data->comm)
1448                 save_comm(elt_data->comm, current);
1449 }
1450
1451 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1452         .elt_alloc      = hist_trigger_elt_data_alloc,
1453         .elt_free       = hist_trigger_elt_data_free,
1454         .elt_init       = hist_trigger_elt_data_init,
1455 };
1456
1457 static const char *get_hist_field_flags(struct hist_field *hist_field)
1458 {
1459         const char *flags_str = NULL;
1460
1461         if (hist_field->flags & HIST_FIELD_FL_HEX)
1462                 flags_str = "hex";
1463         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1464                 flags_str = "sym";
1465         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1466                 flags_str = "sym-offset";
1467         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1468                 flags_str = "execname";
1469         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1470                 flags_str = "syscall";
1471         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1472                 flags_str = "log2";
1473         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1474                 flags_str = "usecs";
1475
1476         return flags_str;
1477 }
1478
1479 static void expr_field_str(struct hist_field *field, char *expr)
1480 {
1481         if (field->flags & HIST_FIELD_FL_VAR_REF)
1482                 strcat(expr, "$");
1483
1484         strcat(expr, hist_field_name(field, 0));
1485
1486         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1487                 const char *flags_str = get_hist_field_flags(field);
1488
1489                 if (flags_str) {
1490                         strcat(expr, ".");
1491                         strcat(expr, flags_str);
1492                 }
1493         }
1494 }
1495
1496 static char *expr_str(struct hist_field *field, unsigned int level)
1497 {
1498         char *expr;
1499
1500         if (level > 1)
1501                 return NULL;
1502
1503         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1504         if (!expr)
1505                 return NULL;
1506
1507         if (!field->operands[0]) {
1508                 expr_field_str(field, expr);
1509                 return expr;
1510         }
1511
1512         if (field->operator == FIELD_OP_UNARY_MINUS) {
1513                 char *subexpr;
1514
1515                 strcat(expr, "-(");
1516                 subexpr = expr_str(field->operands[0], ++level);
1517                 if (!subexpr) {
1518                         kfree(expr);
1519                         return NULL;
1520                 }
1521                 strcat(expr, subexpr);
1522                 strcat(expr, ")");
1523
1524                 kfree(subexpr);
1525
1526                 return expr;
1527         }
1528
1529         expr_field_str(field->operands[0], expr);
1530
1531         switch (field->operator) {
1532         case FIELD_OP_MINUS:
1533                 strcat(expr, "-");
1534                 break;
1535         case FIELD_OP_PLUS:
1536                 strcat(expr, "+");
1537                 break;
1538         default:
1539                 kfree(expr);
1540                 return NULL;
1541         }
1542
1543         expr_field_str(field->operands[1], expr);
1544
1545         return expr;
1546 }
1547
1548 static int contains_operator(char *str)
1549 {
1550         enum field_op_id field_op = FIELD_OP_NONE;
1551         char *op;
1552
1553         op = strpbrk(str, "+-");
1554         if (!op)
1555                 return FIELD_OP_NONE;
1556
1557         switch (*op) {
1558         case '-':
1559                 /*
1560                  * Unfortunately, the modifier ".sym-offset"
1561                  * can confuse things.
1562                  */
1563                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1564                         return FIELD_OP_NONE;
1565
1566                 if (*str == '-')
1567                         field_op = FIELD_OP_UNARY_MINUS;
1568                 else
1569                         field_op = FIELD_OP_MINUS;
1570                 break;
1571         case '+':
1572                 field_op = FIELD_OP_PLUS;
1573                 break;
1574         default:
1575                 break;
1576         }
1577
1578         return field_op;
1579 }
1580
1581 static void get_hist_field(struct hist_field *hist_field)
1582 {
1583         hist_field->ref++;
1584 }
1585
1586 static void __destroy_hist_field(struct hist_field *hist_field)
1587 {
1588         if (--hist_field->ref > 1)
1589                 return;
1590
1591         kfree(hist_field->var.name);
1592         kfree(hist_field->name);
1593         kfree(hist_field->type);
1594
1595         kfree(hist_field->system);
1596         kfree(hist_field->event_name);
1597
1598         kfree(hist_field);
1599 }
1600
1601 static void destroy_hist_field(struct hist_field *hist_field,
1602                                unsigned int level)
1603 {
1604         unsigned int i;
1605
1606         if (level > 3)
1607                 return;
1608
1609         if (!hist_field)
1610                 return;
1611
1612         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1613                 return; /* var refs will be destroyed separately */
1614
1615         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1616                 destroy_hist_field(hist_field->operands[i], level + 1);
1617
1618         __destroy_hist_field(hist_field);
1619 }
1620
1621 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1622                                             struct ftrace_event_field *field,
1623                                             unsigned long flags,
1624                                             char *var_name)
1625 {
1626         struct hist_field *hist_field;
1627
1628         if (field && is_function_field(field))
1629                 return NULL;
1630
1631         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1632         if (!hist_field)
1633                 return NULL;
1634
1635         hist_field->ref = 1;
1636
1637         hist_field->hist_data = hist_data;
1638
1639         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1640                 goto out; /* caller will populate */
1641
1642         if (flags & HIST_FIELD_FL_VAR_REF) {
1643                 hist_field->fn = hist_field_var_ref;
1644                 goto out;
1645         }
1646
1647         if (flags & HIST_FIELD_FL_HITCOUNT) {
1648                 hist_field->fn = hist_field_counter;
1649                 hist_field->size = sizeof(u64);
1650                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1651                 if (!hist_field->type)
1652                         goto free;
1653                 goto out;
1654         }
1655
1656         if (flags & HIST_FIELD_FL_STACKTRACE) {
1657                 hist_field->fn = hist_field_none;
1658                 goto out;
1659         }
1660
1661         if (flags & HIST_FIELD_FL_LOG2) {
1662                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1663                 hist_field->fn = hist_field_log2;
1664                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1665                 hist_field->size = hist_field->operands[0]->size;
1666                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1667                 if (!hist_field->type)
1668                         goto free;
1669                 goto out;
1670         }
1671
1672         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1673                 hist_field->fn = hist_field_timestamp;
1674                 hist_field->size = sizeof(u64);
1675                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1676                 if (!hist_field->type)
1677                         goto free;
1678                 goto out;
1679         }
1680
1681         if (flags & HIST_FIELD_FL_CPU) {
1682                 hist_field->fn = hist_field_cpu;
1683                 hist_field->size = sizeof(int);
1684                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1685                 if (!hist_field->type)
1686                         goto free;
1687                 goto out;
1688         }
1689
1690         if (WARN_ON_ONCE(!field))
1691                 goto out;
1692
1693         /* Pointers to strings are just pointers and dangerous to dereference */
1694         if (is_string_field(field) &&
1695             (field->filter_type != FILTER_PTR_STRING)) {
1696                 flags |= HIST_FIELD_FL_STRING;
1697
1698                 hist_field->size = MAX_FILTER_STR_VAL;
1699                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1700                 if (!hist_field->type)
1701                         goto free;
1702
1703                 if (field->filter_type == FILTER_STATIC_STRING)
1704                         hist_field->fn = hist_field_string;
1705                 else if (field->filter_type == FILTER_DYN_STRING)
1706                         hist_field->fn = hist_field_dynstring;
1707                 else
1708                         hist_field->fn = hist_field_pstring;
1709         } else {
1710                 hist_field->size = field->size;
1711                 hist_field->is_signed = field->is_signed;
1712                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1713                 if (!hist_field->type)
1714                         goto free;
1715
1716                 hist_field->fn = select_value_fn(field->size,
1717                                                  field->is_signed);
1718                 if (!hist_field->fn) {
1719                         destroy_hist_field(hist_field, 0);
1720                         return NULL;
1721                 }
1722         }
1723  out:
1724         hist_field->field = field;
1725         hist_field->flags = flags;
1726
1727         if (var_name) {
1728                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1729                 if (!hist_field->var.name)
1730                         goto free;
1731         }
1732
1733         return hist_field;
1734  free:
1735         destroy_hist_field(hist_field, 0);
1736         return NULL;
1737 }
1738
1739 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1740 {
1741         unsigned int i;
1742
1743         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1744                 if (hist_data->fields[i]) {
1745                         destroy_hist_field(hist_data->fields[i], 0);
1746                         hist_data->fields[i] = NULL;
1747                 }
1748         }
1749
1750         for (i = 0; i < hist_data->n_var_refs; i++) {
1751                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1752                 __destroy_hist_field(hist_data->var_refs[i]);
1753                 hist_data->var_refs[i] = NULL;
1754         }
1755 }
1756
1757 static int init_var_ref(struct hist_field *ref_field,
1758                         struct hist_field *var_field,
1759                         char *system, char *event_name)
1760 {
1761         int err = 0;
1762
1763         ref_field->var.idx = var_field->var.idx;
1764         ref_field->var.hist_data = var_field->hist_data;
1765         ref_field->size = var_field->size;
1766         ref_field->is_signed = var_field->is_signed;
1767         ref_field->flags |= var_field->flags &
1768                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1769
1770         if (system) {
1771                 ref_field->system = kstrdup(system, GFP_KERNEL);
1772                 if (!ref_field->system)
1773                         return -ENOMEM;
1774         }
1775
1776         if (event_name) {
1777                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1778                 if (!ref_field->event_name) {
1779                         err = -ENOMEM;
1780                         goto free;
1781                 }
1782         }
1783
1784         if (var_field->var.name) {
1785                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1786                 if (!ref_field->name) {
1787                         err = -ENOMEM;
1788                         goto free;
1789                 }
1790         } else if (var_field->name) {
1791                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1792                 if (!ref_field->name) {
1793                         err = -ENOMEM;
1794                         goto free;
1795                 }
1796         }
1797
1798         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1799         if (!ref_field->type) {
1800                 err = -ENOMEM;
1801                 goto free;
1802         }
1803  out:
1804         return err;
1805  free:
1806         kfree(ref_field->system);
1807         kfree(ref_field->event_name);
1808         kfree(ref_field->name);
1809
1810         goto out;
1811 }
1812
1813 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1814                             struct hist_field *var_field)
1815 {
1816         struct hist_field *ref_field;
1817         int i;
1818
1819         for (i = 0; i < hist_data->n_var_refs; i++) {
1820                 ref_field = hist_data->var_refs[i];
1821                 if (ref_field->var.idx == var_field->var.idx &&
1822                     ref_field->var.hist_data == var_field->hist_data)
1823                         return i;
1824         }
1825
1826         return -ENOENT;
1827 }
1828
1829 /**
1830  * create_var_ref - Create a variable reference and attach it to trigger
1831  * @hist_data: The trigger that will be referencing the variable
1832  * @var_field: The VAR field to create a reference to
1833  * @system: The optional system string
1834  * @event_name: The optional event_name string
1835  *
1836  * Given a variable hist_field, create a VAR_REF hist_field that
1837  * represents a reference to it.
1838  *
1839  * This function also adds the reference to the trigger that
1840  * now references the variable.
1841  *
1842  * Return: The VAR_REF field if successful, NULL if not
1843  */
1844 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1845                                          struct hist_field *var_field,
1846                                          char *system, char *event_name)
1847 {
1848         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1849         struct hist_field *ref_field;
1850         int i;
1851
1852         /* Check if the variable already exists */
1853         for (i = 0; i < hist_data->n_var_refs; i++) {
1854                 ref_field = hist_data->var_refs[i];
1855                 if (ref_field->var.idx == var_field->var.idx &&
1856                     ref_field->var.hist_data == var_field->hist_data) {
1857                         get_hist_field(ref_field);
1858                         return ref_field;
1859                 }
1860         }
1861
1862         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1863         if (ref_field) {
1864                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1865                         destroy_hist_field(ref_field, 0);
1866                         return NULL;
1867                 }
1868
1869                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1870                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1871         }
1872
1873         return ref_field;
1874 }
1875
1876 static bool is_var_ref(char *var_name)
1877 {
1878         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1879                 return false;
1880
1881         return true;
1882 }
1883
1884 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1885                                  char *var_name)
1886 {
1887         char *name, *field;
1888         unsigned int i;
1889
1890         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1891                 name = hist_data->attrs->var_defs.name[i];
1892
1893                 if (strcmp(var_name, name) == 0) {
1894                         field = hist_data->attrs->var_defs.expr[i];
1895                         if (contains_operator(field) || is_var_ref(field))
1896                                 continue;
1897                         return field;
1898                 }
1899         }
1900
1901         return NULL;
1902 }
1903
1904 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1905                                  char *system, char *event_name,
1906                                  char *var_name)
1907 {
1908         struct trace_event_call *call;
1909
1910         if (system && event_name) {
1911                 call = hist_data->event_file->event_call;
1912
1913                 if (strcmp(system, call->class->system) != 0)
1914                         return NULL;
1915
1916                 if (strcmp(event_name, trace_event_name(call)) != 0)
1917                         return NULL;
1918         }
1919
1920         if (!!system != !!event_name)
1921                 return NULL;
1922
1923         if (!is_var_ref(var_name))
1924                 return NULL;
1925
1926         var_name++;
1927
1928         return field_name_from_var(hist_data, var_name);
1929 }
1930
1931 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1932                                         char *system, char *event_name,
1933                                         char *var_name)
1934 {
1935         struct hist_field *var_field = NULL, *ref_field = NULL;
1936         struct trace_array *tr = hist_data->event_file->tr;
1937
1938         if (!is_var_ref(var_name))
1939                 return NULL;
1940
1941         var_name++;
1942
1943         var_field = find_event_var(hist_data, system, event_name, var_name);
1944         if (var_field)
1945                 ref_field = create_var_ref(hist_data, var_field,
1946                                            system, event_name);
1947
1948         if (!ref_field)
1949                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1950
1951         return ref_field;
1952 }
1953
1954 static struct ftrace_event_field *
1955 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1956             char *field_str, unsigned long *flags)
1957 {
1958         struct ftrace_event_field *field = NULL;
1959         char *field_name, *modifier, *str;
1960         struct trace_array *tr = file->tr;
1961
1962         modifier = str = kstrdup(field_str, GFP_KERNEL);
1963         if (!modifier)
1964                 return ERR_PTR(-ENOMEM);
1965
1966         field_name = strsep(&modifier, ".");
1967         if (modifier) {
1968                 if (strcmp(modifier, "hex") == 0)
1969                         *flags |= HIST_FIELD_FL_HEX;
1970                 else if (strcmp(modifier, "sym") == 0)
1971                         *flags |= HIST_FIELD_FL_SYM;
1972                 else if (strcmp(modifier, "sym-offset") == 0)
1973                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1974                 else if ((strcmp(modifier, "execname") == 0) &&
1975                          (strcmp(field_name, "common_pid") == 0))
1976                         *flags |= HIST_FIELD_FL_EXECNAME;
1977                 else if (strcmp(modifier, "syscall") == 0)
1978                         *flags |= HIST_FIELD_FL_SYSCALL;
1979                 else if (strcmp(modifier, "log2") == 0)
1980                         *flags |= HIST_FIELD_FL_LOG2;
1981                 else if (strcmp(modifier, "usecs") == 0)
1982                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1983                 else {
1984                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1985                         field = ERR_PTR(-EINVAL);
1986                         goto out;
1987                 }
1988         }
1989
1990         if (strcmp(field_name, "common_timestamp") == 0) {
1991                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1992                 hist_data->enable_timestamps = true;
1993                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1994                         hist_data->attrs->ts_in_usecs = true;
1995         } else if (strcmp(field_name, "common_cpu") == 0)
1996                 *flags |= HIST_FIELD_FL_CPU;
1997         else {
1998                 field = trace_find_event_field(file->event_call, field_name);
1999                 if (!field || !field->size) {
2000                         /*
2001                          * For backward compatibility, if field_name
2002                          * was "cpu", then we treat this the same as
2003                          * common_cpu.
2004                          */
2005                         if (strcmp(field_name, "cpu") == 0) {
2006                                 *flags |= HIST_FIELD_FL_CPU;
2007                         } else {
2008                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2009                                          errpos(field_name));
2010                                 field = ERR_PTR(-EINVAL);
2011                                 goto out;
2012                         }
2013                 }
2014         }
2015  out:
2016         kfree(str);
2017
2018         return field;
2019 }
2020
2021 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2022                                        struct hist_field *var_ref,
2023                                        char *var_name)
2024 {
2025         struct hist_field *alias = NULL;
2026         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2027
2028         alias = create_hist_field(hist_data, NULL, flags, var_name);
2029         if (!alias)
2030                 return NULL;
2031
2032         alias->fn = var_ref->fn;
2033         alias->operands[0] = var_ref;
2034
2035         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2036                 destroy_hist_field(alias, 0);
2037                 return NULL;
2038         }
2039
2040         alias->var_ref_idx = var_ref->var_ref_idx;
2041
2042         return alias;
2043 }
2044
2045 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2046                                      struct trace_event_file *file, char *str,
2047                                      unsigned long *flags, char *var_name)
2048 {
2049         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2050         struct ftrace_event_field *field = NULL;
2051         struct hist_field *hist_field = NULL;
2052         int ret = 0;
2053
2054         s = strchr(str, '.');
2055         if (s) {
2056                 s = strchr(++s, '.');
2057                 if (s) {
2058                         ref_system = strsep(&str, ".");
2059                         if (!str) {
2060                                 ret = -EINVAL;
2061                                 goto out;
2062                         }
2063                         ref_event = strsep(&str, ".");
2064                         if (!str) {
2065                                 ret = -EINVAL;
2066                                 goto out;
2067                         }
2068                         ref_var = str;
2069                 }
2070         }
2071
2072         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2073         if (!s) {
2074                 hist_field = parse_var_ref(hist_data, ref_system,
2075                                            ref_event, ref_var);
2076                 if (hist_field) {
2077                         if (var_name) {
2078                                 hist_field = create_alias(hist_data, hist_field, var_name);
2079                                 if (!hist_field) {
2080                                         ret = -ENOMEM;
2081                                         goto out;
2082                                 }
2083                         }
2084                         return hist_field;
2085                 }
2086         } else
2087                 str = s;
2088
2089         field = parse_field(hist_data, file, str, flags);
2090         if (IS_ERR(field)) {
2091                 ret = PTR_ERR(field);
2092                 goto out;
2093         }
2094
2095         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2096         if (!hist_field) {
2097                 ret = -ENOMEM;
2098                 goto out;
2099         }
2100
2101         return hist_field;
2102  out:
2103         return ERR_PTR(ret);
2104 }
2105
2106 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2107                                      struct trace_event_file *file,
2108                                      char *str, unsigned long flags,
2109                                      char *var_name, unsigned int level);
2110
2111 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2112                                       struct trace_event_file *file,
2113                                       char *str, unsigned long flags,
2114                                       char *var_name, unsigned int level)
2115 {
2116         struct hist_field *operand1, *expr = NULL;
2117         unsigned long operand_flags;
2118         int ret = 0;
2119         char *s;
2120
2121         /* we support only -(xxx) i.e. explicit parens required */
2122
2123         if (level > 3) {
2124                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2125                 ret = -EINVAL;
2126                 goto free;
2127         }
2128
2129         str++; /* skip leading '-' */
2130
2131         s = strchr(str, '(');
2132         if (s)
2133                 str++;
2134         else {
2135                 ret = -EINVAL;
2136                 goto free;
2137         }
2138
2139         s = strrchr(str, ')');
2140         if (s)
2141                 *s = '\0';
2142         else {
2143                 ret = -EINVAL; /* no closing ')' */
2144                 goto free;
2145         }
2146
2147         flags |= HIST_FIELD_FL_EXPR;
2148         expr = create_hist_field(hist_data, NULL, flags, var_name);
2149         if (!expr) {
2150                 ret = -ENOMEM;
2151                 goto free;
2152         }
2153
2154         operand_flags = 0;
2155         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2156         if (IS_ERR(operand1)) {
2157                 ret = PTR_ERR(operand1);
2158                 goto free;
2159         }
2160         if (operand1->flags & HIST_FIELD_FL_STRING) {
2161                 /* String type can not be the operand of unary operator. */
2162                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2163                 destroy_hist_field(operand1, 0);
2164                 ret = -EINVAL;
2165                 goto free;
2166         }
2167
2168         expr->flags |= operand1->flags &
2169                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2170         expr->fn = hist_field_unary_minus;
2171         expr->operands[0] = operand1;
2172         expr->operator = FIELD_OP_UNARY_MINUS;
2173         expr->name = expr_str(expr, 0);
2174         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2175         if (!expr->type) {
2176                 ret = -ENOMEM;
2177                 goto free;
2178         }
2179
2180         return expr;
2181  free:
2182         destroy_hist_field(expr, 0);
2183         return ERR_PTR(ret);
2184 }
2185
2186 static int check_expr_operands(struct trace_array *tr,
2187                                struct hist_field *operand1,
2188                                struct hist_field *operand2)
2189 {
2190         unsigned long operand1_flags = operand1->flags;
2191         unsigned long operand2_flags = operand2->flags;
2192
2193         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2194             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2195                 struct hist_field *var;
2196
2197                 var = find_var_field(operand1->var.hist_data, operand1->name);
2198                 if (!var)
2199                         return -EINVAL;
2200                 operand1_flags = var->flags;
2201         }
2202
2203         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2204             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2205                 struct hist_field *var;
2206
2207                 var = find_var_field(operand2->var.hist_data, operand2->name);
2208                 if (!var)
2209                         return -EINVAL;
2210                 operand2_flags = var->flags;
2211         }
2212
2213         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2214             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2215                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2216                 return -EINVAL;
2217         }
2218
2219         return 0;
2220 }
2221
2222 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2223                                      struct trace_event_file *file,
2224                                      char *str, unsigned long flags,
2225                                      char *var_name, unsigned int level)
2226 {
2227         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2228         unsigned long operand_flags;
2229         int field_op, ret = -EINVAL;
2230         char *sep, *operand1_str;
2231
2232         if (level > 3) {
2233                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2234                 return ERR_PTR(-EINVAL);
2235         }
2236
2237         field_op = contains_operator(str);
2238
2239         if (field_op == FIELD_OP_NONE)
2240                 return parse_atom(hist_data, file, str, &flags, var_name);
2241
2242         if (field_op == FIELD_OP_UNARY_MINUS)
2243                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2244
2245         switch (field_op) {
2246         case FIELD_OP_MINUS:
2247                 sep = "-";
2248                 break;
2249         case FIELD_OP_PLUS:
2250                 sep = "+";
2251                 break;
2252         default:
2253                 goto free;
2254         }
2255
2256         operand1_str = strsep(&str, sep);
2257         if (!operand1_str || !str)
2258                 goto free;
2259
2260         operand_flags = 0;
2261         operand1 = parse_atom(hist_data, file, operand1_str,
2262                               &operand_flags, NULL);
2263         if (IS_ERR(operand1)) {
2264                 ret = PTR_ERR(operand1);
2265                 operand1 = NULL;
2266                 goto free;
2267         }
2268         if (operand1->flags & HIST_FIELD_FL_STRING) {
2269                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2270                 ret = -EINVAL;
2271                 goto free;
2272         }
2273
2274         /* rest of string could be another expression e.g. b+c in a+b+c */
2275         operand_flags = 0;
2276         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2277         if (IS_ERR(operand2)) {
2278                 ret = PTR_ERR(operand2);
2279                 operand2 = NULL;
2280                 goto free;
2281         }
2282         if (operand2->flags & HIST_FIELD_FL_STRING) {
2283                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2284                 ret = -EINVAL;
2285                 goto free;
2286         }
2287
2288         ret = check_expr_operands(file->tr, operand1, operand2);
2289         if (ret)
2290                 goto free;
2291
2292         flags |= HIST_FIELD_FL_EXPR;
2293
2294         flags |= operand1->flags &
2295                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2296
2297         expr = create_hist_field(hist_data, NULL, flags, var_name);
2298         if (!expr) {
2299                 ret = -ENOMEM;
2300                 goto free;
2301         }
2302
2303         operand1->read_once = true;
2304         operand2->read_once = true;
2305
2306         expr->operands[0] = operand1;
2307         expr->operands[1] = operand2;
2308
2309         /* The operand sizes should be the same, so just pick one */
2310         expr->size = operand1->size;
2311
2312         expr->operator = field_op;
2313         expr->name = expr_str(expr, 0);
2314         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2315         if (!expr->type) {
2316                 ret = -ENOMEM;
2317                 goto free;
2318         }
2319
2320         switch (field_op) {
2321         case FIELD_OP_MINUS:
2322                 expr->fn = hist_field_minus;
2323                 break;
2324         case FIELD_OP_PLUS:
2325                 expr->fn = hist_field_plus;
2326                 break;
2327         default:
2328                 ret = -EINVAL;
2329                 goto free;
2330         }
2331
2332         return expr;
2333  free:
2334         destroy_hist_field(operand1, 0);
2335         destroy_hist_field(operand2, 0);
2336         destroy_hist_field(expr, 0);
2337
2338         return ERR_PTR(ret);
2339 }
2340
2341 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2342                                  struct trace_event_file *file)
2343 {
2344         struct event_trigger_data *test;
2345
2346         lockdep_assert_held(&event_mutex);
2347
2348         list_for_each_entry(test, &file->triggers, list) {
2349                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2350                         if (test->private_data == hist_data)
2351                                 return test->filter_str;
2352                 }
2353         }
2354
2355         return NULL;
2356 }
2357
2358 static struct event_command trigger_hist_cmd;
2359 static int event_hist_trigger_func(struct event_command *cmd_ops,
2360                                    struct trace_event_file *file,
2361                                    char *glob, char *cmd, char *param);
2362
2363 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2364                             struct hist_trigger_data *hist_data,
2365                             unsigned int n_keys)
2366 {
2367         struct hist_field *target_hist_field, *hist_field;
2368         unsigned int n, i, j;
2369
2370         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2371                 return false;
2372
2373         i = hist_data->n_vals;
2374         j = target_hist_data->n_vals;
2375
2376         for (n = 0; n < n_keys; n++) {
2377                 hist_field = hist_data->fields[i + n];
2378                 target_hist_field = target_hist_data->fields[j + n];
2379
2380                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2381                         return false;
2382                 if (hist_field->size != target_hist_field->size)
2383                         return false;
2384                 if (hist_field->is_signed != target_hist_field->is_signed)
2385                         return false;
2386         }
2387
2388         return true;
2389 }
2390
2391 static struct hist_trigger_data *
2392 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2393                      struct trace_event_file *file)
2394 {
2395         struct hist_trigger_data *hist_data;
2396         struct event_trigger_data *test;
2397         unsigned int n_keys;
2398
2399         lockdep_assert_held(&event_mutex);
2400
2401         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2402
2403         list_for_each_entry(test, &file->triggers, list) {
2404                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2405                         hist_data = test->private_data;
2406
2407                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2408                                 return hist_data;
2409                 }
2410         }
2411
2412         return NULL;
2413 }
2414
2415 static struct trace_event_file *event_file(struct trace_array *tr,
2416                                            char *system, char *event_name)
2417 {
2418         struct trace_event_file *file;
2419
2420         file = __find_event_file(tr, system, event_name);
2421         if (!file)
2422                 return ERR_PTR(-EINVAL);
2423
2424         return file;
2425 }
2426
2427 static struct hist_field *
2428 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2429                          char *system, char *event_name, char *field_name)
2430 {
2431         struct hist_field *event_var;
2432         char *synthetic_name;
2433
2434         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2435         if (!synthetic_name)
2436                 return ERR_PTR(-ENOMEM);
2437
2438         strcpy(synthetic_name, "synthetic_");
2439         strcat(synthetic_name, field_name);
2440
2441         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2442
2443         kfree(synthetic_name);
2444
2445         return event_var;
2446 }
2447
2448 /**
2449  * create_field_var_hist - Automatically create a histogram and var for a field
2450  * @target_hist_data: The target hist trigger
2451  * @subsys_name: Optional subsystem name
2452  * @event_name: Optional event name
2453  * @field_name: The name of the field (and the resulting variable)
2454  *
2455  * Hist trigger actions fetch data from variables, not directly from
2456  * events.  However, for convenience, users are allowed to directly
2457  * specify an event field in an action, which will be automatically
2458  * converted into a variable on their behalf.
2459
2460  * If a user specifies a field on an event that isn't the event the
2461  * histogram currently being defined (the target event histogram), the
2462  * only way that can be accomplished is if a new hist trigger is
2463  * created and the field variable defined on that.
2464  *
2465  * This function creates a new histogram compatible with the target
2466  * event (meaning a histogram with the same key as the target
2467  * histogram), and creates a variable for the specified field, but
2468  * with 'synthetic_' prepended to the variable name in order to avoid
2469  * collision with normal field variables.
2470  *
2471  * Return: The variable created for the field.
2472  */
2473 static struct hist_field *
2474 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2475                       char *subsys_name, char *event_name, char *field_name)
2476 {
2477         struct trace_array *tr = target_hist_data->event_file->tr;
2478         struct hist_trigger_data *hist_data;
2479         unsigned int i, n, first = true;
2480         struct field_var_hist *var_hist;
2481         struct trace_event_file *file;
2482         struct hist_field *key_field;
2483         struct hist_field *event_var;
2484         char *saved_filter;
2485         char *cmd;
2486         int ret;
2487
2488         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2489                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2490                 return ERR_PTR(-EINVAL);
2491         }
2492
2493         file = event_file(tr, subsys_name, event_name);
2494
2495         if (IS_ERR(file)) {
2496                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2497                 ret = PTR_ERR(file);
2498                 return ERR_PTR(ret);
2499         }
2500
2501         /*
2502          * Look for a histogram compatible with target.  We'll use the
2503          * found histogram specification to create a new matching
2504          * histogram with our variable on it.  target_hist_data is not
2505          * yet a registered histogram so we can't use that.
2506          */
2507         hist_data = find_compatible_hist(target_hist_data, file);
2508         if (!hist_data) {
2509                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2510                 return ERR_PTR(-EINVAL);
2511         }
2512
2513         /* See if a synthetic field variable has already been created */
2514         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2515                                              event_name, field_name);
2516         if (!IS_ERR_OR_NULL(event_var))
2517                 return event_var;
2518
2519         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2520         if (!var_hist)
2521                 return ERR_PTR(-ENOMEM);
2522
2523         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2524         if (!cmd) {
2525                 kfree(var_hist);
2526                 return ERR_PTR(-ENOMEM);
2527         }
2528
2529         /* Use the same keys as the compatible histogram */
2530         strcat(cmd, "keys=");
2531
2532         for_each_hist_key_field(i, hist_data) {
2533                 key_field = hist_data->fields[i];
2534                 if (!first)
2535                         strcat(cmd, ",");
2536                 strcat(cmd, key_field->field->name);
2537                 first = false;
2538         }
2539
2540         /* Create the synthetic field variable specification */
2541         strcat(cmd, ":synthetic_");
2542         strcat(cmd, field_name);
2543         strcat(cmd, "=");
2544         strcat(cmd, field_name);
2545
2546         /* Use the same filter as the compatible histogram */
2547         saved_filter = find_trigger_filter(hist_data, file);
2548         if (saved_filter) {
2549                 strcat(cmd, " if ");
2550                 strcat(cmd, saved_filter);
2551         }
2552
2553         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2554         if (!var_hist->cmd) {
2555                 kfree(cmd);
2556                 kfree(var_hist);
2557                 return ERR_PTR(-ENOMEM);
2558         }
2559
2560         /* Save the compatible histogram information */
2561         var_hist->hist_data = hist_data;
2562
2563         /* Create the new histogram with our variable */
2564         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2565                                       "", "hist", cmd);
2566         if (ret) {
2567                 kfree(cmd);
2568                 kfree(var_hist->cmd);
2569                 kfree(var_hist);
2570                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2571                 return ERR_PTR(ret);
2572         }
2573
2574         kfree(cmd);
2575
2576         /* If we can't find the variable, something went wrong */
2577         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2578                                              event_name, field_name);
2579         if (IS_ERR_OR_NULL(event_var)) {
2580                 kfree(var_hist->cmd);
2581                 kfree(var_hist);
2582                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2583                 return ERR_PTR(-EINVAL);
2584         }
2585
2586         n = target_hist_data->n_field_var_hists;
2587         target_hist_data->field_var_hists[n] = var_hist;
2588         target_hist_data->n_field_var_hists++;
2589
2590         return event_var;
2591 }
2592
2593 static struct hist_field *
2594 find_target_event_var(struct hist_trigger_data *hist_data,
2595                       char *subsys_name, char *event_name, char *var_name)
2596 {
2597         struct trace_event_file *file = hist_data->event_file;
2598         struct hist_field *hist_field = NULL;
2599
2600         if (subsys_name) {
2601                 struct trace_event_call *call;
2602
2603                 if (!event_name)
2604                         return NULL;
2605
2606                 call = file->event_call;
2607
2608                 if (strcmp(subsys_name, call->class->system) != 0)
2609                         return NULL;
2610
2611                 if (strcmp(event_name, trace_event_name(call)) != 0)
2612                         return NULL;
2613         }
2614
2615         hist_field = find_var_field(hist_data, var_name);
2616
2617         return hist_field;
2618 }
2619
2620 static inline void __update_field_vars(struct tracing_map_elt *elt,
2621                                        struct trace_buffer *buffer,
2622                                        struct ring_buffer_event *rbe,
2623                                        void *rec,
2624                                        struct field_var **field_vars,
2625                                        unsigned int n_field_vars,
2626                                        unsigned int field_var_str_start)
2627 {
2628         struct hist_elt_data *elt_data = elt->private_data;
2629         unsigned int i, j, var_idx;
2630         u64 var_val;
2631
2632         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2633                 struct field_var *field_var = field_vars[i];
2634                 struct hist_field *var = field_var->var;
2635                 struct hist_field *val = field_var->val;
2636
2637                 var_val = val->fn(val, elt, buffer, rbe, rec);
2638                 var_idx = var->var.idx;
2639
2640                 if (val->flags & HIST_FIELD_FL_STRING) {
2641                         char *str = elt_data->field_var_str[j++];
2642                         char *val_str = (char *)(uintptr_t)var_val;
2643
2644                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2645                         var_val = (u64)(uintptr_t)str;
2646                 }
2647                 tracing_map_set_var(elt, var_idx, var_val);
2648         }
2649 }
2650
2651 static void update_field_vars(struct hist_trigger_data *hist_data,
2652                               struct tracing_map_elt *elt,
2653                               struct trace_buffer *buffer,
2654                               struct ring_buffer_event *rbe,
2655                               void *rec)
2656 {
2657         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2658                             hist_data->n_field_vars, 0);
2659 }
2660
2661 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2662                                  struct tracing_map_elt *elt,
2663                                  struct trace_buffer *buffer,  void *rec,
2664                                  struct ring_buffer_event *rbe, void *key,
2665                                  struct action_data *data, u64 *var_ref_vals)
2666 {
2667         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2668                             hist_data->n_save_vars, hist_data->n_field_var_str);
2669 }
2670
2671 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2672                                      struct trace_event_file *file,
2673                                      char *name, int size, const char *type)
2674 {
2675         struct hist_field *var;
2676         int idx;
2677
2678         if (find_var(hist_data, file, name) && !hist_data->remove) {
2679                 var = ERR_PTR(-EINVAL);
2680                 goto out;
2681         }
2682
2683         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2684         if (!var) {
2685                 var = ERR_PTR(-ENOMEM);
2686                 goto out;
2687         }
2688
2689         idx = tracing_map_add_var(hist_data->map);
2690         if (idx < 0) {
2691                 kfree(var);
2692                 var = ERR_PTR(-EINVAL);
2693                 goto out;
2694         }
2695
2696         var->ref = 1;
2697         var->flags = HIST_FIELD_FL_VAR;
2698         var->var.idx = idx;
2699         var->var.hist_data = var->hist_data = hist_data;
2700         var->size = size;
2701         var->var.name = kstrdup(name, GFP_KERNEL);
2702         var->type = kstrdup(type, GFP_KERNEL);
2703         if (!var->var.name || !var->type) {
2704                 kfree(var->var.name);
2705                 kfree(var->type);
2706                 kfree(var);
2707                 var = ERR_PTR(-ENOMEM);
2708         }
2709  out:
2710         return var;
2711 }
2712
2713 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2714                                           struct trace_event_file *file,
2715                                           char *field_name)
2716 {
2717         struct hist_field *val = NULL, *var = NULL;
2718         unsigned long flags = HIST_FIELD_FL_VAR;
2719         struct trace_array *tr = file->tr;
2720         struct field_var *field_var;
2721         int ret = 0;
2722
2723         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2724                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2725                 ret = -EINVAL;
2726                 goto err;
2727         }
2728
2729         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2730         if (IS_ERR(val)) {
2731                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2732                 ret = PTR_ERR(val);
2733                 goto err;
2734         }
2735
2736         var = create_var(hist_data, file, field_name, val->size, val->type);
2737         if (IS_ERR(var)) {
2738                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2739                 kfree(val);
2740                 ret = PTR_ERR(var);
2741                 goto err;
2742         }
2743
2744         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2745         if (!field_var) {
2746                 kfree(val);
2747                 kfree(var);
2748                 ret =  -ENOMEM;
2749                 goto err;
2750         }
2751
2752         field_var->var = var;
2753         field_var->val = val;
2754  out:
2755         return field_var;
2756  err:
2757         field_var = ERR_PTR(ret);
2758         goto out;
2759 }
2760
2761 /**
2762  * create_target_field_var - Automatically create a variable for a field
2763  * @target_hist_data: The target hist trigger
2764  * @subsys_name: Optional subsystem name
2765  * @event_name: Optional event name
2766  * @var_name: The name of the field (and the resulting variable)
2767  *
2768  * Hist trigger actions fetch data from variables, not directly from
2769  * events.  However, for convenience, users are allowed to directly
2770  * specify an event field in an action, which will be automatically
2771  * converted into a variable on their behalf.
2772
2773  * This function creates a field variable with the name var_name on
2774  * the hist trigger currently being defined on the target event.  If
2775  * subsys_name and event_name are specified, this function simply
2776  * verifies that they do in fact match the target event subsystem and
2777  * event name.
2778  *
2779  * Return: The variable created for the field.
2780  */
2781 static struct field_var *
2782 create_target_field_var(struct hist_trigger_data *target_hist_data,
2783                         char *subsys_name, char *event_name, char *var_name)
2784 {
2785         struct trace_event_file *file = target_hist_data->event_file;
2786
2787         if (subsys_name) {
2788                 struct trace_event_call *call;
2789
2790                 if (!event_name)
2791                         return NULL;
2792
2793                 call = file->event_call;
2794
2795                 if (strcmp(subsys_name, call->class->system) != 0)
2796                         return NULL;
2797
2798                 if (strcmp(event_name, trace_event_name(call)) != 0)
2799                         return NULL;
2800         }
2801
2802         return create_field_var(target_hist_data, file, var_name);
2803 }
2804
2805 static bool check_track_val_max(u64 track_val, u64 var_val)
2806 {
2807         if (var_val <= track_val)
2808                 return false;
2809
2810         return true;
2811 }
2812
2813 static bool check_track_val_changed(u64 track_val, u64 var_val)
2814 {
2815         if (var_val == track_val)
2816                 return false;
2817
2818         return true;
2819 }
2820
2821 static u64 get_track_val(struct hist_trigger_data *hist_data,
2822                          struct tracing_map_elt *elt,
2823                          struct action_data *data)
2824 {
2825         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2826         u64 track_val;
2827
2828         track_val = tracing_map_read_var(elt, track_var_idx);
2829
2830         return track_val;
2831 }
2832
2833 static void save_track_val(struct hist_trigger_data *hist_data,
2834                            struct tracing_map_elt *elt,
2835                            struct action_data *data, u64 var_val)
2836 {
2837         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2838
2839         tracing_map_set_var(elt, track_var_idx, var_val);
2840 }
2841
2842 static void save_track_data(struct hist_trigger_data *hist_data,
2843                             struct tracing_map_elt *elt,
2844                             struct trace_buffer *buffer, void *rec,
2845                             struct ring_buffer_event *rbe, void *key,
2846                             struct action_data *data, u64 *var_ref_vals)
2847 {
2848         if (data->track_data.save_data)
2849                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2850                                            key, data, var_ref_vals);
2851 }
2852
2853 static bool check_track_val(struct tracing_map_elt *elt,
2854                             struct action_data *data,
2855                             u64 var_val)
2856 {
2857         struct hist_trigger_data *hist_data;
2858         u64 track_val;
2859
2860         hist_data = data->track_data.track_var->hist_data;
2861         track_val = get_track_val(hist_data, elt, data);
2862
2863         return data->track_data.check_val(track_val, var_val);
2864 }
2865
2866 #ifdef CONFIG_TRACER_SNAPSHOT
2867 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2868 {
2869         /* called with tr->max_lock held */
2870         struct track_data *track_data = tr->cond_snapshot->cond_data;
2871         struct hist_elt_data *elt_data, *track_elt_data;
2872         struct snapshot_context *context = cond_data;
2873         struct action_data *action;
2874         u64 track_val;
2875
2876         if (!track_data)
2877                 return false;
2878
2879         action = track_data->action_data;
2880
2881         track_val = get_track_val(track_data->hist_data, context->elt,
2882                                   track_data->action_data);
2883
2884         if (!action->track_data.check_val(track_data->track_val, track_val))
2885                 return false;
2886
2887         track_data->track_val = track_val;
2888         memcpy(track_data->key, context->key, track_data->key_len);
2889
2890         elt_data = context->elt->private_data;
2891         track_elt_data = track_data->elt.private_data;
2892         if (elt_data->comm)
2893                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2894
2895         track_data->updated = true;
2896
2897         return true;
2898 }
2899
2900 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2901                                      struct tracing_map_elt *elt,
2902                                      struct trace_buffer *buffer, void *rec,
2903                                      struct ring_buffer_event *rbe, void *key,
2904                                      struct action_data *data,
2905                                      u64 *var_ref_vals)
2906 {
2907         struct trace_event_file *file = hist_data->event_file;
2908         struct snapshot_context context;
2909
2910         context.elt = elt;
2911         context.key = key;
2912
2913         tracing_snapshot_cond(file->tr, &context);
2914 }
2915
2916 static void hist_trigger_print_key(struct seq_file *m,
2917                                    struct hist_trigger_data *hist_data,
2918                                    void *key,
2919                                    struct tracing_map_elt *elt);
2920
2921 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2922 {
2923         unsigned int i;
2924
2925         if (!hist_data->n_actions)
2926                 return NULL;
2927
2928         for (i = 0; i < hist_data->n_actions; i++) {
2929                 struct action_data *data = hist_data->actions[i];
2930
2931                 if (data->action == ACTION_SNAPSHOT)
2932                         return data;
2933         }
2934
2935         return NULL;
2936 }
2937
2938 static void track_data_snapshot_print(struct seq_file *m,
2939                                       struct hist_trigger_data *hist_data)
2940 {
2941         struct trace_event_file *file = hist_data->event_file;
2942         struct track_data *track_data;
2943         struct action_data *action;
2944
2945         track_data = tracing_cond_snapshot_data(file->tr);
2946         if (!track_data)
2947                 return;
2948
2949         if (!track_data->updated)
2950                 return;
2951
2952         action = snapshot_action(hist_data);
2953         if (!action)
2954                 return;
2955
2956         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2957         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2958                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2959                    action->track_data.var_str, track_data->track_val);
2960
2961         seq_puts(m, "\ttriggered by event with key: ");
2962         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2963         seq_putc(m, '\n');
2964 }
2965 #else
2966 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2967 {
2968         return false;
2969 }
2970 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2971                                      struct tracing_map_elt *elt,
2972                                      struct trace_buffer *buffer, void *rec,
2973                                      struct ring_buffer_event *rbe, void *key,
2974                                      struct action_data *data,
2975                                      u64 *var_ref_vals) {}
2976 static void track_data_snapshot_print(struct seq_file *m,
2977                                       struct hist_trigger_data *hist_data) {}
2978 #endif /* CONFIG_TRACER_SNAPSHOT */
2979
2980 static void track_data_print(struct seq_file *m,
2981                              struct hist_trigger_data *hist_data,
2982                              struct tracing_map_elt *elt,
2983                              struct action_data *data)
2984 {
2985         u64 track_val = get_track_val(hist_data, elt, data);
2986         unsigned int i, save_var_idx;
2987
2988         if (data->handler == HANDLER_ONMAX)
2989                 seq_printf(m, "\n\tmax: %10llu", track_val);
2990         else if (data->handler == HANDLER_ONCHANGE)
2991                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2992
2993         if (data->action == ACTION_SNAPSHOT)
2994                 return;
2995
2996         for (i = 0; i < hist_data->n_save_vars; i++) {
2997                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2998                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2999                 u64 val;
3000
3001                 save_var_idx = save_var->var.idx;
3002
3003                 val = tracing_map_read_var(elt, save_var_idx);
3004
3005                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3006                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3007                                    (char *)(uintptr_t)(val));
3008                 } else
3009                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3010         }
3011 }
3012
3013 static void ontrack_action(struct hist_trigger_data *hist_data,
3014                            struct tracing_map_elt *elt,
3015                            struct trace_buffer *buffer, void *rec,
3016                            struct ring_buffer_event *rbe, void *key,
3017                            struct action_data *data, u64 *var_ref_vals)
3018 {
3019         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3020
3021         if (check_track_val(elt, data, var_val)) {
3022                 save_track_val(hist_data, elt, data, var_val);
3023                 save_track_data(hist_data, elt, buffer, rec, rbe,
3024                                 key, data, var_ref_vals);
3025         }
3026 }
3027
3028 static void action_data_destroy(struct action_data *data)
3029 {
3030         unsigned int i;
3031
3032         lockdep_assert_held(&event_mutex);
3033
3034         kfree(data->action_name);
3035
3036         for (i = 0; i < data->n_params; i++)
3037                 kfree(data->params[i]);
3038
3039         if (data->synth_event)
3040                 data->synth_event->ref--;
3041
3042         kfree(data->synth_event_name);
3043
3044         kfree(data);
3045 }
3046
3047 static void track_data_destroy(struct hist_trigger_data *hist_data,
3048                                struct action_data *data)
3049 {
3050         struct trace_event_file *file = hist_data->event_file;
3051
3052         destroy_hist_field(data->track_data.track_var, 0);
3053
3054         if (data->action == ACTION_SNAPSHOT) {
3055                 struct track_data *track_data;
3056
3057                 track_data = tracing_cond_snapshot_data(file->tr);
3058                 if (track_data && track_data->hist_data == hist_data) {
3059                         tracing_snapshot_cond_disable(file->tr);
3060                         track_data_free(track_data);
3061                 }
3062         }
3063
3064         kfree(data->track_data.var_str);
3065
3066         action_data_destroy(data);
3067 }
3068
3069 static int action_create(struct hist_trigger_data *hist_data,
3070                          struct action_data *data);
3071
3072 static int track_data_create(struct hist_trigger_data *hist_data,
3073                              struct action_data *data)
3074 {
3075         struct hist_field *var_field, *ref_field, *track_var = NULL;
3076         struct trace_event_file *file = hist_data->event_file;
3077         struct trace_array *tr = file->tr;
3078         char *track_data_var_str;
3079         int ret = 0;
3080
3081         track_data_var_str = data->track_data.var_str;
3082         if (track_data_var_str[0] != '$') {
3083                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3084                 return -EINVAL;
3085         }
3086         track_data_var_str++;
3087
3088         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3089         if (!var_field) {
3090                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3091                 return -EINVAL;
3092         }
3093
3094         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3095         if (!ref_field)
3096                 return -ENOMEM;
3097
3098         data->track_data.var_ref = ref_field;
3099
3100         if (data->handler == HANDLER_ONMAX)
3101                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3102         if (IS_ERR(track_var)) {
3103                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3104                 ret = PTR_ERR(track_var);
3105                 goto out;
3106         }
3107
3108         if (data->handler == HANDLER_ONCHANGE)
3109                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3110         if (IS_ERR(track_var)) {
3111                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3112                 ret = PTR_ERR(track_var);
3113                 goto out;
3114         }
3115         data->track_data.track_var = track_var;
3116
3117         ret = action_create(hist_data, data);
3118  out:
3119         return ret;
3120 }
3121
3122 static int parse_action_params(struct trace_array *tr, char *params,
3123                                struct action_data *data)
3124 {
3125         char *param, *saved_param;
3126         bool first_param = true;
3127         int ret = 0;
3128
3129         while (params) {
3130                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3131                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3132                         goto out;
3133                 }
3134
3135                 param = strsep(&params, ",");
3136                 if (!param) {
3137                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3138                         ret = -EINVAL;
3139                         goto out;
3140                 }
3141
3142                 param = strstrip(param);
3143                 if (strlen(param) < 2) {
3144                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3145                         ret = -EINVAL;
3146                         goto out;
3147                 }
3148
3149                 saved_param = kstrdup(param, GFP_KERNEL);
3150                 if (!saved_param) {
3151                         ret = -ENOMEM;
3152                         goto out;
3153                 }
3154
3155                 if (first_param && data->use_trace_keyword) {
3156                         data->synth_event_name = saved_param;
3157                         first_param = false;
3158                         continue;
3159                 }
3160                 first_param = false;
3161
3162                 data->params[data->n_params++] = saved_param;
3163         }
3164  out:
3165         return ret;
3166 }
3167
3168 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3169                         enum handler_id handler)
3170 {
3171         char *action_name;
3172         int ret = 0;
3173
3174         strsep(&str, ".");
3175         if (!str) {
3176                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3177                 ret = -EINVAL;
3178                 goto out;
3179         }
3180
3181         action_name = strsep(&str, "(");
3182         if (!action_name || !str) {
3183                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3184                 ret = -EINVAL;
3185                 goto out;
3186         }
3187
3188         if (str_has_prefix(action_name, "save")) {
3189                 char *params = strsep(&str, ")");
3190
3191                 if (!params) {
3192                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3193                         ret = -EINVAL;
3194                         goto out;
3195                 }
3196
3197                 ret = parse_action_params(tr, params, data);
3198                 if (ret)
3199                         goto out;
3200
3201                 if (handler == HANDLER_ONMAX)
3202                         data->track_data.check_val = check_track_val_max;
3203                 else if (handler == HANDLER_ONCHANGE)
3204                         data->track_data.check_val = check_track_val_changed;
3205                 else {
3206                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3207                         ret = -EINVAL;
3208                         goto out;
3209                 }
3210
3211                 data->track_data.save_data = save_track_data_vars;
3212                 data->fn = ontrack_action;
3213                 data->action = ACTION_SAVE;
3214         } else if (str_has_prefix(action_name, "snapshot")) {
3215                 char *params = strsep(&str, ")");
3216
3217                 if (!str) {
3218                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3219                         ret = -EINVAL;
3220                         goto out;
3221                 }
3222
3223                 if (handler == HANDLER_ONMAX)
3224                         data->track_data.check_val = check_track_val_max;
3225                 else if (handler == HANDLER_ONCHANGE)
3226                         data->track_data.check_val = check_track_val_changed;
3227                 else {
3228                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3229                         ret = -EINVAL;
3230                         goto out;
3231                 }
3232
3233                 data->track_data.save_data = save_track_data_snapshot;
3234                 data->fn = ontrack_action;
3235                 data->action = ACTION_SNAPSHOT;
3236         } else {
3237                 char *params = strsep(&str, ")");
3238
3239                 if (str_has_prefix(action_name, "trace"))
3240                         data->use_trace_keyword = true;
3241
3242                 if (params) {
3243                         ret = parse_action_params(tr, params, data);
3244                         if (ret)
3245                                 goto out;
3246                 }
3247
3248                 if (handler == HANDLER_ONMAX)
3249                         data->track_data.check_val = check_track_val_max;
3250                 else if (handler == HANDLER_ONCHANGE)
3251                         data->track_data.check_val = check_track_val_changed;
3252
3253                 if (handler != HANDLER_ONMATCH) {
3254                         data->track_data.save_data = action_trace;
3255                         data->fn = ontrack_action;
3256                 } else
3257                         data->fn = action_trace;
3258
3259                 data->action = ACTION_TRACE;
3260         }
3261
3262         data->action_name = kstrdup(action_name, GFP_KERNEL);
3263         if (!data->action_name) {
3264                 ret = -ENOMEM;
3265                 goto out;
3266         }
3267
3268         data->handler = handler;
3269  out:
3270         return ret;
3271 }
3272
3273 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3274                                             char *str, enum handler_id handler)
3275 {
3276         struct action_data *data;
3277         int ret = -EINVAL;
3278         char *var_str;
3279
3280         data = kzalloc(sizeof(*data), GFP_KERNEL);
3281         if (!data)
3282                 return ERR_PTR(-ENOMEM);
3283
3284         var_str = strsep(&str, ")");
3285         if (!var_str || !str) {
3286                 ret = -EINVAL;
3287                 goto free;
3288         }
3289
3290         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3291         if (!data->track_data.var_str) {
3292                 ret = -ENOMEM;
3293                 goto free;
3294         }
3295
3296         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3297         if (ret)
3298                 goto free;
3299  out:
3300         return data;
3301  free:
3302         track_data_destroy(hist_data, data);
3303         data = ERR_PTR(ret);
3304         goto out;
3305 }
3306
3307 static void onmatch_destroy(struct action_data *data)
3308 {
3309         kfree(data->match_data.event);
3310         kfree(data->match_data.event_system);
3311
3312         action_data_destroy(data);
3313 }
3314
3315 static void destroy_field_var(struct field_var *field_var)
3316 {
3317         if (!field_var)
3318                 return;
3319
3320         destroy_hist_field(field_var->var, 0);
3321         destroy_hist_field(field_var->val, 0);
3322
3323         kfree(field_var);
3324 }
3325
3326 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3327 {
3328         unsigned int i;
3329
3330         for (i = 0; i < hist_data->n_field_vars; i++)
3331                 destroy_field_var(hist_data->field_vars[i]);
3332
3333         for (i = 0; i < hist_data->n_save_vars; i++)
3334                 destroy_field_var(hist_data->save_vars[i]);
3335 }
3336
3337 static void save_field_var(struct hist_trigger_data *hist_data,
3338                            struct field_var *field_var)
3339 {
3340         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3341
3342         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3343                 hist_data->n_field_var_str++;
3344 }
3345
3346
3347 static int check_synth_field(struct synth_event *event,
3348                              struct hist_field *hist_field,
3349                              unsigned int field_pos)
3350 {
3351         struct synth_field *field;
3352
3353         if (field_pos >= event->n_fields)
3354                 return -EINVAL;
3355
3356         field = event->fields[field_pos];
3357
3358         /*
3359          * A dynamic string synth field can accept static or
3360          * dynamic. A static string synth field can only accept a
3361          * same-sized static string, which is checked for later.
3362          */
3363         if (strstr(hist_field->type, "char[") && field->is_string
3364             && field->is_dynamic)
3365                 return 0;
3366
3367         if (strcmp(field->type, hist_field->type) != 0) {
3368                 if (field->size != hist_field->size ||
3369                     field->is_signed != hist_field->is_signed)
3370                         return -EINVAL;
3371         }
3372
3373         return 0;
3374 }
3375
3376 static struct hist_field *
3377 trace_action_find_var(struct hist_trigger_data *hist_data,
3378                       struct action_data *data,
3379                       char *system, char *event, char *var)
3380 {
3381         struct trace_array *tr = hist_data->event_file->tr;
3382         struct hist_field *hist_field;
3383
3384         var++; /* skip '$' */
3385
3386         hist_field = find_target_event_var(hist_data, system, event, var);
3387         if (!hist_field) {
3388                 if (!system && data->handler == HANDLER_ONMATCH) {
3389                         system = data->match_data.event_system;
3390                         event = data->match_data.event;
3391                 }
3392
3393                 hist_field = find_event_var(hist_data, system, event, var);
3394         }
3395
3396         if (!hist_field)
3397                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3398
3399         return hist_field;
3400 }
3401
3402 static struct hist_field *
3403 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3404                               struct action_data *data, char *system,
3405                               char *event, char *var)
3406 {
3407         struct hist_field *hist_field = NULL;
3408         struct field_var *field_var;
3409
3410         /*
3411          * First try to create a field var on the target event (the
3412          * currently being defined).  This will create a variable for
3413          * unqualified fields on the target event, or if qualified,
3414          * target fields that have qualified names matching the target.
3415          */
3416         field_var = create_target_field_var(hist_data, system, event, var);
3417
3418         if (field_var && !IS_ERR(field_var)) {
3419                 save_field_var(hist_data, field_var);
3420                 hist_field = field_var->var;
3421         } else {
3422                 field_var = NULL;
3423                 /*
3424                  * If no explicit system.event is specified, default to
3425                  * looking for fields on the onmatch(system.event.xxx)
3426                  * event.
3427                  */
3428                 if (!system && data->handler == HANDLER_ONMATCH) {
3429                         system = data->match_data.event_system;
3430                         event = data->match_data.event;
3431                 }
3432
3433                 if (!event)
3434                         goto free;
3435                 /*
3436                  * At this point, we're looking at a field on another
3437                  * event.  Because we can't modify a hist trigger on
3438                  * another event to add a variable for a field, we need
3439                  * to create a new trigger on that event and create the
3440                  * variable at the same time.
3441                  */
3442                 hist_field = create_field_var_hist(hist_data, system, event, var);
3443                 if (IS_ERR(hist_field))
3444                         goto free;
3445         }
3446  out:
3447         return hist_field;
3448  free:
3449         destroy_field_var(field_var);
3450         hist_field = NULL;
3451         goto out;
3452 }
3453
3454 static int trace_action_create(struct hist_trigger_data *hist_data,
3455                                struct action_data *data)
3456 {
3457         struct trace_array *tr = hist_data->event_file->tr;
3458         char *event_name, *param, *system = NULL;
3459         struct hist_field *hist_field, *var_ref;
3460         unsigned int i;
3461         unsigned int field_pos = 0;
3462         struct synth_event *event;
3463         char *synth_event_name;
3464         int var_ref_idx, ret = 0;
3465
3466         lockdep_assert_held(&event_mutex);
3467
3468         if (data->use_trace_keyword)
3469                 synth_event_name = data->synth_event_name;
3470         else
3471                 synth_event_name = data->action_name;
3472
3473         event = find_synth_event(synth_event_name);
3474         if (!event) {
3475                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3476                 return -EINVAL;
3477         }
3478
3479         event->ref++;
3480
3481         for (i = 0; i < data->n_params; i++) {
3482                 char *p;
3483
3484                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3485                 if (!param) {
3486                         ret = -ENOMEM;
3487                         goto err;
3488                 }
3489
3490                 system = strsep(&param, ".");
3491                 if (!param) {
3492                         param = (char *)system;
3493                         system = event_name = NULL;
3494                 } else {
3495                         event_name = strsep(&param, ".");
3496                         if (!param) {
3497                                 kfree(p);
3498                                 ret = -EINVAL;
3499                                 goto err;
3500                         }
3501                 }
3502
3503                 if (param[0] == '$')
3504                         hist_field = trace_action_find_var(hist_data, data,
3505                                                            system, event_name,
3506                                                            param);
3507                 else
3508                         hist_field = trace_action_create_field_var(hist_data,
3509                                                                    data,
3510                                                                    system,
3511                                                                    event_name,
3512                                                                    param);
3513
3514                 if (!hist_field) {
3515                         kfree(p);
3516                         ret = -EINVAL;
3517                         goto err;
3518                 }
3519
3520                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3521                         var_ref = create_var_ref(hist_data, hist_field,
3522                                                  system, event_name);
3523                         if (!var_ref) {
3524                                 kfree(p);
3525                                 ret = -ENOMEM;
3526                                 goto err;
3527                         }
3528
3529                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3530                         if (WARN_ON(var_ref_idx < 0)) {
3531                                 ret = var_ref_idx;
3532                                 goto err;
3533                         }
3534
3535                         data->var_ref_idx[i] = var_ref_idx;
3536
3537                         field_pos++;
3538                         kfree(p);
3539                         continue;
3540                 }
3541
3542                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3543                 kfree(p);
3544                 ret = -EINVAL;
3545                 goto err;
3546         }
3547
3548         if (field_pos != event->n_fields) {
3549                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3550                 ret = -EINVAL;
3551                 goto err;
3552         }
3553
3554         data->synth_event = event;
3555  out:
3556         return ret;
3557  err:
3558         event->ref--;
3559
3560         goto out;
3561 }
3562
3563 static int action_create(struct hist_trigger_data *hist_data,
3564                          struct action_data *data)
3565 {
3566         struct trace_event_file *file = hist_data->event_file;
3567         struct trace_array *tr = file->tr;
3568         struct track_data *track_data;
3569         struct field_var *field_var;
3570         unsigned int i;
3571         char *param;
3572         int ret = 0;
3573
3574         if (data->action == ACTION_TRACE)
3575                 return trace_action_create(hist_data, data);
3576
3577         if (data->action == ACTION_SNAPSHOT) {
3578                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3579                 if (IS_ERR(track_data)) {
3580                         ret = PTR_ERR(track_data);
3581                         goto out;
3582                 }
3583
3584                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3585                                                    cond_snapshot_update);
3586                 if (ret)
3587                         track_data_free(track_data);
3588
3589                 goto out;
3590         }
3591
3592         if (data->action == ACTION_SAVE) {
3593                 if (hist_data->n_save_vars) {
3594                         ret = -EEXIST;
3595                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3596                         goto out;
3597                 }
3598
3599                 for (i = 0; i < data->n_params; i++) {
3600                         param = kstrdup(data->params[i], GFP_KERNEL);
3601                         if (!param) {
3602                                 ret = -ENOMEM;
3603                                 goto out;
3604                         }
3605
3606                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3607                         if (IS_ERR(field_var)) {
3608                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3609                                          errpos(param));
3610                                 ret = PTR_ERR(field_var);
3611                                 kfree(param);
3612                                 goto out;
3613                         }
3614
3615                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3616                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3617                                 hist_data->n_save_var_str++;
3618                         kfree(param);
3619                 }
3620         }
3621  out:
3622         return ret;
3623 }
3624
3625 static int onmatch_create(struct hist_trigger_data *hist_data,
3626                           struct action_data *data)
3627 {
3628         return action_create(hist_data, data);
3629 }
3630
3631 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3632 {
3633         char *match_event, *match_event_system;
3634         struct action_data *data;
3635         int ret = -EINVAL;
3636
3637         data = kzalloc(sizeof(*data), GFP_KERNEL);
3638         if (!data)
3639                 return ERR_PTR(-ENOMEM);
3640
3641         match_event = strsep(&str, ")");
3642         if (!match_event || !str) {
3643                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3644                 goto free;
3645         }
3646
3647         match_event_system = strsep(&match_event, ".");
3648         if (!match_event) {
3649                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3650                 goto free;
3651         }
3652
3653         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3654                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3655                 goto free;
3656         }
3657
3658         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3659         if (!data->match_data.event) {
3660                 ret = -ENOMEM;
3661                 goto free;
3662         }
3663
3664         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3665         if (!data->match_data.event_system) {
3666                 ret = -ENOMEM;
3667                 goto free;
3668         }
3669
3670         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3671         if (ret)
3672                 goto free;
3673  out:
3674         return data;
3675  free:
3676         onmatch_destroy(data);
3677         data = ERR_PTR(ret);
3678         goto out;
3679 }
3680
3681 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3682 {
3683         hist_data->fields[HITCOUNT_IDX] =
3684                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3685         if (!hist_data->fields[HITCOUNT_IDX])
3686                 return -ENOMEM;
3687
3688         hist_data->n_vals++;
3689         hist_data->n_fields++;
3690
3691         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3692                 return -EINVAL;
3693
3694         return 0;
3695 }
3696
3697 static int __create_val_field(struct hist_trigger_data *hist_data,
3698                               unsigned int val_idx,
3699                               struct trace_event_file *file,
3700                               char *var_name, char *field_str,
3701                               unsigned long flags)
3702 {
3703         struct hist_field *hist_field;
3704         int ret = 0;
3705
3706         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3707         if (IS_ERR(hist_field)) {
3708                 ret = PTR_ERR(hist_field);
3709                 goto out;
3710         }
3711
3712         hist_data->fields[val_idx] = hist_field;
3713
3714         ++hist_data->n_vals;
3715         ++hist_data->n_fields;
3716
3717         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3718                 ret = -EINVAL;
3719  out:
3720         return ret;
3721 }
3722
3723 static int create_val_field(struct hist_trigger_data *hist_data,
3724                             unsigned int val_idx,
3725                             struct trace_event_file *file,
3726                             char *field_str)
3727 {
3728         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3729                 return -EINVAL;
3730
3731         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3732 }
3733
3734 static int create_var_field(struct hist_trigger_data *hist_data,
3735                             unsigned int val_idx,
3736                             struct trace_event_file *file,
3737                             char *var_name, char *expr_str)
3738 {
3739         struct trace_array *tr = hist_data->event_file->tr;
3740         unsigned long flags = 0;
3741         int ret;
3742
3743         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3744                 return -EINVAL;
3745
3746         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3747                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3748                 return -EINVAL;
3749         }
3750
3751         flags |= HIST_FIELD_FL_VAR;
3752         hist_data->n_vars++;
3753         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3754                 return -EINVAL;
3755
3756         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3757
3758         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3759                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3760
3761         return ret;
3762 }
3763
3764 static int create_val_fields(struct hist_trigger_data *hist_data,
3765                              struct trace_event_file *file)
3766 {
3767         char *fields_str, *field_str;
3768         unsigned int i, j = 1;
3769         int ret;
3770
3771         ret = create_hitcount_val(hist_data);
3772         if (ret)
3773                 goto out;
3774
3775         fields_str = hist_data->attrs->vals_str;
3776         if (!fields_str)
3777                 goto out;
3778
3779         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3780                      j < TRACING_MAP_VALS_MAX; i++) {
3781                 field_str = strsep(&fields_str, ",");
3782                 if (!field_str)
3783                         break;
3784
3785                 if (strcmp(field_str, "hitcount") == 0)
3786                         continue;
3787
3788                 ret = create_val_field(hist_data, j++, file, field_str);
3789                 if (ret)
3790                         goto out;
3791         }
3792
3793         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3794                 ret = -EINVAL;
3795  out:
3796         return ret;
3797 }
3798
3799 static int create_key_field(struct hist_trigger_data *hist_data,
3800                             unsigned int key_idx,
3801                             unsigned int key_offset,
3802                             struct trace_event_file *file,
3803                             char *field_str)
3804 {
3805         struct trace_array *tr = hist_data->event_file->tr;
3806         struct hist_field *hist_field = NULL;
3807         unsigned long flags = 0;
3808         unsigned int key_size;
3809         int ret = 0;
3810
3811         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3812                 return -EINVAL;
3813
3814         flags |= HIST_FIELD_FL_KEY;
3815
3816         if (strcmp(field_str, "stacktrace") == 0) {
3817                 flags |= HIST_FIELD_FL_STACKTRACE;
3818                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3819                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3820         } else {
3821                 hist_field = parse_expr(hist_data, file, field_str, flags,
3822                                         NULL, 0);
3823                 if (IS_ERR(hist_field)) {
3824                         ret = PTR_ERR(hist_field);
3825                         goto out;
3826                 }
3827
3828                 if (field_has_hist_vars(hist_field, 0)) {
3829                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3830                         destroy_hist_field(hist_field, 0);
3831                         ret = -EINVAL;
3832                         goto out;
3833                 }
3834
3835                 key_size = hist_field->size;
3836         }
3837
3838         hist_data->fields[key_idx] = hist_field;
3839
3840         key_size = ALIGN(key_size, sizeof(u64));
3841         hist_data->fields[key_idx]->size = key_size;
3842         hist_data->fields[key_idx]->offset = key_offset;
3843
3844         hist_data->key_size += key_size;
3845
3846         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3847                 ret = -EINVAL;
3848                 goto out;
3849         }
3850
3851         hist_data->n_keys++;
3852         hist_data->n_fields++;
3853
3854         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3855                 return -EINVAL;
3856
3857         ret = key_size;
3858  out:
3859         return ret;
3860 }
3861
3862 static int create_key_fields(struct hist_trigger_data *hist_data,
3863                              struct trace_event_file *file)
3864 {
3865         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3866         char *fields_str, *field_str;
3867         int ret = -EINVAL;
3868
3869         fields_str = hist_data->attrs->keys_str;
3870         if (!fields_str)
3871                 goto out;
3872
3873         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3874                 field_str = strsep(&fields_str, ",");
3875                 if (!field_str)
3876                         break;
3877                 ret = create_key_field(hist_data, i, key_offset,
3878                                        file, field_str);
3879                 if (ret < 0)
3880                         goto out;
3881                 key_offset += ret;
3882         }
3883         if (fields_str) {
3884                 ret = -EINVAL;
3885                 goto out;
3886         }
3887         ret = 0;
3888  out:
3889         return ret;
3890 }
3891
3892 static int create_var_fields(struct hist_trigger_data *hist_data,
3893                              struct trace_event_file *file)
3894 {
3895         unsigned int i, j = hist_data->n_vals;
3896         int ret = 0;
3897
3898         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3899
3900         for (i = 0; i < n_vars; i++) {
3901                 char *var_name = hist_data->attrs->var_defs.name[i];
3902                 char *expr = hist_data->attrs->var_defs.expr[i];
3903
3904                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3905                 if (ret)
3906                         goto out;
3907         }
3908  out:
3909         return ret;
3910 }
3911
3912 static void free_var_defs(struct hist_trigger_data *hist_data)
3913 {
3914         unsigned int i;
3915
3916         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3917                 kfree(hist_data->attrs->var_defs.name[i]);
3918                 kfree(hist_data->attrs->var_defs.expr[i]);
3919         }
3920
3921         hist_data->attrs->var_defs.n_vars = 0;
3922 }
3923
3924 static int parse_var_defs(struct hist_trigger_data *hist_data)
3925 {
3926         struct trace_array *tr = hist_data->event_file->tr;
3927         char *s, *str, *var_name, *field_str;
3928         unsigned int i, j, n_vars = 0;
3929         int ret = 0;
3930
3931         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3932                 str = hist_data->attrs->assignment_str[i];
3933                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3934                         field_str = strsep(&str, ",");
3935                         if (!field_str)
3936                                 break;
3937
3938                         var_name = strsep(&field_str, "=");
3939                         if (!var_name || !field_str) {
3940                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3941                                          errpos(var_name));
3942                                 ret = -EINVAL;
3943                                 goto free;
3944                         }
3945
3946                         if (n_vars == TRACING_MAP_VARS_MAX) {
3947                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3948                                 ret = -EINVAL;
3949                                 goto free;
3950                         }
3951
3952                         s = kstrdup(var_name, GFP_KERNEL);
3953                         if (!s) {
3954                                 ret = -ENOMEM;
3955                                 goto free;
3956                         }
3957                         hist_data->attrs->var_defs.name[n_vars] = s;
3958
3959                         s = kstrdup(field_str, GFP_KERNEL);
3960                         if (!s) {
3961                                 ret = -ENOMEM;
3962                                 goto free;
3963                         }
3964                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3965
3966                         hist_data->attrs->var_defs.n_vars = n_vars;
3967                 }
3968         }
3969
3970         return ret;
3971  free:
3972         free_var_defs(hist_data);
3973
3974         return ret;
3975 }
3976
3977 static int create_hist_fields(struct hist_trigger_data *hist_data,
3978                               struct trace_event_file *file)
3979 {
3980         int ret;
3981
3982         ret = parse_var_defs(hist_data);
3983         if (ret)
3984                 goto out;
3985
3986         ret = create_val_fields(hist_data, file);
3987         if (ret)
3988                 goto out;
3989
3990         ret = create_var_fields(hist_data, file);
3991         if (ret)
3992                 goto out;
3993
3994         ret = create_key_fields(hist_data, file);
3995         if (ret)
3996                 goto out;
3997  out:
3998         free_var_defs(hist_data);
3999
4000         return ret;
4001 }
4002
4003 static int is_descending(struct trace_array *tr, const char *str)
4004 {
4005         if (!str)
4006                 return 0;
4007
4008         if (strcmp(str, "descending") == 0)
4009                 return 1;
4010
4011         if (strcmp(str, "ascending") == 0)
4012                 return 0;
4013
4014         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4015
4016         return -EINVAL;
4017 }
4018
4019 static int create_sort_keys(struct hist_trigger_data *hist_data)
4020 {
4021         struct trace_array *tr = hist_data->event_file->tr;
4022         char *fields_str = hist_data->attrs->sort_key_str;
4023         struct tracing_map_sort_key *sort_key;
4024         int descending, ret = 0;
4025         unsigned int i, j, k;
4026
4027         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4028
4029         if (!fields_str)
4030                 goto out;
4031
4032         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4033                 struct hist_field *hist_field;
4034                 char *field_str, *field_name;
4035                 const char *test_name;
4036
4037                 sort_key = &hist_data->sort_keys[i];
4038
4039                 field_str = strsep(&fields_str, ",");
4040                 if (!field_str)
4041                         break;
4042
4043                 if (!*field_str) {
4044                         ret = -EINVAL;
4045                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4046                         break;
4047                 }
4048
4049                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4050                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4051                         ret = -EINVAL;
4052                         break;
4053                 }
4054
4055                 field_name = strsep(&field_str, ".");
4056                 if (!field_name || !*field_name) {
4057                         ret = -EINVAL;
4058                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4059                         break;
4060                 }
4061
4062                 if (strcmp(field_name, "hitcount") == 0) {
4063                         descending = is_descending(tr, field_str);
4064                         if (descending < 0) {
4065                                 ret = descending;
4066                                 break;
4067                         }
4068                         sort_key->descending = descending;
4069                         continue;
4070                 }
4071
4072                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4073                         unsigned int idx;
4074
4075                         hist_field = hist_data->fields[j];
4076                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4077                                 continue;
4078
4079                         idx = k++;
4080
4081                         test_name = hist_field_name(hist_field, 0);
4082
4083                         if (strcmp(field_name, test_name) == 0) {
4084                                 sort_key->field_idx = idx;
4085                                 descending = is_descending(tr, field_str);
4086                                 if (descending < 0) {
4087                                         ret = descending;
4088                                         goto out;
4089                                 }
4090                                 sort_key->descending = descending;
4091                                 break;
4092                         }
4093                 }
4094                 if (j == hist_data->n_fields) {
4095                         ret = -EINVAL;
4096                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4097                         break;
4098                 }
4099         }
4100
4101         hist_data->n_sort_keys = i;
4102  out:
4103         return ret;
4104 }
4105
4106 static void destroy_actions(struct hist_trigger_data *hist_data)
4107 {
4108         unsigned int i;
4109
4110         for (i = 0; i < hist_data->n_actions; i++) {
4111                 struct action_data *data = hist_data->actions[i];
4112
4113                 if (data->handler == HANDLER_ONMATCH)
4114                         onmatch_destroy(data);
4115                 else if (data->handler == HANDLER_ONMAX ||
4116                          data->handler == HANDLER_ONCHANGE)
4117                         track_data_destroy(hist_data, data);
4118                 else
4119                         kfree(data);
4120         }
4121 }
4122
4123 static int parse_actions(struct hist_trigger_data *hist_data)
4124 {
4125         struct trace_array *tr = hist_data->event_file->tr;
4126         struct action_data *data;
4127         unsigned int i;
4128         int ret = 0;
4129         char *str;
4130         int len;
4131
4132         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4133                 str = hist_data->attrs->action_str[i];
4134
4135                 if ((len = str_has_prefix(str, "onmatch("))) {
4136                         char *action_str = str + len;
4137
4138                         data = onmatch_parse(tr, action_str);
4139                         if (IS_ERR(data)) {
4140                                 ret = PTR_ERR(data);
4141                                 break;
4142                         }
4143                 } else if ((len = str_has_prefix(str, "onmax("))) {
4144                         char *action_str = str + len;
4145
4146                         data = track_data_parse(hist_data, action_str,
4147                                                 HANDLER_ONMAX);
4148                         if (IS_ERR(data)) {
4149                                 ret = PTR_ERR(data);
4150                                 break;
4151                         }
4152                 } else if ((len = str_has_prefix(str, "onchange("))) {
4153                         char *action_str = str + len;
4154
4155                         data = track_data_parse(hist_data, action_str,
4156                                                 HANDLER_ONCHANGE);
4157                         if (IS_ERR(data)) {
4158                                 ret = PTR_ERR(data);
4159                                 break;
4160                         }
4161                 } else {
4162                         ret = -EINVAL;
4163                         break;
4164                 }
4165
4166                 hist_data->actions[hist_data->n_actions++] = data;
4167         }
4168
4169         return ret;
4170 }
4171
4172 static int create_actions(struct hist_trigger_data *hist_data)
4173 {
4174         struct action_data *data;
4175         unsigned int i;
4176         int ret = 0;
4177
4178         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4179                 data = hist_data->actions[i];
4180
4181                 if (data->handler == HANDLER_ONMATCH) {
4182                         ret = onmatch_create(hist_data, data);
4183                         if (ret)
4184                                 break;
4185                 } else if (data->handler == HANDLER_ONMAX ||
4186                            data->handler == HANDLER_ONCHANGE) {
4187                         ret = track_data_create(hist_data, data);
4188                         if (ret)
4189                                 break;
4190                 } else {
4191                         ret = -EINVAL;
4192                         break;
4193                 }
4194         }
4195
4196         return ret;
4197 }
4198
4199 static void print_actions(struct seq_file *m,
4200                           struct hist_trigger_data *hist_data,
4201                           struct tracing_map_elt *elt)
4202 {
4203         unsigned int i;
4204
4205         for (i = 0; i < hist_data->n_actions; i++) {
4206                 struct action_data *data = hist_data->actions[i];
4207
4208                 if (data->action == ACTION_SNAPSHOT)
4209                         continue;
4210
4211                 if (data->handler == HANDLER_ONMAX ||
4212                     data->handler == HANDLER_ONCHANGE)
4213                         track_data_print(m, hist_data, elt, data);
4214         }
4215 }
4216
4217 static void print_action_spec(struct seq_file *m,
4218                               struct hist_trigger_data *hist_data,
4219                               struct action_data *data)
4220 {
4221         unsigned int i;
4222
4223         if (data->action == ACTION_SAVE) {
4224                 for (i = 0; i < hist_data->n_save_vars; i++) {
4225                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4226                         if (i < hist_data->n_save_vars - 1)
4227                                 seq_puts(m, ",");
4228                 }
4229         } else if (data->action == ACTION_TRACE) {
4230                 if (data->use_trace_keyword)
4231                         seq_printf(m, "%s", data->synth_event_name);
4232                 for (i = 0; i < data->n_params; i++) {
4233                         if (i || data->use_trace_keyword)
4234                                 seq_puts(m, ",");
4235                         seq_printf(m, "%s", data->params[i]);
4236                 }
4237         }
4238 }
4239
4240 static void print_track_data_spec(struct seq_file *m,
4241                                   struct hist_trigger_data *hist_data,
4242                                   struct action_data *data)
4243 {
4244         if (data->handler == HANDLER_ONMAX)
4245                 seq_puts(m, ":onmax(");
4246         else if (data->handler == HANDLER_ONCHANGE)
4247                 seq_puts(m, ":onchange(");
4248         seq_printf(m, "%s", data->track_data.var_str);
4249         seq_printf(m, ").%s(", data->action_name);
4250
4251         print_action_spec(m, hist_data, data);
4252
4253         seq_puts(m, ")");
4254 }
4255
4256 static void print_onmatch_spec(struct seq_file *m,
4257                                struct hist_trigger_data *hist_data,
4258                                struct action_data *data)
4259 {
4260         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4261                    data->match_data.event);
4262
4263         seq_printf(m, "%s(", data->action_name);
4264
4265         print_action_spec(m, hist_data, data);
4266
4267         seq_puts(m, ")");
4268 }
4269
4270 static bool actions_match(struct hist_trigger_data *hist_data,
4271                           struct hist_trigger_data *hist_data_test)
4272 {
4273         unsigned int i, j;
4274
4275         if (hist_data->n_actions != hist_data_test->n_actions)
4276                 return false;
4277
4278         for (i = 0; i < hist_data->n_actions; i++) {
4279                 struct action_data *data = hist_data->actions[i];
4280                 struct action_data *data_test = hist_data_test->actions[i];
4281                 char *action_name, *action_name_test;
4282
4283                 if (data->handler != data_test->handler)
4284                         return false;
4285                 if (data->action != data_test->action)
4286                         return false;
4287
4288                 if (data->n_params != data_test->n_params)
4289                         return false;
4290
4291                 for (j = 0; j < data->n_params; j++) {
4292                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4293                                 return false;
4294                 }
4295
4296                 if (data->use_trace_keyword)
4297                         action_name = data->synth_event_name;
4298                 else
4299                         action_name = data->action_name;
4300
4301                 if (data_test->use_trace_keyword)
4302                         action_name_test = data_test->synth_event_name;
4303                 else
4304                         action_name_test = data_test->action_name;
4305
4306                 if (strcmp(action_name, action_name_test) != 0)
4307                         return false;
4308
4309                 if (data->handler == HANDLER_ONMATCH) {
4310                         if (strcmp(data->match_data.event_system,
4311                                    data_test->match_data.event_system) != 0)
4312                                 return false;
4313                         if (strcmp(data->match_data.event,
4314                                    data_test->match_data.event) != 0)
4315                                 return false;
4316                 } else if (data->handler == HANDLER_ONMAX ||
4317                            data->handler == HANDLER_ONCHANGE) {
4318                         if (strcmp(data->track_data.var_str,
4319                                    data_test->track_data.var_str) != 0)
4320                                 return false;
4321                 }
4322         }
4323
4324         return true;
4325 }
4326
4327
4328 static void print_actions_spec(struct seq_file *m,
4329                                struct hist_trigger_data *hist_data)
4330 {
4331         unsigned int i;
4332
4333         for (i = 0; i < hist_data->n_actions; i++) {
4334                 struct action_data *data = hist_data->actions[i];
4335
4336                 if (data->handler == HANDLER_ONMATCH)
4337                         print_onmatch_spec(m, hist_data, data);
4338                 else if (data->handler == HANDLER_ONMAX ||
4339                          data->handler == HANDLER_ONCHANGE)
4340                         print_track_data_spec(m, hist_data, data);
4341         }
4342 }
4343
4344 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4345 {
4346         unsigned int i;
4347
4348         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4349                 kfree(hist_data->field_var_hists[i]->cmd);
4350                 kfree(hist_data->field_var_hists[i]);
4351         }
4352 }
4353
4354 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4355 {
4356         if (!hist_data)
4357                 return;
4358
4359         destroy_hist_trigger_attrs(hist_data->attrs);
4360         destroy_hist_fields(hist_data);
4361         tracing_map_destroy(hist_data->map);
4362
4363         destroy_actions(hist_data);
4364         destroy_field_vars(hist_data);
4365         destroy_field_var_hists(hist_data);
4366
4367         kfree(hist_data);
4368 }
4369
4370 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4371 {
4372         struct tracing_map *map = hist_data->map;
4373         struct ftrace_event_field *field;
4374         struct hist_field *hist_field;
4375         int i, idx = 0;
4376
4377         for_each_hist_field(i, hist_data) {
4378                 hist_field = hist_data->fields[i];
4379                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4380                         tracing_map_cmp_fn_t cmp_fn;
4381
4382                         field = hist_field->field;
4383
4384                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4385                                 cmp_fn = tracing_map_cmp_none;
4386                         else if (!field)
4387                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4388                                                              hist_field->is_signed);
4389                         else if (is_string_field(field))
4390                                 cmp_fn = tracing_map_cmp_string;
4391                         else
4392                                 cmp_fn = tracing_map_cmp_num(field->size,
4393                                                              field->is_signed);
4394                         idx = tracing_map_add_key_field(map,
4395                                                         hist_field->offset,
4396                                                         cmp_fn);
4397                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4398                         idx = tracing_map_add_sum_field(map);
4399
4400                 if (idx < 0)
4401                         return idx;
4402
4403                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4404                         idx = tracing_map_add_var(map);
4405                         if (idx < 0)
4406                                 return idx;
4407                         hist_field->var.idx = idx;
4408                         hist_field->var.hist_data = hist_data;
4409                 }
4410         }
4411
4412         return 0;
4413 }
4414
4415 static struct hist_trigger_data *
4416 create_hist_data(unsigned int map_bits,
4417                  struct hist_trigger_attrs *attrs,
4418                  struct trace_event_file *file,
4419                  bool remove)
4420 {
4421         const struct tracing_map_ops *map_ops = NULL;
4422         struct hist_trigger_data *hist_data;
4423         int ret = 0;
4424
4425         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4426         if (!hist_data)
4427                 return ERR_PTR(-ENOMEM);
4428
4429         hist_data->attrs = attrs;
4430         hist_data->remove = remove;
4431         hist_data->event_file = file;
4432
4433         ret = parse_actions(hist_data);
4434         if (ret)
4435                 goto free;
4436
4437         ret = create_hist_fields(hist_data, file);
4438         if (ret)
4439                 goto free;
4440
4441         ret = create_sort_keys(hist_data);
4442         if (ret)
4443                 goto free;
4444
4445         map_ops = &hist_trigger_elt_data_ops;
4446
4447         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4448                                             map_ops, hist_data);
4449         if (IS_ERR(hist_data->map)) {
4450                 ret = PTR_ERR(hist_data->map);
4451                 hist_data->map = NULL;
4452                 goto free;
4453         }
4454
4455         ret = create_tracing_map_fields(hist_data);
4456         if (ret)
4457                 goto free;
4458  out:
4459         return hist_data;
4460  free:
4461         hist_data->attrs = NULL;
4462
4463         destroy_hist_data(hist_data);
4464
4465         hist_data = ERR_PTR(ret);
4466
4467         goto out;
4468 }
4469
4470 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4471                                     struct tracing_map_elt *elt,
4472                                     struct trace_buffer *buffer, void *rec,
4473                                     struct ring_buffer_event *rbe,
4474                                     u64 *var_ref_vals)
4475 {
4476         struct hist_elt_data *elt_data;
4477         struct hist_field *hist_field;
4478         unsigned int i, var_idx;
4479         u64 hist_val;
4480
4481         elt_data = elt->private_data;
4482         elt_data->var_ref_vals = var_ref_vals;
4483
4484         for_each_hist_val_field(i, hist_data) {
4485                 hist_field = hist_data->fields[i];
4486                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4487                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4488                         var_idx = hist_field->var.idx;
4489
4490                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4491                                 unsigned int str_start, var_str_idx, idx;
4492                                 char *str, *val_str;
4493
4494                                 str_start = hist_data->n_field_var_str +
4495                                         hist_data->n_save_var_str;
4496                                 var_str_idx = hist_field->var_str_idx;
4497                                 idx = str_start + var_str_idx;
4498
4499                                 str = elt_data->field_var_str[idx];
4500                                 val_str = (char *)(uintptr_t)hist_val;
4501                                 strscpy(str, val_str, STR_VAR_LEN_MAX);
4502
4503                                 hist_val = (u64)(uintptr_t)str;
4504                         }
4505                         tracing_map_set_var(elt, var_idx, hist_val);
4506                         continue;
4507                 }
4508                 tracing_map_update_sum(elt, i, hist_val);
4509         }
4510
4511         for_each_hist_key_field(i, hist_data) {
4512                 hist_field = hist_data->fields[i];
4513                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4514                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4515                         var_idx = hist_field->var.idx;
4516                         tracing_map_set_var(elt, var_idx, hist_val);
4517                 }
4518         }
4519
4520         update_field_vars(hist_data, elt, buffer, rbe, rec);
4521 }
4522
4523 static inline void add_to_key(char *compound_key, void *key,
4524                               struct hist_field *key_field, void *rec)
4525 {
4526         size_t size = key_field->size;
4527
4528         if (key_field->flags & HIST_FIELD_FL_STRING) {
4529                 struct ftrace_event_field *field;
4530
4531                 field = key_field->field;
4532                 if (field->filter_type == FILTER_DYN_STRING)
4533                         size = *(u32 *)(rec + field->offset) >> 16;
4534                 else if (field->filter_type == FILTER_STATIC_STRING)
4535                         size = field->size;
4536
4537                 /* ensure NULL-termination */
4538                 if (size > key_field->size - 1)
4539                         size = key_field->size - 1;
4540
4541                 strncpy(compound_key + key_field->offset, (char *)key, size);
4542         } else
4543                 memcpy(compound_key + key_field->offset, key, size);
4544 }
4545
4546 static void
4547 hist_trigger_actions(struct hist_trigger_data *hist_data,
4548                      struct tracing_map_elt *elt,
4549                      struct trace_buffer *buffer, void *rec,
4550                      struct ring_buffer_event *rbe, void *key,
4551                      u64 *var_ref_vals)
4552 {
4553         struct action_data *data;
4554         unsigned int i;
4555
4556         for (i = 0; i < hist_data->n_actions; i++) {
4557                 data = hist_data->actions[i];
4558                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4559         }
4560 }
4561
4562 static void event_hist_trigger(struct event_trigger_data *data,
4563                                struct trace_buffer *buffer, void *rec,
4564                                struct ring_buffer_event *rbe)
4565 {
4566         struct hist_trigger_data *hist_data = data->private_data;
4567         bool use_compound_key = (hist_data->n_keys > 1);
4568         unsigned long entries[HIST_STACKTRACE_DEPTH];
4569         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4570         char compound_key[HIST_KEY_SIZE_MAX];
4571         struct tracing_map_elt *elt = NULL;
4572         struct hist_field *key_field;
4573         u64 field_contents;
4574         void *key = NULL;
4575         unsigned int i;
4576
4577         memset(compound_key, 0, hist_data->key_size);
4578
4579         for_each_hist_key_field(i, hist_data) {
4580                 key_field = hist_data->fields[i];
4581
4582                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4583                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4584                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4585                                          HIST_STACKTRACE_SKIP);
4586                         key = entries;
4587                 } else {
4588                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4589                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4590                                 key = (void *)(unsigned long)field_contents;
4591                                 use_compound_key = true;
4592                         } else
4593                                 key = (void *)&field_contents;
4594                 }
4595
4596                 if (use_compound_key)
4597                         add_to_key(compound_key, key, key_field, rec);
4598         }
4599
4600         if (use_compound_key)
4601                 key = compound_key;
4602
4603         if (hist_data->n_var_refs &&
4604             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4605                 return;
4606
4607         elt = tracing_map_insert(hist_data->map, key);
4608         if (!elt)
4609                 return;
4610
4611         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4612
4613         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4614                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4615 }
4616
4617 static void hist_trigger_stacktrace_print(struct seq_file *m,
4618                                           unsigned long *stacktrace_entries,
4619                                           unsigned int max_entries)
4620 {
4621         char str[KSYM_SYMBOL_LEN];
4622         unsigned int spaces = 8;
4623         unsigned int i;
4624
4625         for (i = 0; i < max_entries; i++) {
4626                 if (!stacktrace_entries[i])
4627                         return;
4628
4629                 seq_printf(m, "%*c", 1 + spaces, ' ');
4630                 sprint_symbol(str, stacktrace_entries[i]);
4631                 seq_printf(m, "%s\n", str);
4632         }
4633 }
4634
4635 static void hist_trigger_print_key(struct seq_file *m,
4636                                    struct hist_trigger_data *hist_data,
4637                                    void *key,
4638                                    struct tracing_map_elt *elt)
4639 {
4640         struct hist_field *key_field;
4641         char str[KSYM_SYMBOL_LEN];
4642         bool multiline = false;
4643         const char *field_name;
4644         unsigned int i;
4645         u64 uval;
4646
4647         seq_puts(m, "{ ");
4648
4649         for_each_hist_key_field(i, hist_data) {
4650                 key_field = hist_data->fields[i];
4651
4652                 if (i > hist_data->n_vals)
4653                         seq_puts(m, ", ");
4654
4655                 field_name = hist_field_name(key_field, 0);
4656
4657                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4658                         uval = *(u64 *)(key + key_field->offset);
4659                         seq_printf(m, "%s: %llx", field_name, uval);
4660                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4661                         uval = *(u64 *)(key + key_field->offset);
4662                         sprint_symbol_no_offset(str, uval);
4663                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4664                                    uval, str);
4665                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4666                         uval = *(u64 *)(key + key_field->offset);
4667                         sprint_symbol(str, uval);
4668                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4669                                    uval, str);
4670                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4671                         struct hist_elt_data *elt_data = elt->private_data;
4672                         char *comm;
4673
4674                         if (WARN_ON_ONCE(!elt_data))
4675                                 return;
4676
4677                         comm = elt_data->comm;
4678
4679                         uval = *(u64 *)(key + key_field->offset);
4680                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4681                                    comm, uval);
4682                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4683                         const char *syscall_name;
4684
4685                         uval = *(u64 *)(key + key_field->offset);
4686                         syscall_name = get_syscall_name(uval);
4687                         if (!syscall_name)
4688                                 syscall_name = "unknown_syscall";
4689
4690                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4691                                    syscall_name, uval);
4692                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4693                         seq_puts(m, "stacktrace:\n");
4694                         hist_trigger_stacktrace_print(m,
4695                                                       key + key_field->offset,
4696                                                       HIST_STACKTRACE_DEPTH);
4697                         multiline = true;
4698                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4699                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4700                                    *(u64 *)(key + key_field->offset));
4701                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4702                         seq_printf(m, "%s: %-50s", field_name,
4703                                    (char *)(key + key_field->offset));
4704                 } else {
4705                         uval = *(u64 *)(key + key_field->offset);
4706                         seq_printf(m, "%s: %10llu", field_name, uval);
4707                 }
4708         }
4709
4710         if (!multiline)
4711                 seq_puts(m, " ");
4712
4713         seq_puts(m, "}");
4714 }
4715
4716 static void hist_trigger_entry_print(struct seq_file *m,
4717                                      struct hist_trigger_data *hist_data,
4718                                      void *key,
4719                                      struct tracing_map_elt *elt)
4720 {
4721         const char *field_name;
4722         unsigned int i;
4723
4724         hist_trigger_print_key(m, hist_data, key, elt);
4725
4726         seq_printf(m, " hitcount: %10llu",
4727                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4728
4729         for (i = 1; i < hist_data->n_vals; i++) {
4730                 field_name = hist_field_name(hist_data->fields[i], 0);
4731
4732                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4733                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4734                         continue;
4735
4736                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4737                         seq_printf(m, "  %s: %10llx", field_name,
4738                                    tracing_map_read_sum(elt, i));
4739                 } else {
4740                         seq_printf(m, "  %s: %10llu", field_name,
4741                                    tracing_map_read_sum(elt, i));
4742                 }
4743         }
4744
4745         print_actions(m, hist_data, elt);
4746
4747         seq_puts(m, "\n");
4748 }
4749
4750 static int print_entries(struct seq_file *m,
4751                          struct hist_trigger_data *hist_data)
4752 {
4753         struct tracing_map_sort_entry **sort_entries = NULL;
4754         struct tracing_map *map = hist_data->map;
4755         int i, n_entries;
4756
4757         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4758                                              hist_data->n_sort_keys,
4759                                              &sort_entries);
4760         if (n_entries < 0)
4761                 return n_entries;
4762
4763         for (i = 0; i < n_entries; i++)
4764                 hist_trigger_entry_print(m, hist_data,
4765                                          sort_entries[i]->key,
4766                                          sort_entries[i]->elt);
4767
4768         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4769
4770         return n_entries;
4771 }
4772
4773 static void hist_trigger_show(struct seq_file *m,
4774                               struct event_trigger_data *data, int n)
4775 {
4776         struct hist_trigger_data *hist_data;
4777         int n_entries;
4778
4779         if (n > 0)
4780                 seq_puts(m, "\n\n");
4781
4782         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4783         data->ops->print(m, data->ops, data);
4784         seq_puts(m, "#\n\n");
4785
4786         hist_data = data->private_data;
4787         n_entries = print_entries(m, hist_data);
4788         if (n_entries < 0)
4789                 n_entries = 0;
4790
4791         track_data_snapshot_print(m, hist_data);
4792
4793         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4794                    (u64)atomic64_read(&hist_data->map->hits),
4795                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4796 }
4797
4798 static int hist_show(struct seq_file *m, void *v)
4799 {
4800         struct event_trigger_data *data;
4801         struct trace_event_file *event_file;
4802         int n = 0, ret = 0;
4803
4804         mutex_lock(&event_mutex);
4805
4806         event_file = event_file_data(m->private);
4807         if (unlikely(!event_file)) {
4808                 ret = -ENODEV;
4809                 goto out_unlock;
4810         }
4811
4812         list_for_each_entry(data, &event_file->triggers, list) {
4813                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4814                         hist_trigger_show(m, data, n++);
4815         }
4816
4817  out_unlock:
4818         mutex_unlock(&event_mutex);
4819
4820         return ret;
4821 }
4822
4823 static int event_hist_open(struct inode *inode, struct file *file)
4824 {
4825         int ret;
4826
4827         ret = security_locked_down(LOCKDOWN_TRACEFS);
4828         if (ret)
4829                 return ret;
4830
4831         return single_open(file, hist_show, file);
4832 }
4833
4834 const struct file_operations event_hist_fops = {
4835         .open = event_hist_open,
4836         .read = seq_read,
4837         .llseek = seq_lseek,
4838         .release = single_release,
4839 };
4840
4841 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4842 static void hist_field_debug_show_flags(struct seq_file *m,
4843                                         unsigned long flags)
4844 {
4845         seq_puts(m, "      flags:\n");
4846
4847         if (flags & HIST_FIELD_FL_KEY)
4848                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4849         else if (flags & HIST_FIELD_FL_HITCOUNT)
4850                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4851         else if (flags & HIST_FIELD_FL_VAR)
4852                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4853         else if (flags & HIST_FIELD_FL_VAR_REF)
4854                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4855         else
4856                 seq_puts(m, "        VAL: normal u64 value\n");
4857
4858         if (flags & HIST_FIELD_FL_ALIAS)
4859                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4860 }
4861
4862 static int hist_field_debug_show(struct seq_file *m,
4863                                  struct hist_field *field, unsigned long flags)
4864 {
4865         if ((field->flags & flags) != flags) {
4866                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4867                 return -EINVAL;
4868         }
4869
4870         hist_field_debug_show_flags(m, field->flags);
4871         if (field->field)
4872                 seq_printf(m, "      ftrace_event_field name: %s\n",
4873                            field->field->name);
4874
4875         if (field->flags & HIST_FIELD_FL_VAR) {
4876                 seq_printf(m, "      var.name: %s\n", field->var.name);
4877                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4878                            field->var.idx);
4879         }
4880
4881         if (field->flags & HIST_FIELD_FL_ALIAS)
4882                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4883                            field->var_ref_idx);
4884
4885         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4886                 seq_printf(m, "      name: %s\n", field->name);
4887                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4888                            field->var.idx);
4889                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4890                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4891                            field->var_ref_idx);
4892                 if (field->system)
4893                         seq_printf(m, "      system: %s\n", field->system);
4894                 if (field->event_name)
4895                         seq_printf(m, "      event_name: %s\n", field->event_name);
4896         }
4897
4898         seq_printf(m, "      type: %s\n", field->type);
4899         seq_printf(m, "      size: %u\n", field->size);
4900         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4901
4902         return 0;
4903 }
4904
4905 static int field_var_debug_show(struct seq_file *m,
4906                                 struct field_var *field_var, unsigned int i,
4907                                 bool save_vars)
4908 {
4909         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4910         struct hist_field *field;
4911         int ret = 0;
4912
4913         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4914
4915         field = field_var->var;
4916
4917         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4918
4919         hist_field_debug_show_flags(m, field->flags);
4920         seq_printf(m, "      var.name: %s\n", field->var.name);
4921         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4922                    field->var.idx);
4923
4924         field = field_var->val;
4925
4926         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4927         if (field->field)
4928                 seq_printf(m, "      ftrace_event_field name: %s\n",
4929                            field->field->name);
4930         else {
4931                 ret = -EINVAL;
4932                 goto out;
4933         }
4934
4935         seq_printf(m, "      type: %s\n", field->type);
4936         seq_printf(m, "      size: %u\n", field->size);
4937         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4938 out:
4939         return ret;
4940 }
4941
4942 static int hist_action_debug_show(struct seq_file *m,
4943                                   struct action_data *data, int i)
4944 {
4945         int ret = 0;
4946
4947         if (data->handler == HANDLER_ONMAX ||
4948             data->handler == HANDLER_ONCHANGE) {
4949                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4950                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4951                                             HIST_FIELD_FL_VAR_REF);
4952                 if (ret)
4953                         goto out;
4954
4955                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4956                 ret = hist_field_debug_show(m, data->track_data.track_var,
4957                                             HIST_FIELD_FL_VAR);
4958                 if (ret)
4959                         goto out;
4960         }
4961
4962         if (data->handler == HANDLER_ONMATCH) {
4963                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4964                            i, data->match_data.event_system);
4965                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4966                            i, data->match_data.event);
4967         }
4968 out:
4969         return ret;
4970 }
4971
4972 static int hist_actions_debug_show(struct seq_file *m,
4973                                    struct hist_trigger_data *hist_data)
4974 {
4975         int i, ret = 0;
4976
4977         if (hist_data->n_actions)
4978                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4979
4980         for (i = 0; i < hist_data->n_actions; i++) {
4981                 struct action_data *action = hist_data->actions[i];
4982
4983                 ret = hist_action_debug_show(m, action, i);
4984                 if (ret)
4985                         goto out;
4986         }
4987
4988         if (hist_data->n_save_vars)
4989                 seq_puts(m, "\n  save action variables (save() params):\n");
4990
4991         for (i = 0; i < hist_data->n_save_vars; i++) {
4992                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4993                 if (ret)
4994                         goto out;
4995         }
4996 out:
4997         return ret;
4998 }
4999
5000 static void hist_trigger_debug_show(struct seq_file *m,
5001                                     struct event_trigger_data *data, int n)
5002 {
5003         struct hist_trigger_data *hist_data;
5004         int i, ret;
5005
5006         if (n > 0)
5007                 seq_puts(m, "\n\n");
5008
5009         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5010         data->ops->print(m, data->ops, data);
5011         seq_puts(m, "#\n\n");
5012
5013         hist_data = data->private_data;
5014
5015         seq_printf(m, "hist_data: %p\n\n", hist_data);
5016         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5017         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5018         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5019
5020         seq_puts(m, "\n  val fields:\n\n");
5021
5022         seq_puts(m, "    hist_data->fields[0]:\n");
5023         ret = hist_field_debug_show(m, hist_data->fields[0],
5024                                     HIST_FIELD_FL_HITCOUNT);
5025         if (ret)
5026                 return;
5027
5028         for (i = 1; i < hist_data->n_vals; i++) {
5029                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5030                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5031                 if (ret)
5032                         return;
5033         }
5034
5035         seq_puts(m, "\n  key fields:\n");
5036
5037         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5038                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5039                 ret = hist_field_debug_show(m, hist_data->fields[i],
5040                                             HIST_FIELD_FL_KEY);
5041                 if (ret)
5042                         return;
5043         }
5044
5045         if (hist_data->n_var_refs)
5046                 seq_puts(m, "\n  variable reference fields:\n");
5047
5048         for (i = 0; i < hist_data->n_var_refs; i++) {
5049                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5050                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5051                                             HIST_FIELD_FL_VAR_REF);
5052                 if (ret)
5053                         return;
5054         }
5055
5056         if (hist_data->n_field_vars)
5057                 seq_puts(m, "\n  field variables:\n");
5058
5059         for (i = 0; i < hist_data->n_field_vars; i++) {
5060                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5061                 if (ret)
5062                         return;
5063         }
5064
5065         ret = hist_actions_debug_show(m, hist_data);
5066         if (ret)
5067                 return;
5068 }
5069
5070 static int hist_debug_show(struct seq_file *m, void *v)
5071 {
5072         struct event_trigger_data *data;
5073         struct trace_event_file *event_file;
5074         int n = 0, ret = 0;
5075
5076         mutex_lock(&event_mutex);
5077
5078         event_file = event_file_data(m->private);
5079         if (unlikely(!event_file)) {
5080                 ret = -ENODEV;
5081                 goto out_unlock;
5082         }
5083
5084         list_for_each_entry(data, &event_file->triggers, list) {
5085                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5086                         hist_trigger_debug_show(m, data, n++);
5087         }
5088
5089  out_unlock:
5090         mutex_unlock(&event_mutex);
5091
5092         return ret;
5093 }
5094
5095 static int event_hist_debug_open(struct inode *inode, struct file *file)
5096 {
5097         int ret;
5098
5099         ret = security_locked_down(LOCKDOWN_TRACEFS);
5100         if (ret)
5101                 return ret;
5102
5103         return single_open(file, hist_debug_show, file);
5104 }
5105
5106 const struct file_operations event_hist_debug_fops = {
5107         .open = event_hist_debug_open,
5108         .read = seq_read,
5109         .llseek = seq_lseek,
5110         .release = single_release,
5111 };
5112 #endif
5113
5114 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5115 {
5116         const char *field_name = hist_field_name(hist_field, 0);
5117
5118         if (hist_field->var.name)
5119                 seq_printf(m, "%s=", hist_field->var.name);
5120
5121         if (hist_field->flags & HIST_FIELD_FL_CPU)
5122                 seq_puts(m, "common_cpu");
5123         else if (field_name) {
5124                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5125                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5126                         seq_putc(m, '$');
5127                 seq_printf(m, "%s", field_name);
5128         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5129                 seq_puts(m, "common_timestamp");
5130
5131         if (hist_field->flags) {
5132                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5133                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5134                         const char *flags = get_hist_field_flags(hist_field);
5135
5136                         if (flags)
5137                                 seq_printf(m, ".%s", flags);
5138                 }
5139         }
5140 }
5141
5142 static int event_hist_trigger_print(struct seq_file *m,
5143                                     struct event_trigger_ops *ops,
5144                                     struct event_trigger_data *data)
5145 {
5146         struct hist_trigger_data *hist_data = data->private_data;
5147         struct hist_field *field;
5148         bool have_var = false;
5149         unsigned int i;
5150
5151         seq_puts(m, "hist:");
5152
5153         if (data->name)
5154                 seq_printf(m, "%s:", data->name);
5155
5156         seq_puts(m, "keys=");
5157
5158         for_each_hist_key_field(i, hist_data) {
5159                 field = hist_data->fields[i];
5160
5161                 if (i > hist_data->n_vals)
5162                         seq_puts(m, ",");
5163
5164                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5165                         seq_puts(m, "stacktrace");
5166                 else
5167                         hist_field_print(m, field);
5168         }
5169
5170         seq_puts(m, ":vals=");
5171
5172         for_each_hist_val_field(i, hist_data) {
5173                 field = hist_data->fields[i];
5174                 if (field->flags & HIST_FIELD_FL_VAR) {
5175                         have_var = true;
5176                         continue;
5177                 }
5178
5179                 if (i == HITCOUNT_IDX)
5180                         seq_puts(m, "hitcount");
5181                 else {
5182                         seq_puts(m, ",");
5183                         hist_field_print(m, field);
5184                 }
5185         }
5186
5187         if (have_var) {
5188                 unsigned int n = 0;
5189
5190                 seq_puts(m, ":");
5191
5192                 for_each_hist_val_field(i, hist_data) {
5193                         field = hist_data->fields[i];
5194
5195                         if (field->flags & HIST_FIELD_FL_VAR) {
5196                                 if (n++)
5197                                         seq_puts(m, ",");
5198                                 hist_field_print(m, field);
5199                         }
5200                 }
5201         }
5202
5203         seq_puts(m, ":sort=");
5204
5205         for (i = 0; i < hist_data->n_sort_keys; i++) {
5206                 struct tracing_map_sort_key *sort_key;
5207                 unsigned int idx, first_key_idx;
5208
5209                 /* skip VAR vals */
5210                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5211
5212                 sort_key = &hist_data->sort_keys[i];
5213                 idx = sort_key->field_idx;
5214
5215                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5216                         return -EINVAL;
5217
5218                 if (i > 0)
5219                         seq_puts(m, ",");
5220
5221                 if (idx == HITCOUNT_IDX)
5222                         seq_puts(m, "hitcount");
5223                 else {
5224                         if (idx >= first_key_idx)
5225                                 idx += hist_data->n_vars;
5226                         hist_field_print(m, hist_data->fields[idx]);
5227                 }
5228
5229                 if (sort_key->descending)
5230                         seq_puts(m, ".descending");
5231         }
5232         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5233         if (hist_data->enable_timestamps)
5234                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5235
5236         print_actions_spec(m, hist_data);
5237
5238         if (data->filter_str)
5239                 seq_printf(m, " if %s", data->filter_str);
5240
5241         if (data->paused)
5242                 seq_puts(m, " [paused]");
5243         else
5244                 seq_puts(m, " [active]");
5245
5246         seq_putc(m, '\n');
5247
5248         return 0;
5249 }
5250
5251 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5252                                    struct event_trigger_data *data)
5253 {
5254         struct hist_trigger_data *hist_data = data->private_data;
5255
5256         if (!data->ref && hist_data->attrs->name)
5257                 save_named_trigger(hist_data->attrs->name, data);
5258
5259         data->ref++;
5260
5261         return 0;
5262 }
5263
5264 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5265 {
5266         struct trace_event_file *file;
5267         unsigned int i;
5268         char *cmd;
5269         int ret;
5270
5271         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5272                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5273                 cmd = hist_data->field_var_hists[i]->cmd;
5274                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5275                                               "!hist", "hist", cmd);
5276                 WARN_ON_ONCE(ret < 0);
5277         }
5278 }
5279
5280 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5281                                     struct event_trigger_data *data)
5282 {
5283         struct hist_trigger_data *hist_data = data->private_data;
5284
5285         if (WARN_ON_ONCE(data->ref <= 0))
5286                 return;
5287
5288         data->ref--;
5289         if (!data->ref) {
5290                 if (data->name)
5291                         del_named_trigger(data);
5292
5293                 trigger_data_free(data);
5294
5295                 remove_hist_vars(hist_data);
5296
5297                 unregister_field_var_hists(hist_data);
5298
5299                 destroy_hist_data(hist_data);
5300         }
5301 }
5302
5303 static struct event_trigger_ops event_hist_trigger_ops = {
5304         .func                   = event_hist_trigger,
5305         .print                  = event_hist_trigger_print,
5306         .init                   = event_hist_trigger_init,
5307         .free                   = event_hist_trigger_free,
5308 };
5309
5310 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5311                                          struct event_trigger_data *data)
5312 {
5313         data->ref++;
5314
5315         save_named_trigger(data->named_data->name, data);
5316
5317         event_hist_trigger_init(ops, data->named_data);
5318
5319         return 0;
5320 }
5321
5322 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5323                                           struct event_trigger_data *data)
5324 {
5325         if (WARN_ON_ONCE(data->ref <= 0))
5326                 return;
5327
5328         event_hist_trigger_free(ops, data->named_data);
5329
5330         data->ref--;
5331         if (!data->ref) {
5332                 del_named_trigger(data);
5333                 trigger_data_free(data);
5334         }
5335 }
5336
5337 static struct event_trigger_ops event_hist_trigger_named_ops = {
5338         .func                   = event_hist_trigger,
5339         .print                  = event_hist_trigger_print,
5340         .init                   = event_hist_trigger_named_init,
5341         .free                   = event_hist_trigger_named_free,
5342 };
5343
5344 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5345                                                             char *param)
5346 {
5347         return &event_hist_trigger_ops;
5348 }
5349
5350 static void hist_clear(struct event_trigger_data *data)
5351 {
5352         struct hist_trigger_data *hist_data = data->private_data;
5353
5354         if (data->name)
5355                 pause_named_trigger(data);
5356
5357         tracepoint_synchronize_unregister();
5358
5359         tracing_map_clear(hist_data->map);
5360
5361         if (data->name)
5362                 unpause_named_trigger(data);
5363 }
5364
5365 static bool compatible_field(struct ftrace_event_field *field,
5366                              struct ftrace_event_field *test_field)
5367 {
5368         if (field == test_field)
5369                 return true;
5370         if (field == NULL || test_field == NULL)
5371                 return false;
5372         if (strcmp(field->name, test_field->name) != 0)
5373                 return false;
5374         if (strcmp(field->type, test_field->type) != 0)
5375                 return false;
5376         if (field->size != test_field->size)
5377                 return false;
5378         if (field->is_signed != test_field->is_signed)
5379                 return false;
5380
5381         return true;
5382 }
5383
5384 static bool hist_trigger_match(struct event_trigger_data *data,
5385                                struct event_trigger_data *data_test,
5386                                struct event_trigger_data *named_data,
5387                                bool ignore_filter)
5388 {
5389         struct tracing_map_sort_key *sort_key, *sort_key_test;
5390         struct hist_trigger_data *hist_data, *hist_data_test;
5391         struct hist_field *key_field, *key_field_test;
5392         unsigned int i;
5393
5394         if (named_data && (named_data != data_test) &&
5395             (named_data != data_test->named_data))
5396                 return false;
5397
5398         if (!named_data && is_named_trigger(data_test))
5399                 return false;
5400
5401         hist_data = data->private_data;
5402         hist_data_test = data_test->private_data;
5403
5404         if (hist_data->n_vals != hist_data_test->n_vals ||
5405             hist_data->n_fields != hist_data_test->n_fields ||
5406             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5407                 return false;
5408
5409         if (!ignore_filter) {
5410                 if ((data->filter_str && !data_test->filter_str) ||
5411                    (!data->filter_str && data_test->filter_str))
5412                         return false;
5413         }
5414
5415         for_each_hist_field(i, hist_data) {
5416                 key_field = hist_data->fields[i];
5417                 key_field_test = hist_data_test->fields[i];
5418
5419                 if (key_field->flags != key_field_test->flags)
5420                         return false;
5421                 if (!compatible_field(key_field->field, key_field_test->field))
5422                         return false;
5423                 if (key_field->offset != key_field_test->offset)
5424                         return false;
5425                 if (key_field->size != key_field_test->size)
5426                         return false;
5427                 if (key_field->is_signed != key_field_test->is_signed)
5428                         return false;
5429                 if (!!key_field->var.name != !!key_field_test->var.name)
5430                         return false;
5431                 if (key_field->var.name &&
5432                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5433                         return false;
5434         }
5435
5436         for (i = 0; i < hist_data->n_sort_keys; i++) {
5437                 sort_key = &hist_data->sort_keys[i];
5438                 sort_key_test = &hist_data_test->sort_keys[i];
5439
5440                 if (sort_key->field_idx != sort_key_test->field_idx ||
5441                     sort_key->descending != sort_key_test->descending)
5442                         return false;
5443         }
5444
5445         if (!ignore_filter && data->filter_str &&
5446             (strcmp(data->filter_str, data_test->filter_str) != 0))
5447                 return false;
5448
5449         if (!actions_match(hist_data, hist_data_test))
5450                 return false;
5451
5452         return true;
5453 }
5454
5455 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5456                                  struct event_trigger_data *data,
5457                                  struct trace_event_file *file)
5458 {
5459         struct hist_trigger_data *hist_data = data->private_data;
5460         struct event_trigger_data *test, *named_data = NULL;
5461         struct trace_array *tr = file->tr;
5462         int ret = 0;
5463
5464         if (hist_data->attrs->name) {
5465                 named_data = find_named_trigger(hist_data->attrs->name);
5466                 if (named_data) {
5467                         if (!hist_trigger_match(data, named_data, named_data,
5468                                                 true)) {
5469                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5470                                 ret = -EINVAL;
5471                                 goto out;
5472                         }
5473                 }
5474         }
5475
5476         if (hist_data->attrs->name && !named_data)
5477                 goto new;
5478
5479         lockdep_assert_held(&event_mutex);
5480
5481         list_for_each_entry(test, &file->triggers, list) {
5482                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5483                         if (!hist_trigger_match(data, test, named_data, false))
5484                                 continue;
5485                         if (hist_data->attrs->pause)
5486                                 test->paused = true;
5487                         else if (hist_data->attrs->cont)
5488                                 test->paused = false;
5489                         else if (hist_data->attrs->clear)
5490                                 hist_clear(test);
5491                         else {
5492                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5493                                 ret = -EEXIST;
5494                         }
5495                         goto out;
5496                 }
5497         }
5498  new:
5499         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5500                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5501                 ret = -ENOENT;
5502                 goto out;
5503         }
5504
5505         if (hist_data->attrs->pause)
5506                 data->paused = true;
5507
5508         if (named_data) {
5509                 data->private_data = named_data->private_data;
5510                 set_named_trigger_data(data, named_data);
5511                 data->ops = &event_hist_trigger_named_ops;
5512         }
5513
5514         if (data->ops->init) {
5515                 ret = data->ops->init(data->ops, data);
5516                 if (ret < 0)
5517                         goto out;
5518         }
5519
5520         if (hist_data->enable_timestamps) {
5521                 char *clock = hist_data->attrs->clock;
5522
5523                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5524                 if (ret) {
5525                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5526                         goto out;
5527                 }
5528
5529                 tracing_set_filter_buffering(file->tr, true);
5530         }
5531
5532         if (named_data)
5533                 destroy_hist_data(hist_data);
5534
5535         ret++;
5536  out:
5537         return ret;
5538 }
5539
5540 static int hist_trigger_enable(struct event_trigger_data *data,
5541                                struct trace_event_file *file)
5542 {
5543         int ret = 0;
5544
5545         list_add_tail_rcu(&data->list, &file->triggers);
5546
5547         update_cond_flag(file);
5548
5549         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5550                 list_del_rcu(&data->list);
5551                 update_cond_flag(file);
5552                 ret--;
5553         }
5554
5555         return ret;
5556 }
5557
5558 static bool have_hist_trigger_match(struct event_trigger_data *data,
5559                                     struct trace_event_file *file)
5560 {
5561         struct hist_trigger_data *hist_data = data->private_data;
5562         struct event_trigger_data *test, *named_data = NULL;
5563         bool match = false;
5564
5565         lockdep_assert_held(&event_mutex);
5566
5567         if (hist_data->attrs->name)
5568                 named_data = find_named_trigger(hist_data->attrs->name);
5569
5570         list_for_each_entry(test, &file->triggers, list) {
5571                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5572                         if (hist_trigger_match(data, test, named_data, false)) {
5573                                 match = true;
5574                                 break;
5575                         }
5576                 }
5577         }
5578
5579         return match;
5580 }
5581
5582 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5583                                     struct trace_event_file *file)
5584 {
5585         struct hist_trigger_data *hist_data = data->private_data;
5586         struct event_trigger_data *test, *named_data = NULL;
5587
5588         lockdep_assert_held(&event_mutex);
5589
5590         if (hist_data->attrs->name)
5591                 named_data = find_named_trigger(hist_data->attrs->name);
5592
5593         list_for_each_entry(test, &file->triggers, list) {
5594                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5595                         if (!hist_trigger_match(data, test, named_data, false))
5596                                 continue;
5597                         hist_data = test->private_data;
5598                         if (check_var_refs(hist_data))
5599                                 return true;
5600                         break;
5601                 }
5602         }
5603
5604         return false;
5605 }
5606
5607 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5608                                     struct event_trigger_data *data,
5609                                     struct trace_event_file *file)
5610 {
5611         struct hist_trigger_data *hist_data = data->private_data;
5612         struct event_trigger_data *test, *named_data = NULL;
5613         bool unregistered = false;
5614
5615         lockdep_assert_held(&event_mutex);
5616
5617         if (hist_data->attrs->name)
5618                 named_data = find_named_trigger(hist_data->attrs->name);
5619
5620         list_for_each_entry(test, &file->triggers, list) {
5621                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5622                         if (!hist_trigger_match(data, test, named_data, false))
5623                                 continue;
5624                         unregistered = true;
5625                         list_del_rcu(&test->list);
5626                         trace_event_trigger_enable_disable(file, 0);
5627                         update_cond_flag(file);
5628                         break;
5629                 }
5630         }
5631
5632         if (unregistered && test->ops->free)
5633                 test->ops->free(test->ops, test);
5634
5635         if (hist_data->enable_timestamps) {
5636                 if (!hist_data->remove || unregistered)
5637                         tracing_set_filter_buffering(file->tr, false);
5638         }
5639 }
5640
5641 static bool hist_file_check_refs(struct trace_event_file *file)
5642 {
5643         struct hist_trigger_data *hist_data;
5644         struct event_trigger_data *test;
5645
5646         lockdep_assert_held(&event_mutex);
5647
5648         list_for_each_entry(test, &file->triggers, list) {
5649                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5650                         hist_data = test->private_data;
5651                         if (check_var_refs(hist_data))
5652                                 return true;
5653                 }
5654         }
5655
5656         return false;
5657 }
5658
5659 static void hist_unreg_all(struct trace_event_file *file)
5660 {
5661         struct event_trigger_data *test, *n;
5662         struct hist_trigger_data *hist_data;
5663         struct synth_event *se;
5664         const char *se_name;
5665
5666         lockdep_assert_held(&event_mutex);
5667
5668         if (hist_file_check_refs(file))
5669                 return;
5670
5671         list_for_each_entry_safe(test, n, &file->triggers, list) {
5672                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5673                         hist_data = test->private_data;
5674                         list_del_rcu(&test->list);
5675                         trace_event_trigger_enable_disable(file, 0);
5676
5677                         se_name = trace_event_name(file->event_call);
5678                         se = find_synth_event(se_name);
5679                         if (se)
5680                                 se->ref--;
5681
5682                         update_cond_flag(file);
5683                         if (hist_data->enable_timestamps)
5684                                 tracing_set_filter_buffering(file->tr, false);
5685                         if (test->ops->free)
5686                                 test->ops->free(test->ops, test);
5687                 }
5688         }
5689 }
5690
5691 static int event_hist_trigger_func(struct event_command *cmd_ops,
5692                                    struct trace_event_file *file,
5693                                    char *glob, char *cmd, char *param)
5694 {
5695         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5696         struct event_trigger_data *trigger_data;
5697         struct hist_trigger_attrs *attrs;
5698         struct event_trigger_ops *trigger_ops;
5699         struct hist_trigger_data *hist_data;
5700         struct synth_event *se;
5701         const char *se_name;
5702         bool remove = false;
5703         char *trigger, *p;
5704         int ret = 0;
5705
5706         lockdep_assert_held(&event_mutex);
5707
5708         if (glob && strlen(glob)) {
5709                 hist_err_clear();
5710                 last_cmd_set(file, param);
5711         }
5712
5713         if (!param)
5714                 return -EINVAL;
5715
5716         if (glob[0] == '!')
5717                 remove = true;
5718
5719         /*
5720          * separate the trigger from the filter (k:v [if filter])
5721          * allowing for whitespace in the trigger
5722          */
5723         p = trigger = param;
5724         do {
5725                 p = strstr(p, "if");
5726                 if (!p)
5727                         break;
5728                 if (p == param)
5729                         return -EINVAL;
5730                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5731                         p++;
5732                         continue;
5733                 }
5734                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5735                         return -EINVAL;
5736                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5737                         p++;
5738                         continue;
5739                 }
5740                 break;
5741         } while (p);
5742
5743         if (!p)
5744                 param = NULL;
5745         else {
5746                 *(p - 1) = '\0';
5747                 param = strstrip(p);
5748                 trigger = strstrip(trigger);
5749         }
5750
5751         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5752         if (IS_ERR(attrs))
5753                 return PTR_ERR(attrs);
5754
5755         if (attrs->map_bits)
5756                 hist_trigger_bits = attrs->map_bits;
5757
5758         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5759         if (IS_ERR(hist_data)) {
5760                 destroy_hist_trigger_attrs(attrs);
5761                 return PTR_ERR(hist_data);
5762         }
5763
5764         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5765
5766         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5767         if (!trigger_data) {
5768                 ret = -ENOMEM;
5769                 goto out_free;
5770         }
5771
5772         trigger_data->count = -1;
5773         trigger_data->ops = trigger_ops;
5774         trigger_data->cmd_ops = cmd_ops;
5775
5776         INIT_LIST_HEAD(&trigger_data->list);
5777         RCU_INIT_POINTER(trigger_data->filter, NULL);
5778
5779         trigger_data->private_data = hist_data;
5780
5781         /* if param is non-empty, it's supposed to be a filter */
5782         if (param && cmd_ops->set_filter) {
5783                 ret = cmd_ops->set_filter(param, trigger_data, file);
5784                 if (ret < 0)
5785                         goto out_free;
5786         }
5787
5788         if (remove) {
5789                 if (!have_hist_trigger_match(trigger_data, file))
5790                         goto out_free;
5791
5792                 if (hist_trigger_check_refs(trigger_data, file)) {
5793                         ret = -EBUSY;
5794                         goto out_free;
5795                 }
5796
5797                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5798                 se_name = trace_event_name(file->event_call);
5799                 se = find_synth_event(se_name);
5800                 if (se)
5801                         se->ref--;
5802                 ret = 0;
5803                 goto out_free;
5804         }
5805
5806         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5807         /*
5808          * The above returns on success the # of triggers registered,
5809          * but if it didn't register any it returns zero.  Consider no
5810          * triggers registered a failure too.
5811          */
5812         if (!ret) {
5813                 if (!(attrs->pause || attrs->cont || attrs->clear))
5814                         ret = -ENOENT;
5815                 goto out_free;
5816         } else if (ret < 0)
5817                 goto out_free;
5818
5819         if (get_named_trigger_data(trigger_data))
5820                 goto enable;
5821
5822         if (has_hist_vars(hist_data))
5823                 save_hist_vars(hist_data);
5824
5825         ret = create_actions(hist_data);
5826         if (ret)
5827                 goto out_unreg;
5828
5829         ret = tracing_map_init(hist_data->map);
5830         if (ret)
5831                 goto out_unreg;
5832 enable:
5833         ret = hist_trigger_enable(trigger_data, file);
5834         if (ret)
5835                 goto out_unreg;
5836
5837         se_name = trace_event_name(file->event_call);
5838         se = find_synth_event(se_name);
5839         if (se)
5840                 se->ref++;
5841         /* Just return zero, not the number of registered triggers */
5842         ret = 0;
5843  out:
5844         if (ret == 0)
5845                 hist_err_clear();
5846
5847         return ret;
5848  out_unreg:
5849         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5850  out_free:
5851         if (cmd_ops->set_filter)
5852                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5853
5854         remove_hist_vars(hist_data);
5855
5856         kfree(trigger_data);
5857
5858         destroy_hist_data(hist_data);
5859         goto out;
5860 }
5861
5862 static struct event_command trigger_hist_cmd = {
5863         .name                   = "hist",
5864         .trigger_type           = ETT_EVENT_HIST,
5865         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5866         .func                   = event_hist_trigger_func,
5867         .reg                    = hist_register_trigger,
5868         .unreg                  = hist_unregister_trigger,
5869         .unreg_all              = hist_unreg_all,
5870         .get_trigger_ops        = event_hist_get_trigger_ops,
5871         .set_filter             = set_trigger_filter,
5872 };
5873
5874 __init int register_trigger_hist_cmd(void)
5875 {
5876         int ret;
5877
5878         ret = register_event_command(&trigger_hist_cmd);
5879         WARN_ON(ret < 0);
5880
5881         return ret;
5882 }
5883
5884 static void
5885 hist_enable_trigger(struct event_trigger_data *data,
5886                     struct trace_buffer *buffer,  void *rec,
5887                     struct ring_buffer_event *event)
5888 {
5889         struct enable_trigger_data *enable_data = data->private_data;
5890         struct event_trigger_data *test;
5891
5892         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5893                                 lockdep_is_held(&event_mutex)) {
5894                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5895                         if (enable_data->enable)
5896                                 test->paused = false;
5897                         else
5898                                 test->paused = true;
5899                 }
5900         }
5901 }
5902
5903 static void
5904 hist_enable_count_trigger(struct event_trigger_data *data,
5905                           struct trace_buffer *buffer,  void *rec,
5906                           struct ring_buffer_event *event)
5907 {
5908         if (!data->count)
5909                 return;
5910
5911         if (data->count != -1)
5912                 (data->count)--;
5913
5914         hist_enable_trigger(data, buffer, rec, event);
5915 }
5916
5917 static struct event_trigger_ops hist_enable_trigger_ops = {
5918         .func                   = hist_enable_trigger,
5919         .print                  = event_enable_trigger_print,
5920         .init                   = event_trigger_init,
5921         .free                   = event_enable_trigger_free,
5922 };
5923
5924 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5925         .func                   = hist_enable_count_trigger,
5926         .print                  = event_enable_trigger_print,
5927         .init                   = event_trigger_init,
5928         .free                   = event_enable_trigger_free,
5929 };
5930
5931 static struct event_trigger_ops hist_disable_trigger_ops = {
5932         .func                   = hist_enable_trigger,
5933         .print                  = event_enable_trigger_print,
5934         .init                   = event_trigger_init,
5935         .free                   = event_enable_trigger_free,
5936 };
5937
5938 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5939         .func                   = hist_enable_count_trigger,
5940         .print                  = event_enable_trigger_print,
5941         .init                   = event_trigger_init,
5942         .free                   = event_enable_trigger_free,
5943 };
5944
5945 static struct event_trigger_ops *
5946 hist_enable_get_trigger_ops(char *cmd, char *param)
5947 {
5948         struct event_trigger_ops *ops;
5949         bool enable;
5950
5951         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5952
5953         if (enable)
5954                 ops = param ? &hist_enable_count_trigger_ops :
5955                         &hist_enable_trigger_ops;
5956         else
5957                 ops = param ? &hist_disable_count_trigger_ops :
5958                         &hist_disable_trigger_ops;
5959
5960         return ops;
5961 }
5962
5963 static void hist_enable_unreg_all(struct trace_event_file *file)
5964 {
5965         struct event_trigger_data *test, *n;
5966
5967         list_for_each_entry_safe(test, n, &file->triggers, list) {
5968                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5969                         list_del_rcu(&test->list);
5970                         update_cond_flag(file);
5971                         trace_event_trigger_enable_disable(file, 0);
5972                         if (test->ops->free)
5973                                 test->ops->free(test->ops, test);
5974                 }
5975         }
5976 }
5977
5978 static struct event_command trigger_hist_enable_cmd = {
5979         .name                   = ENABLE_HIST_STR,
5980         .trigger_type           = ETT_HIST_ENABLE,
5981         .func                   = event_enable_trigger_func,
5982         .reg                    = event_enable_register_trigger,
5983         .unreg                  = event_enable_unregister_trigger,
5984         .unreg_all              = hist_enable_unreg_all,
5985         .get_trigger_ops        = hist_enable_get_trigger_ops,
5986         .set_filter             = set_trigger_filter,
5987 };
5988
5989 static struct event_command trigger_hist_disable_cmd = {
5990         .name                   = DISABLE_HIST_STR,
5991         .trigger_type           = ETT_HIST_ENABLE,
5992         .func                   = event_enable_trigger_func,
5993         .reg                    = event_enable_register_trigger,
5994         .unreg                  = event_enable_unregister_trigger,
5995         .unreg_all              = hist_enable_unreg_all,
5996         .get_trigger_ops        = hist_enable_get_trigger_ops,
5997         .set_filter             = set_trigger_filter,
5998 };
5999
6000 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6001 {
6002         unregister_event_command(&trigger_hist_enable_cmd);
6003         unregister_event_command(&trigger_hist_disable_cmd);
6004 }
6005
6006 __init int register_trigger_hist_enable_disable_cmds(void)
6007 {
6008         int ret;
6009
6010         ret = register_event_command(&trigger_hist_enable_cmd);
6011         if (WARN_ON(ret < 0))
6012                 return ret;
6013         ret = register_event_command(&trigger_hist_disable_cmd);
6014         if (WARN_ON(ret < 0))
6015                 unregister_trigger_hist_enable_disable_cmds();
6016
6017         return ret;
6018 }