kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"),
  70
  71 #undef C
  72 #define C(a, b)         HIST_ERR_##a
  73
  74 enum { ERRORS };
  75
  76 #undef C
  77 #define C(a, b)         b
  78
  79 static const char *err_text[] = { ERRORS };
  80
  81 struct hist_field;
  82
  83 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84                                 struct tracing_map_elt *elt,
  85                                 struct trace_buffer *buffer,
  86                                 struct ring_buffer_event *rbe,
  87                                 void *event);
  88
  89 #define HIST_FIELD_OPERANDS_MAX 2
  90 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  91 #define HIST_ACTIONS_MAX        8
  92
  93 enum field_op_id {
  94         FIELD_OP_NONE,
  95         FIELD_OP_PLUS,
  96         FIELD_OP_MINUS,
  97         FIELD_OP_UNARY_MINUS,
  98 };
  99
 100 /*
 101  * A hist_var (histogram variable) contains variable information for
 102  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 103  * flag set.  A hist_var has a variable name e.g. ts0, and is
 104  * associated with a given histogram trigger, as specified by
 105  * hist_data.  The hist_var idx is the unique index assigned to the
 106  * variable by the hist trigger's tracing_map.  The idx is what is
 107  * used to set a variable's value and, by a variable reference, to
 108  * retrieve it.
 109  */
 110 struct hist_var {
 111         char                            *name;
 112         struct hist_trigger_data        *hist_data;
 113         unsigned int                    idx;
 114 };
 115
 116 struct hist_field {
 117         struct ftrace_event_field       *field;
 118         unsigned long                   flags;
 119         hist_field_fn_t                 fn;
 120         unsigned int                    ref;
 121         unsigned int                    size;
 122         unsigned int                    offset;
 123         unsigned int                    is_signed;
 124         const char                      *type;
 125         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 126         struct hist_trigger_data        *hist_data;
 127
 128         /*
 129          * Variable fields contain variable-specific info in var.
 130          */
 131         struct hist_var                 var;
 132         enum field_op_id                operator;
 133         char                            *system;
 134         char                            *event_name;
 135
 136         /*
 137          * The name field is used for EXPR and VAR_REF fields.  VAR
 138          * fields contain the variable name in var.name.
 139          */
 140         char                            *name;
 141
 142         /*
 143          * When a histogram trigger is hit, if it has any references
 144          * to variables, the values of those variables are collected
 145          * into a var_ref_vals array by resolve_var_refs().  The
 146          * current value of each variable is read from the tracing_map
 147          * using the hist field's hist_var.idx and entered into the
 148          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 149          */
 150         unsigned int                    var_ref_idx;
 151         bool                            read_once;
 152
 153         unsigned int                    var_str_idx;
 154 };
 155
 156 static u64 hist_field_none(struct hist_field *field,
 157                            struct tracing_map_elt *elt,
 158                            struct trace_buffer *buffer,
 159                            struct ring_buffer_event *rbe,
 160                            void *event)
 161 {
 162         return 0;
 163 }
 164
 165 static u64 hist_field_counter(struct hist_field *field,
 166                               struct tracing_map_elt *elt,
 167                               struct trace_buffer *buffer,
 168                               struct ring_buffer_event *rbe,
 169                               void *event)
 170 {
 171         return 1;
 172 }
 173
 174 static u64 hist_field_string(struct hist_field *hist_field,
 175                              struct tracing_map_elt *elt,
 176                              struct trace_buffer *buffer,
 177                              struct ring_buffer_event *rbe,
 178                              void *event)
 179 {
 180         char *addr = (char *)(event + hist_field->field->offset);
 181
 182         return (u64)(unsigned long)addr;
 183 }
 184
 185 static u64 hist_field_dynstring(struct hist_field *hist_field,
 186                                 struct tracing_map_elt *elt,
 187                                 struct trace_buffer *buffer,
 188                                 struct ring_buffer_event *rbe,
 189                                 void *event)
 190 {
 191         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 192         int str_loc = str_item & 0xffff;
 193         char *addr = (char *)(event + str_loc);
 194
 195         return (u64)(unsigned long)addr;
 196 }
 197
 198 static u64 hist_field_pstring(struct hist_field *hist_field,
 199                               struct tracing_map_elt *elt,
 200                               struct trace_buffer *buffer,
 201                               struct ring_buffer_event *rbe,
 202                               void *event)
 203 {
 204         char **addr = (char **)(event + hist_field->field->offset);
 205
 206         return (u64)(unsigned long)*addr;
 207 }
 208
 209 static u64 hist_field_log2(struct hist_field *hist_field,
 210                            struct tracing_map_elt *elt,
 211                            struct trace_buffer *buffer,
 212                            struct ring_buffer_event *rbe,
 213                            void *event)
 214 {
 215         struct hist_field *operand = hist_field->operands[0];
 216
 217         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 218
 219         return (u64) ilog2(roundup_pow_of_two(val));
 220 }
 221
 222 static u64 hist_field_plus(struct hist_field *hist_field,
 223                            struct tracing_map_elt *elt,
 224                            struct trace_buffer *buffer,
 225                            struct ring_buffer_event *rbe,
 226                            void *event)
 227 {
 228         struct hist_field *operand1 = hist_field->operands[0];
 229         struct hist_field *operand2 = hist_field->operands[1];
 230
 231         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 232         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 233
 234         return val1 + val2;
 235 }
 236
 237 static u64 hist_field_minus(struct hist_field *hist_field,
 238                             struct tracing_map_elt *elt,
 239                             struct trace_buffer *buffer,
 240                             struct ring_buffer_event *rbe,
 241                             void *event)
 242 {
 243         struct hist_field *operand1 = hist_field->operands[0];
 244         struct hist_field *operand2 = hist_field->operands[1];
 245
 246         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 247         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 248
 249         return val1 - val2;
 250 }
 251
 252 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 253                                   struct tracing_map_elt *elt,
 254                                   struct trace_buffer *buffer,
 255                                   struct ring_buffer_event *rbe,
 256                                   void *event)
 257 {
 258         struct hist_field *operand = hist_field->operands[0];
 259
 260         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 261         u64 val = (u64)-sval;
 262
 263         return val;
 264 }
 265
 266 #define DEFINE_HIST_FIELD_FN(type)                                      \
 267         static u64 hist_field_##type(struct hist_field *hist_field,     \
 268                                      struct tracing_map_elt *elt,       \
 269                                      struct trace_buffer *buffer,       \
 270                                      struct ring_buffer_event *rbe,     \
 271                                      void *event)                       \
 272 {                                                                       \
 273         type *addr = (type *)(event + hist_field->field->offset);       \
 274                                                                         \
 275         return (u64)(unsigned long)*addr;                               \
 276 }
 277
 278 DEFINE_HIST_FIELD_FN(s64);
 279 DEFINE_HIST_FIELD_FN(u64);
 280 DEFINE_HIST_FIELD_FN(s32);
 281 DEFINE_HIST_FIELD_FN(u32);
 282 DEFINE_HIST_FIELD_FN(s16);
 283 DEFINE_HIST_FIELD_FN(u16);
 284 DEFINE_HIST_FIELD_FN(s8);
 285 DEFINE_HIST_FIELD_FN(u8);
 286
 287 #define for_each_hist_field(i, hist_data)       \
 288         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 289
 290 #define for_each_hist_val_field(i, hist_data)   \
 291         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 292
 293 #define for_each_hist_key_field(i, hist_data)   \
 294         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 295
 296 #define HIST_STACKTRACE_DEPTH   16
 297 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 298 #define HIST_STACKTRACE_SKIP    5
 299
 300 #define HITCOUNT_IDX            0
 301 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 302
 303 enum hist_field_flags {
 304         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 305         HIST_FIELD_FL_KEY               = 1 << 1,
 306         HIST_FIELD_FL_STRING            = 1 << 2,
 307         HIST_FIELD_FL_HEX               = 1 << 3,
 308         HIST_FIELD_FL_SYM               = 1 << 4,
 309         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 310         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 311         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 312         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 313         HIST_FIELD_FL_LOG2              = 1 << 9,
 314         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 315         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 316         HIST_FIELD_FL_VAR               = 1 << 12,
 317         HIST_FIELD_FL_EXPR              = 1 << 13,
 318         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 319         HIST_FIELD_FL_CPU               = 1 << 15,
 320         HIST_FIELD_FL_ALIAS             = 1 << 16,
 321 };
 322
 323 struct var_defs {
 324         unsigned int    n_vars;
 325         char            *name[TRACING_MAP_VARS_MAX];
 326         char            *expr[TRACING_MAP_VARS_MAX];
 327 };
 328
 329 struct hist_trigger_attrs {
 330         char            *keys_str;
 331         char            *vals_str;
 332         char            *sort_key_str;
 333         char            *name;
 334         char            *clock;
 335         bool            pause;
 336         bool            cont;
 337         bool            clear;
 338         bool            ts_in_usecs;
 339         unsigned int    map_bits;
 340
 341         char            *assignment_str[TRACING_MAP_VARS_MAX];
 342         unsigned int    n_assignments;
 343
 344         char            *action_str[HIST_ACTIONS_MAX];
 345         unsigned int    n_actions;
 346
 347         struct var_defs var_defs;
 348 };
 349
 350 struct field_var {
 351         struct hist_field       *var;
 352         struct hist_field       *val;
 353 };
 354
 355 struct field_var_hist {
 356         struct hist_trigger_data        *hist_data;
 357         char                            *cmd;
 358 };
 359
 360 struct hist_trigger_data {
 361         struct hist_field               *fields[HIST_FIELDS_MAX];
 362         unsigned int                    n_vals;
 363         unsigned int                    n_keys;
 364         unsigned int                    n_fields;
 365         unsigned int                    n_vars;
 366         unsigned int                    n_var_str;
 367         unsigned int                    key_size;
 368         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 369         unsigned int                    n_sort_keys;
 370         struct trace_event_file         *event_file;
 371         struct hist_trigger_attrs       *attrs;
 372         struct tracing_map              *map;
 373         bool                            enable_timestamps;
 374         bool                            remove;
 375         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 376         unsigned int                    n_var_refs;
 377
 378         struct action_data              *actions[HIST_ACTIONS_MAX];
 379         unsigned int                    n_actions;
 380
 381         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 382         unsigned int                    n_field_vars;
 383         unsigned int                    n_field_var_str;
 384         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 385         unsigned int                    n_field_var_hists;
 386
 387         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 388         unsigned int                    n_save_vars;
 389         unsigned int                    n_save_var_str;
 390 };
 391
 392 struct action_data;
 393
 394 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 395                              struct tracing_map_elt *elt,
 396                              struct trace_buffer *buffer, void *rec,
 397                              struct ring_buffer_event *rbe, void *key,
 398                              struct action_data *data, u64 *var_ref_vals);
 399
 400 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 401
 402 enum handler_id {
 403         HANDLER_ONMATCH = 1,
 404         HANDLER_ONMAX,
 405         HANDLER_ONCHANGE,
 406 };
 407
 408 enum action_id {
 409         ACTION_SAVE = 1,
 410         ACTION_TRACE,
 411         ACTION_SNAPSHOT,
 412 };
 413
 414 struct action_data {
 415         enum handler_id         handler;
 416         enum action_id          action;
 417         char                    *action_name;
 418         action_fn_t             fn;
 419
 420         unsigned int            n_params;
 421         char                    *params[SYNTH_FIELDS_MAX];
 422
 423         /*
 424          * When a histogram trigger is hit, the values of any
 425          * references to variables, including variables being passed
 426          * as parameters to synthetic events, are collected into a
 427          * var_ref_vals array.  This var_ref_idx array is an array of
 428          * indices into the var_ref_vals array, one for each synthetic
 429          * event param, and is passed to the synthetic event
 430          * invocation.
 431          */
 432         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 433         struct synth_event      *synth_event;
 434         bool                    use_trace_keyword;
 435         char                    *synth_event_name;
 436
 437         union {
 438                 struct {
 439                         char                    *event;
 440                         char                    *event_system;
 441                 } match_data;
 442
 443                 struct {
 444                         /*
 445                          * var_str contains the $-unstripped variable
 446                          * name referenced by var_ref, and used when
 447                          * printing the action.  Because var_ref
 448                          * creation is deferred to create_actions(),
 449                          * we need a per-action way to save it until
 450                          * then, thus var_str.
 451                          */
 452                         char                    *var_str;
 453
 454                         /*
 455                          * var_ref refers to the variable being
 456                          * tracked e.g onmax($var).
 457                          */
 458                         struct hist_field       *var_ref;
 459
 460                         /*
 461                          * track_var contains the 'invisible' tracking
 462                          * variable created to keep the current
 463                          * e.g. max value.
 464                          */
 465                         struct hist_field       *track_var;
 466
 467                         check_track_val_fn_t    check_val;
 468                         action_fn_t             save_data;
 469                 } track_data;
 470         };
 471 };
 472
 473 struct track_data {
 474         u64                             track_val;
 475         bool                            updated;
 476
 477         unsigned int                    key_len;
 478         void                            *key;
 479         struct tracing_map_elt          elt;
 480
 481         struct action_data              *action_data;
 482         struct hist_trigger_data        *hist_data;
 483 };
 484
 485 struct hist_elt_data {
 486         char *comm;
 487         u64 *var_ref_vals;
 488         char *field_var_str[SYNTH_FIELDS_MAX];
 489 };
 490
 491 struct snapshot_context {
 492         struct tracing_map_elt  *elt;
 493         void                    *key;
 494 };
 495
 496 static void track_data_free(struct track_data *track_data)
 497 {
 498         struct hist_elt_data *elt_data;
 499
 500         if (!track_data)
 501                 return;
 502
 503         kfree(track_data->key);
 504
 505         elt_data = track_data->elt.private_data;
 506         if (elt_data) {
 507                 kfree(elt_data->comm);
 508                 kfree(elt_data);
 509         }
 510
 511         kfree(track_data);
 512 }
 513
 514 static struct track_data *track_data_alloc(unsigned int key_len,
 515                                            struct action_data *action_data,
 516                                            struct hist_trigger_data *hist_data)
 517 {
 518         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 519         struct hist_elt_data *elt_data;
 520
 521         if (!data)
 522                 return ERR_PTR(-ENOMEM);
 523
 524         data->key = kzalloc(key_len, GFP_KERNEL);
 525         if (!data->key) {
 526                 track_data_free(data);
 527                 return ERR_PTR(-ENOMEM);
 528         }
 529
 530         data->key_len = key_len;
 531         data->action_data = action_data;
 532         data->hist_data = hist_data;
 533
 534         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 535         if (!elt_data) {
 536                 track_data_free(data);
 537                 return ERR_PTR(-ENOMEM);
 538         }
 539
 540         data->elt.private_data = elt_data;
 541
 542         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 543         if (!elt_data->comm) {
 544                 track_data_free(data);
 545                 return ERR_PTR(-ENOMEM);
 546         }
 547
 548         return data;
 549 }
 550
 551 static char last_cmd[MAX_FILTER_STR_VAL];
 552 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 553
 554 static int errpos(char *str)
 555 {
 556         return err_pos(last_cmd, str);
 557 }
 558
 559 static void last_cmd_set(struct trace_event_file *file, char *str)
 560 {
 561         const char *system = NULL, *name = NULL;
 562         struct trace_event_call *call;
 563
 564         if (!str)
 565                 return;
 566
 567         strcpy(last_cmd, "hist:");
 568         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 569
 570         if (file) {
 571                 call = file->event_call;
 572                 system = call->class->system;
 573                 if (system) {
 574                         name = trace_event_name(call);
 575                         if (!name)
 576                                 system = NULL;
 577                 }
 578         }
 579
 580         if (system)
 581                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 582 }
 583
 584 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 585 {
 586         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 587                         err_type, err_pos);
 588 }
 589
 590 static void hist_err_clear(void)
 591 {
 592         last_cmd[0] = '\0';
 593         last_cmd_loc[0] = '\0';
 594 }
 595
 596 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 597                                     unsigned int *var_ref_idx);
 598
 599 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 600                                unsigned int *var_ref_idx)
 601 {
 602         struct tracepoint *tp = event->tp;
 603
 604         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 605                 struct tracepoint_func *probe_func_ptr;
 606                 synth_probe_func_t probe_func;
 607                 void *__data;
 608
 609                 if (!(cpu_online(raw_smp_processor_id())))
 610                         return;
 611
 612                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 613                 if (probe_func_ptr) {
 614                         do {
 615                                 probe_func = probe_func_ptr->func;
 616                                 __data = probe_func_ptr->data;
 617                                 probe_func(__data, var_ref_vals, var_ref_idx);
 618                         } while ((++probe_func_ptr)->func);
 619                 }
 620         }
 621 }
 622
 623 static void action_trace(struct hist_trigger_data *hist_data,
 624                          struct tracing_map_elt *elt,
 625                          struct trace_buffer *buffer, void *rec,
 626                          struct ring_buffer_event *rbe, void *key,
 627                          struct action_data *data, u64 *var_ref_vals)
 628 {
 629         struct synth_event *event = data->synth_event;
 630
 631         trace_synth(event, var_ref_vals, data->var_ref_idx);
 632 }
 633
 634 struct hist_var_data {
 635         struct list_head list;
 636         struct hist_trigger_data *hist_data;
 637 };
 638
 639 static u64 hist_field_timestamp(struct hist_field *hist_field,
 640                                 struct tracing_map_elt *elt,
 641                                 struct trace_buffer *buffer,
 642                                 struct ring_buffer_event *rbe,
 643                                 void *event)
 644 {
 645         struct hist_trigger_data *hist_data = hist_field->hist_data;
 646         struct trace_array *tr = hist_data->event_file->tr;
 647
 648         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 649
 650         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 651                 ts = ns2usecs(ts);
 652
 653         return ts;
 654 }
 655
 656 static u64 hist_field_cpu(struct hist_field *hist_field,
 657                           struct tracing_map_elt *elt,
 658                           struct trace_buffer *buffer,
 659                           struct ring_buffer_event *rbe,
 660                           void *event)
 661 {
 662         int cpu = smp_processor_id();
 663
 664         return cpu;
 665 }
 666
 667 /**
 668  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 669  * @hist_field: The VAR_REF field to check
 670  * @var_data: The hist trigger that owns the variable
 671  * @var_idx: The trigger variable identifier
 672  *
 673  * Check the given VAR_REF field to see whether or not it references
 674  * the given variable associated with the given trigger.
 675  *
 676  * Return: The VAR_REF field if it does reference the variable, NULL if not
 677  */
 678 static struct hist_field *
 679 check_field_for_var_ref(struct hist_field *hist_field,
 680                         struct hist_trigger_data *var_data,
 681                         unsigned int var_idx)
 682 {
 683         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 684
 685         if (hist_field && hist_field->var.idx == var_idx &&
 686             hist_field->var.hist_data == var_data)
 687                 return hist_field;
 688
 689         return NULL;
 690 }
 691
 692 /**
 693  * find_var_ref - Check if a trigger has a reference to a trigger variable
 694  * @hist_data: The hist trigger that might have a reference to the variable
 695  * @var_data: The hist trigger that owns the variable
 696  * @var_idx: The trigger variable identifier
 697  *
 698  * Check the list of var_refs[] on the first hist trigger to see
 699  * whether any of them are references to the variable on the second
 700  * trigger.
 701  *
 702  * Return: The VAR_REF field referencing the variable if so, NULL if not
 703  */
 704 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 705                                        struct hist_trigger_data *var_data,
 706                                        unsigned int var_idx)
 707 {
 708         struct hist_field *hist_field;
 709         unsigned int i;
 710
 711         for (i = 0; i < hist_data->n_var_refs; i++) {
 712                 hist_field = hist_data->var_refs[i];
 713                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 714                         return hist_field;
 715         }
 716
 717         return NULL;
 718 }
 719
 720 /**
 721  * find_any_var_ref - Check if there is a reference to a given trigger variable
 722  * @hist_data: The hist trigger
 723  * @var_idx: The trigger variable identifier
 724  *
 725  * Check to see whether the given variable is currently referenced by
 726  * any other trigger.
 727  *
 728  * The trigger the variable is defined on is explicitly excluded - the
 729  * assumption being that a self-reference doesn't prevent a trigger
 730  * from being removed.
 731  *
 732  * Return: The VAR_REF field referencing the variable if so, NULL if not
 733  */
 734 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 735                                            unsigned int var_idx)
 736 {
 737         struct trace_array *tr = hist_data->event_file->tr;
 738         struct hist_field *found = NULL;
 739         struct hist_var_data *var_data;
 740
 741         list_for_each_entry(var_data, &tr->hist_vars, list) {
 742                 if (var_data->hist_data == hist_data)
 743                         continue;
 744                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 745                 if (found)
 746                         break;
 747         }
 748
 749         return found;
 750 }
 751
 752 /**
 753  * check_var_refs - Check if there is a reference to any of trigger's variables
 754  * @hist_data: The hist trigger
 755  *
 756  * A trigger can define one or more variables.  If any one of them is
 757  * currently referenced by any other trigger, this function will
 758  * determine that.
 759
 760  * Typically used to determine whether or not a trigger can be removed
 761  * - if there are any references to a trigger's variables, it cannot.
 762  *
 763  * Return: True if there is a reference to any of trigger's variables
 764  */
 765 static bool check_var_refs(struct hist_trigger_data *hist_data)
 766 {
 767         struct hist_field *field;
 768         bool found = false;
 769         int i;
 770
 771         for_each_hist_field(i, hist_data) {
 772                 field = hist_data->fields[i];
 773                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 774                         if (find_any_var_ref(hist_data, field->var.idx)) {
 775                                 found = true;
 776                                 break;
 777                         }
 778                 }
 779         }
 780
 781         return found;
 782 }
 783
 784 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 785 {
 786         struct trace_array *tr = hist_data->event_file->tr;
 787         struct hist_var_data *var_data, *found = NULL;
 788
 789         list_for_each_entry(var_data, &tr->hist_vars, list) {
 790                 if (var_data->hist_data == hist_data) {
 791                         found = var_data;
 792                         break;
 793                 }
 794         }
 795
 796         return found;
 797 }
 798
 799 static bool field_has_hist_vars(struct hist_field *hist_field,
 800                                 unsigned int level)
 801 {
 802         int i;
 803
 804         if (level > 3)
 805                 return false;
 806
 807         if (!hist_field)
 808                 return false;
 809
 810         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 811             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 812                 return true;
 813
 814         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 815                 struct hist_field *operand;
 816
 817                 operand = hist_field->operands[i];
 818                 if (field_has_hist_vars(operand, level + 1))
 819                         return true;
 820         }
 821
 822         return false;
 823 }
 824
 825 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 826 {
 827         struct hist_field *hist_field;
 828         int i;
 829
 830         for_each_hist_field(i, hist_data) {
 831                 hist_field = hist_data->fields[i];
 832                 if (field_has_hist_vars(hist_field, 0))
 833                         return true;
 834         }
 835
 836         return false;
 837 }
 838
 839 static int save_hist_vars(struct hist_trigger_data *hist_data)
 840 {
 841         struct trace_array *tr = hist_data->event_file->tr;
 842         struct hist_var_data *var_data;
 843
 844         var_data = find_hist_vars(hist_data);
 845         if (var_data)
 846                 return 0;
 847
 848         if (tracing_check_open_get_tr(tr))
 849                 return -ENODEV;
 850
 851         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 852         if (!var_data) {
 853                 trace_array_put(tr);
 854                 return -ENOMEM;
 855         }
 856
 857         var_data->hist_data = hist_data;
 858         list_add(&var_data->list, &tr->hist_vars);
 859
 860         return 0;
 861 }
 862
 863 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 864 {
 865         struct trace_array *tr = hist_data->event_file->tr;
 866         struct hist_var_data *var_data;
 867
 868         var_data = find_hist_vars(hist_data);
 869         if (!var_data)
 870                 return;
 871
 872         if (WARN_ON(check_var_refs(hist_data)))
 873                 return;
 874
 875         list_del(&var_data->list);
 876
 877         kfree(var_data);
 878
 879         trace_array_put(tr);
 880 }
 881
 882 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 883                                          const char *var_name)
 884 {
 885         struct hist_field *hist_field, *found = NULL;
 886         int i;
 887
 888         for_each_hist_field(i, hist_data) {
 889                 hist_field = hist_data->fields[i];
 890                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 891                     strcmp(hist_field->var.name, var_name) == 0) {
 892                         found = hist_field;
 893                         break;
 894                 }
 895         }
 896
 897         return found;
 898 }
 899
 900 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 901                                    struct trace_event_file *file,
 902                                    const char *var_name)
 903 {
 904         struct hist_trigger_data *test_data;
 905         struct event_trigger_data *test;
 906         struct hist_field *hist_field;
 907
 908         lockdep_assert_held(&event_mutex);
 909
 910         hist_field = find_var_field(hist_data, var_name);
 911         if (hist_field)
 912                 return hist_field;
 913
 914         list_for_each_entry(test, &file->triggers, list) {
 915                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 916                         test_data = test->private_data;
 917                         hist_field = find_var_field(test_data, var_name);
 918                         if (hist_field)
 919                                 return hist_field;
 920                 }
 921         }
 922
 923         return NULL;
 924 }
 925
 926 static struct trace_event_file *find_var_file(struct trace_array *tr,
 927                                               char *system,
 928                                               char *event_name,
 929                                               char *var_name)
 930 {
 931         struct hist_trigger_data *var_hist_data;
 932         struct hist_var_data *var_data;
 933         struct trace_event_file *file, *found = NULL;
 934
 935         if (system)
 936                 return find_event_file(tr, system, event_name);
 937
 938         list_for_each_entry(var_data, &tr->hist_vars, list) {
 939                 var_hist_data = var_data->hist_data;
 940                 file = var_hist_data->event_file;
 941                 if (file == found)
 942                         continue;
 943
 944                 if (find_var_field(var_hist_data, var_name)) {
 945                         if (found) {
 946                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 947                                 return NULL;
 948                         }
 949
 950                         found = file;
 951                 }
 952         }
 953
 954         return found;
 955 }
 956
 957 static struct hist_field *find_file_var(struct trace_event_file *file,
 958                                         const char *var_name)
 959 {
 960         struct hist_trigger_data *test_data;
 961         struct event_trigger_data *test;
 962         struct hist_field *hist_field;
 963
 964         lockdep_assert_held(&event_mutex);
 965
 966         list_for_each_entry(test, &file->triggers, list) {
 967                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 968                         test_data = test->private_data;
 969                         hist_field = find_var_field(test_data, var_name);
 970                         if (hist_field)
 971                                 return hist_field;
 972                 }
 973         }
 974
 975         return NULL;
 976 }
 977
 978 static struct hist_field *
 979 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 980 {
 981         struct trace_array *tr = hist_data->event_file->tr;
 982         struct hist_field *hist_field, *found = NULL;
 983         struct trace_event_file *file;
 984         unsigned int i;
 985
 986         for (i = 0; i < hist_data->n_actions; i++) {
 987                 struct action_data *data = hist_data->actions[i];
 988
 989                 if (data->handler == HANDLER_ONMATCH) {
 990                         char *system = data->match_data.event_system;
 991                         char *event_name = data->match_data.event;
 992
 993                         file = find_var_file(tr, system, event_name, var_name);
 994                         if (!file)
 995                                 continue;
 996                         hist_field = find_file_var(file, var_name);
 997                         if (hist_field) {
 998                                 if (found) {
 999                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1000                                                  errpos(var_name));
1001                                         return ERR_PTR(-EINVAL);
1002                                 }
1003
1004                                 found = hist_field;
1005                         }
1006                 }
1007         }
1008         return found;
1009 }
1010
1011 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1012                                          char *system,
1013                                          char *event_name,
1014                                          char *var_name)
1015 {
1016         struct trace_array *tr = hist_data->event_file->tr;
1017         struct hist_field *hist_field = NULL;
1018         struct trace_event_file *file;
1019
1020         if (!system || !event_name) {
1021                 hist_field = find_match_var(hist_data, var_name);
1022                 if (IS_ERR(hist_field))
1023                         return NULL;
1024                 if (hist_field)
1025                         return hist_field;
1026         }
1027
1028         file = find_var_file(tr, system, event_name, var_name);
1029         if (!file)
1030                 return NULL;
1031
1032         hist_field = find_file_var(file, var_name);
1033
1034         return hist_field;
1035 }
1036
1037 static u64 hist_field_var_ref(struct hist_field *hist_field,
1038                               struct tracing_map_elt *elt,
1039                               struct trace_buffer *buffer,
1040                               struct ring_buffer_event *rbe,
1041                               void *event)
1042 {
1043         struct hist_elt_data *elt_data;
1044         u64 var_val = 0;
1045
1046         if (WARN_ON_ONCE(!elt))
1047                 return var_val;
1048
1049         elt_data = elt->private_data;
1050         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1051
1052         return var_val;
1053 }
1054
1055 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1056                              u64 *var_ref_vals, bool self)
1057 {
1058         struct hist_trigger_data *var_data;
1059         struct tracing_map_elt *var_elt;
1060         struct hist_field *hist_field;
1061         unsigned int i, var_idx;
1062         bool resolved = true;
1063         u64 var_val = 0;
1064
1065         for (i = 0; i < hist_data->n_var_refs; i++) {
1066                 hist_field = hist_data->var_refs[i];
1067                 var_idx = hist_field->var.idx;
1068                 var_data = hist_field->var.hist_data;
1069
1070                 if (var_data == NULL) {
1071                         resolved = false;
1072                         break;
1073                 }
1074
1075                 if ((self && var_data != hist_data) ||
1076                     (!self && var_data == hist_data))
1077                         continue;
1078
1079                 var_elt = tracing_map_lookup(var_data->map, key);
1080                 if (!var_elt) {
1081                         resolved = false;
1082                         break;
1083                 }
1084
1085                 if (!tracing_map_var_set(var_elt, var_idx)) {
1086                         resolved = false;
1087                         break;
1088                 }
1089
1090                 if (self || !hist_field->read_once)
1091                         var_val = tracing_map_read_var(var_elt, var_idx);
1092                 else
1093                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1094
1095                 var_ref_vals[i] = var_val;
1096         }
1097
1098         return resolved;
1099 }
1100
1101 static const char *hist_field_name(struct hist_field *field,
1102                                    unsigned int level)
1103 {
1104         const char *field_name = "";
1105
1106         if (level > 1)
1107                 return field_name;
1108
1109         if (field->field)
1110                 field_name = field->field->name;
1111         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1112                  field->flags & HIST_FIELD_FL_ALIAS)
1113                 field_name = hist_field_name(field->operands[0], ++level);
1114         else if (field->flags & HIST_FIELD_FL_CPU)
1115                 field_name = "common_cpu";
1116         else if (field->flags & HIST_FIELD_FL_EXPR ||
1117                  field->flags & HIST_FIELD_FL_VAR_REF) {
1118                 if (field->system) {
1119                         static char full_name[MAX_FILTER_STR_VAL];
1120
1121                         strcat(full_name, field->system);
1122                         strcat(full_name, ".");
1123                         strcat(full_name, field->event_name);
1124                         strcat(full_name, ".");
1125                         strcat(full_name, field->name);
1126                         field_name = full_name;
1127                 } else
1128                         field_name = field->name;
1129         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1130                 field_name = "common_timestamp";
1131
1132         if (field_name == NULL)
1133                 field_name = "";
1134
1135         return field_name;
1136 }
1137
1138 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1139 {
1140         hist_field_fn_t fn = NULL;
1141
1142         switch (field_size) {
1143         case 8:
1144                 if (field_is_signed)
1145                         fn = hist_field_s64;
1146                 else
1147                         fn = hist_field_u64;
1148                 break;
1149         case 4:
1150                 if (field_is_signed)
1151                         fn = hist_field_s32;
1152                 else
1153                         fn = hist_field_u32;
1154                 break;
1155         case 2:
1156                 if (field_is_signed)
1157                         fn = hist_field_s16;
1158                 else
1159                         fn = hist_field_u16;
1160                 break;
1161         case 1:
1162                 if (field_is_signed)
1163                         fn = hist_field_s8;
1164                 else
1165                         fn = hist_field_u8;
1166                 break;
1167         }
1168
1169         return fn;
1170 }
1171
1172 static int parse_map_size(char *str)
1173 {
1174         unsigned long size, map_bits;
1175         int ret;
1176
1177         ret = kstrtoul(str, 0, &size);
1178         if (ret)
1179                 goto out;
1180
1181         map_bits = ilog2(roundup_pow_of_two(size));
1182         if (map_bits < TRACING_MAP_BITS_MIN ||
1183             map_bits > TRACING_MAP_BITS_MAX)
1184                 ret = -EINVAL;
1185         else
1186                 ret = map_bits;
1187  out:
1188         return ret;
1189 }
1190
1191 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1192 {
1193         unsigned int i;
1194
1195         if (!attrs)
1196                 return;
1197
1198         for (i = 0; i < attrs->n_assignments; i++)
1199                 kfree(attrs->assignment_str[i]);
1200
1201         for (i = 0; i < attrs->n_actions; i++)
1202                 kfree(attrs->action_str[i]);
1203
1204         kfree(attrs->name);
1205         kfree(attrs->sort_key_str);
1206         kfree(attrs->keys_str);
1207         kfree(attrs->vals_str);
1208         kfree(attrs->clock);
1209         kfree(attrs);
1210 }
1211
1212 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1213 {
1214         int ret = -EINVAL;
1215
1216         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1217                 return ret;
1218
1219         if ((str_has_prefix(str, "onmatch(")) ||
1220             (str_has_prefix(str, "onmax(")) ||
1221             (str_has_prefix(str, "onchange("))) {
1222                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1223                 if (!attrs->action_str[attrs->n_actions]) {
1224                         ret = -ENOMEM;
1225                         return ret;
1226                 }
1227                 attrs->n_actions++;
1228                 ret = 0;
1229         }
1230         return ret;
1231 }
1232
1233 static int parse_assignment(struct trace_array *tr,
1234                             char *str, struct hist_trigger_attrs *attrs)
1235 {
1236         int len, ret = 0;
1237
1238         if ((len = str_has_prefix(str, "key=")) ||
1239             (len = str_has_prefix(str, "keys="))) {
1240                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1241                 if (!attrs->keys_str) {
1242                         ret = -ENOMEM;
1243                         goto out;
1244                 }
1245         } else if ((len = str_has_prefix(str, "val=")) ||
1246                    (len = str_has_prefix(str, "vals=")) ||
1247                    (len = str_has_prefix(str, "values="))) {
1248                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1249                 if (!attrs->vals_str) {
1250                         ret = -ENOMEM;
1251                         goto out;
1252                 }
1253         } else if ((len = str_has_prefix(str, "sort="))) {
1254                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1255                 if (!attrs->sort_key_str) {
1256                         ret = -ENOMEM;
1257                         goto out;
1258                 }
1259         } else if (str_has_prefix(str, "name=")) {
1260                 attrs->name = kstrdup(str, GFP_KERNEL);
1261                 if (!attrs->name) {
1262                         ret = -ENOMEM;
1263                         goto out;
1264                 }
1265         } else if ((len = str_has_prefix(str, "clock="))) {
1266                 str += len;
1267
1268                 str = strstrip(str);
1269                 attrs->clock = kstrdup(str, GFP_KERNEL);
1270                 if (!attrs->clock) {
1271                         ret = -ENOMEM;
1272                         goto out;
1273                 }
1274         } else if ((len = str_has_prefix(str, "size="))) {
1275                 int map_bits = parse_map_size(str + len);
1276
1277                 if (map_bits < 0) {
1278                         ret = map_bits;
1279                         goto out;
1280                 }
1281                 attrs->map_bits = map_bits;
1282         } else {
1283                 char *assignment;
1284
1285                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1286                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1287                         ret = -EINVAL;
1288                         goto out;
1289                 }
1290
1291                 assignment = kstrdup(str, GFP_KERNEL);
1292                 if (!assignment) {
1293                         ret = -ENOMEM;
1294                         goto out;
1295                 }
1296
1297                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1298         }
1299  out:
1300         return ret;
1301 }
1302
1303 static struct hist_trigger_attrs *
1304 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1305 {
1306         struct hist_trigger_attrs *attrs;
1307         int ret = 0;
1308
1309         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1310         if (!attrs)
1311                 return ERR_PTR(-ENOMEM);
1312
1313         while (trigger_str) {
1314                 char *str = strsep(&trigger_str, ":");
1315                 char *rhs;
1316
1317                 rhs = strchr(str, '=');
1318                 if (rhs) {
1319                         if (!strlen(++rhs)) {
1320                                 ret = -EINVAL;
1321                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1322                                 goto free;
1323                         }
1324                         ret = parse_assignment(tr, str, attrs);
1325                         if (ret)
1326                                 goto free;
1327                 } else if (strcmp(str, "pause") == 0)
1328                         attrs->pause = true;
1329                 else if ((strcmp(str, "cont") == 0) ||
1330                          (strcmp(str, "continue") == 0))
1331                         attrs->cont = true;
1332                 else if (strcmp(str, "clear") == 0)
1333                         attrs->clear = true;
1334                 else {
1335                         ret = parse_action(str, attrs);
1336                         if (ret)
1337                                 goto free;
1338                 }
1339         }
1340
1341         if (!attrs->keys_str) {
1342                 ret = -EINVAL;
1343                 goto free;
1344         }
1345
1346         if (!attrs->clock) {
1347                 attrs->clock = kstrdup("global", GFP_KERNEL);
1348                 if (!attrs->clock) {
1349                         ret = -ENOMEM;
1350                         goto free;
1351                 }
1352         }
1353
1354         return attrs;
1355  free:
1356         destroy_hist_trigger_attrs(attrs);
1357
1358         return ERR_PTR(ret);
1359 }
1360
1361 static inline void save_comm(char *comm, struct task_struct *task)
1362 {
1363         if (!task->pid) {
1364                 strcpy(comm, "<idle>");
1365                 return;
1366         }
1367
1368         if (WARN_ON_ONCE(task->pid < 0)) {
1369                 strcpy(comm, "<XXX>");
1370                 return;
1371         }
1372
1373         strncpy(comm, task->comm, TASK_COMM_LEN);
1374 }
1375
1376 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1377 {
1378         unsigned int i;
1379
1380         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1381                 kfree(elt_data->field_var_str[i]);
1382
1383         kfree(elt_data->comm);
1384         kfree(elt_data);
1385 }
1386
1387 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1388 {
1389         struct hist_elt_data *elt_data = elt->private_data;
1390
1391         hist_elt_data_free(elt_data);
1392 }
1393
1394 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1395 {
1396         struct hist_trigger_data *hist_data = elt->map->private_data;
1397         unsigned int size = TASK_COMM_LEN;
1398         struct hist_elt_data *elt_data;
1399         struct hist_field *key_field;
1400         unsigned int i, n_str;
1401
1402         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1403         if (!elt_data)
1404                 return -ENOMEM;
1405
1406         for_each_hist_key_field(i, hist_data) {
1407                 key_field = hist_data->fields[i];
1408
1409                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1410                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1411                         if (!elt_data->comm) {
1412                                 kfree(elt_data);
1413                                 return -ENOMEM;
1414                         }
1415                         break;
1416                 }
1417         }
1418
1419         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1420                 hist_data->n_var_str;
1421         if (n_str > SYNTH_FIELDS_MAX) {
1422                 hist_elt_data_free(elt_data);
1423                 return -EINVAL;
1424         }
1425
1426         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1427
1428         size = STR_VAR_LEN_MAX;
1429
1430         for (i = 0; i < n_str; i++) {
1431                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1432                 if (!elt_data->field_var_str[i]) {
1433                         hist_elt_data_free(elt_data);
1434                         return -ENOMEM;
1435                 }
1436         }
1437
1438         elt->private_data = elt_data;
1439
1440         return 0;
1441 }
1442
1443 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1444 {
1445         struct hist_elt_data *elt_data = elt->private_data;
1446
1447         if (elt_data->comm)
1448                 save_comm(elt_data->comm, current);
1449 }
1450
1451 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1452         .elt_alloc      = hist_trigger_elt_data_alloc,
1453         .elt_free       = hist_trigger_elt_data_free,
1454         .elt_init       = hist_trigger_elt_data_init,
1455 };
1456
1457 static const char *get_hist_field_flags(struct hist_field *hist_field)
1458 {
1459         const char *flags_str = NULL;
1460
1461         if (hist_field->flags & HIST_FIELD_FL_HEX)
1462                 flags_str = "hex";
1463         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1464                 flags_str = "sym";
1465         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1466                 flags_str = "sym-offset";
1467         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1468                 flags_str = "execname";
1469         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1470                 flags_str = "syscall";
1471         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1472                 flags_str = "log2";
1473         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1474                 flags_str = "usecs";
1475
1476         return flags_str;
1477 }
1478
1479 static void expr_field_str(struct hist_field *field, char *expr)
1480 {
1481         if (field->flags & HIST_FIELD_FL_VAR_REF)
1482                 strcat(expr, "$");
1483
1484         strcat(expr, hist_field_name(field, 0));
1485
1486         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1487                 const char *flags_str = get_hist_field_flags(field);
1488
1489                 if (flags_str) {
1490                         strcat(expr, ".");
1491                         strcat(expr, flags_str);
1492                 }
1493         }
1494 }
1495
1496 static char *expr_str(struct hist_field *field, unsigned int level)
1497 {
1498         char *expr;
1499
1500         if (level > 1)
1501                 return NULL;
1502
1503         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1504         if (!expr)
1505                 return NULL;
1506
1507         if (!field->operands[0]) {
1508                 expr_field_str(field, expr);
1509                 return expr;
1510         }
1511
1512         if (field->operator == FIELD_OP_UNARY_MINUS) {
1513                 char *subexpr;
1514
1515                 strcat(expr, "-(");
1516                 subexpr = expr_str(field->operands[0], ++level);
1517                 if (!subexpr) {
1518                         kfree(expr);
1519                         return NULL;
1520                 }
1521                 strcat(expr, subexpr);
1522                 strcat(expr, ")");
1523
1524                 kfree(subexpr);
1525
1526                 return expr;
1527         }
1528
1529         expr_field_str(field->operands[0], expr);
1530
1531         switch (field->operator) {
1532         case FIELD_OP_MINUS:
1533                 strcat(expr, "-");
1534                 break;
1535         case FIELD_OP_PLUS:
1536                 strcat(expr, "+");
1537                 break;
1538         default:
1539                 kfree(expr);
1540                 return NULL;
1541         }
1542
1543         expr_field_str(field->operands[1], expr);
1544
1545         return expr;
1546 }
1547
1548 static int contains_operator(char *str)
1549 {
1550         enum field_op_id field_op = FIELD_OP_NONE;
1551         char *op;
1552
1553         op = strpbrk(str, "+-");
1554         if (!op)
1555                 return FIELD_OP_NONE;
1556
1557         switch (*op) {
1558         case '-':
1559                 /*
1560                  * Unfortunately, the modifier ".sym-offset"
1561                  * can confuse things.
1562                  */
1563                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1564                         return FIELD_OP_NONE;
1565
1566                 if (*str == '-')
1567                         field_op = FIELD_OP_UNARY_MINUS;
1568                 else
1569                         field_op = FIELD_OP_MINUS;
1570                 break;
1571         case '+':
1572                 field_op = FIELD_OP_PLUS;
1573                 break;
1574         default:
1575                 break;
1576         }
1577
1578         return field_op;
1579 }
1580
1581 static void get_hist_field(struct hist_field *hist_field)
1582 {
1583         hist_field->ref++;
1584 }
1585
1586 static void __destroy_hist_field(struct hist_field *hist_field)
1587 {
1588         if (--hist_field->ref > 1)
1589                 return;
1590
1591         kfree(hist_field->var.name);
1592         kfree(hist_field->name);
1593         kfree(hist_field->type);
1594
1595         kfree(hist_field->system);
1596         kfree(hist_field->event_name);
1597
1598         kfree(hist_field);
1599 }
1600
1601 static void destroy_hist_field(struct hist_field *hist_field,
1602                                unsigned int level)
1603 {
1604         unsigned int i;
1605
1606         if (level > 3)
1607                 return;
1608
1609         if (!hist_field)
1610                 return;
1611
1612         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1613                 return; /* var refs will be destroyed separately */
1614
1615         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1616                 destroy_hist_field(hist_field->operands[i], level + 1);
1617
1618         __destroy_hist_field(hist_field);
1619 }
1620
1621 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1622                                             struct ftrace_event_field *field,
1623                                             unsigned long flags,
1624                                             char *var_name)
1625 {
1626         struct hist_field *hist_field;
1627
1628         if (field && is_function_field(field))
1629                 return NULL;
1630
1631         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1632         if (!hist_field)
1633                 return NULL;
1634
1635         hist_field->ref = 1;
1636
1637         hist_field->hist_data = hist_data;
1638
1639         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1640                 goto out; /* caller will populate */
1641
1642         if (flags & HIST_FIELD_FL_VAR_REF) {
1643                 hist_field->fn = hist_field_var_ref;
1644                 goto out;
1645         }
1646
1647         if (flags & HIST_FIELD_FL_HITCOUNT) {
1648                 hist_field->fn = hist_field_counter;
1649                 hist_field->size = sizeof(u64);
1650                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1651                 if (!hist_field->type)
1652                         goto free;
1653                 goto out;
1654         }
1655
1656         if (flags & HIST_FIELD_FL_STACKTRACE) {
1657                 hist_field->fn = hist_field_none;
1658                 goto out;
1659         }
1660
1661         if (flags & HIST_FIELD_FL_LOG2) {
1662                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1663                 hist_field->fn = hist_field_log2;
1664                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1665                 hist_field->size = hist_field->operands[0]->size;
1666                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1667                 if (!hist_field->type)
1668                         goto free;
1669                 goto out;
1670         }
1671
1672         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1673                 hist_field->fn = hist_field_timestamp;
1674                 hist_field->size = sizeof(u64);
1675                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1676                 if (!hist_field->type)
1677                         goto free;
1678                 goto out;
1679         }
1680
1681         if (flags & HIST_FIELD_FL_CPU) {
1682                 hist_field->fn = hist_field_cpu;
1683                 hist_field->size = sizeof(int);
1684                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1685                 if (!hist_field->type)
1686                         goto free;
1687                 goto out;
1688         }
1689
1690         if (WARN_ON_ONCE(!field))
1691                 goto out;
1692
1693         /* Pointers to strings are just pointers and dangerous to dereference */
1694         if (is_string_field(field) &&
1695             (field->filter_type != FILTER_PTR_STRING)) {
1696                 flags |= HIST_FIELD_FL_STRING;
1697
1698                 hist_field->size = MAX_FILTER_STR_VAL;
1699                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1700                 if (!hist_field->type)
1701                         goto free;
1702
1703                 if (field->filter_type == FILTER_STATIC_STRING)
1704                         hist_field->fn = hist_field_string;
1705                 else if (field->filter_type == FILTER_DYN_STRING)
1706                         hist_field->fn = hist_field_dynstring;
1707                 else
1708                         hist_field->fn = hist_field_pstring;
1709         } else {
1710                 hist_field->size = field->size;
1711                 hist_field->is_signed = field->is_signed;
1712                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1713                 if (!hist_field->type)
1714                         goto free;
1715
1716                 hist_field->fn = select_value_fn(field->size,
1717                                                  field->is_signed);
1718                 if (!hist_field->fn) {
1719                         destroy_hist_field(hist_field, 0);
1720                         return NULL;
1721                 }
1722         }
1723  out:
1724         hist_field->field = field;
1725         hist_field->flags = flags;
1726
1727         if (var_name) {
1728                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1729                 if (!hist_field->var.name)
1730                         goto free;
1731         }
1732
1733         return hist_field;
1734  free:
1735         destroy_hist_field(hist_field, 0);
1736         return NULL;
1737 }
1738
1739 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1740 {
1741         unsigned int i;
1742
1743         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1744                 if (hist_data->fields[i]) {
1745                         destroy_hist_field(hist_data->fields[i], 0);
1746                         hist_data->fields[i] = NULL;
1747                 }
1748         }
1749
1750         for (i = 0; i < hist_data->n_var_refs; i++) {
1751                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1752                 __destroy_hist_field(hist_data->var_refs[i]);
1753                 hist_data->var_refs[i] = NULL;
1754         }
1755 }
1756
1757 static int init_var_ref(struct hist_field *ref_field,
1758                         struct hist_field *var_field,
1759                         char *system, char *event_name)
1760 {
1761         int err = 0;
1762
1763         ref_field->var.idx = var_field->var.idx;
1764         ref_field->var.hist_data = var_field->hist_data;
1765         ref_field->size = var_field->size;
1766         ref_field->is_signed = var_field->is_signed;
1767         ref_field->flags |= var_field->flags &
1768                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1769
1770         if (system) {
1771                 ref_field->system = kstrdup(system, GFP_KERNEL);
1772                 if (!ref_field->system)
1773                         return -ENOMEM;
1774         }
1775
1776         if (event_name) {
1777                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1778                 if (!ref_field->event_name) {
1779                         err = -ENOMEM;
1780                         goto free;
1781                 }
1782         }
1783
1784         if (var_field->var.name) {
1785                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1786                 if (!ref_field->name) {
1787                         err = -ENOMEM;
1788                         goto free;
1789                 }
1790         } else if (var_field->name) {
1791                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1792                 if (!ref_field->name) {
1793                         err = -ENOMEM;
1794                         goto free;
1795                 }
1796         }
1797
1798         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1799         if (!ref_field->type) {
1800                 err = -ENOMEM;
1801                 goto free;
1802         }
1803  out:
1804         return err;
1805  free:
1806         kfree(ref_field->system);
1807         kfree(ref_field->event_name);
1808         kfree(ref_field->name);
1809
1810         goto out;
1811 }
1812
1813 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1814                             struct hist_field *var_field)
1815 {
1816         struct hist_field *ref_field;
1817         int i;
1818
1819         for (i = 0; i < hist_data->n_var_refs; i++) {
1820                 ref_field = hist_data->var_refs[i];
1821                 if (ref_field->var.idx == var_field->var.idx &&
1822                     ref_field->var.hist_data == var_field->hist_data)
1823                         return i;
1824         }
1825
1826         return -ENOENT;
1827 }
1828
1829 /**
1830  * create_var_ref - Create a variable reference and attach it to trigger
1831  * @hist_data: The trigger that will be referencing the variable
1832  * @var_field: The VAR field to create a reference to
1833  * @system: The optional system string
1834  * @event_name: The optional event_name string
1835  *
1836  * Given a variable hist_field, create a VAR_REF hist_field that
1837  * represents a reference to it.
1838  *
1839  * This function also adds the reference to the trigger that
1840  * now references the variable.
1841  *
1842  * Return: The VAR_REF field if successful, NULL if not
1843  */
1844 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1845                                          struct hist_field *var_field,
1846                                          char *system, char *event_name)
1847 {
1848         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1849         struct hist_field *ref_field;
1850         int i;
1851
1852         /* Check if the variable already exists */
1853         for (i = 0; i < hist_data->n_var_refs; i++) {
1854                 ref_field = hist_data->var_refs[i];
1855                 if (ref_field->var.idx == var_field->var.idx &&
1856                     ref_field->var.hist_data == var_field->hist_data) {
1857                         get_hist_field(ref_field);
1858                         return ref_field;
1859                 }
1860         }
1861
1862         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1863         if (ref_field) {
1864                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1865                         destroy_hist_field(ref_field, 0);
1866                         return NULL;
1867                 }
1868
1869                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1870                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1871         }
1872
1873         return ref_field;
1874 }
1875
1876 static bool is_var_ref(char *var_name)
1877 {
1878         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1879                 return false;
1880
1881         return true;
1882 }
1883
1884 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1885                                  char *var_name)
1886 {
1887         char *name, *field;
1888         unsigned int i;
1889
1890         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1891                 name = hist_data->attrs->var_defs.name[i];
1892
1893                 if (strcmp(var_name, name) == 0) {
1894                         field = hist_data->attrs->var_defs.expr[i];
1895                         if (contains_operator(field) || is_var_ref(field))
1896                                 continue;
1897                         return field;
1898                 }
1899         }
1900
1901         return NULL;
1902 }
1903
1904 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1905                                  char *system, char *event_name,
1906                                  char *var_name)
1907 {
1908         struct trace_event_call *call;
1909
1910         if (system && event_name) {
1911                 call = hist_data->event_file->event_call;
1912
1913                 if (strcmp(system, call->class->system) != 0)
1914                         return NULL;
1915
1916                 if (strcmp(event_name, trace_event_name(call)) != 0)
1917                         return NULL;
1918         }
1919
1920         if (!!system != !!event_name)
1921                 return NULL;
1922
1923         if (!is_var_ref(var_name))
1924                 return NULL;
1925
1926         var_name++;
1927
1928         return field_name_from_var(hist_data, var_name);
1929 }
1930
1931 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1932                                         char *system, char *event_name,
1933                                         char *var_name)
1934 {
1935         struct hist_field *var_field = NULL, *ref_field = NULL;
1936         struct trace_array *tr = hist_data->event_file->tr;
1937
1938         if (!is_var_ref(var_name))
1939                 return NULL;
1940
1941         var_name++;
1942
1943         var_field = find_event_var(hist_data, system, event_name, var_name);
1944         if (var_field)
1945                 ref_field = create_var_ref(hist_data, var_field,
1946                                            system, event_name);
1947
1948         if (!ref_field)
1949                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1950
1951         return ref_field;
1952 }
1953
1954 static struct ftrace_event_field *
1955 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1956             char *field_str, unsigned long *flags)
1957 {
1958         struct ftrace_event_field *field = NULL;
1959         char *field_name, *modifier, *str;
1960         struct trace_array *tr = file->tr;
1961
1962         modifier = str = kstrdup(field_str, GFP_KERNEL);
1963         if (!modifier)
1964                 return ERR_PTR(-ENOMEM);
1965
1966         field_name = strsep(&modifier, ".");
1967         if (modifier) {
1968                 if (strcmp(modifier, "hex") == 0)
1969                         *flags |= HIST_FIELD_FL_HEX;
1970                 else if (strcmp(modifier, "sym") == 0)
1971                         *flags |= HIST_FIELD_FL_SYM;
1972                 else if (strcmp(modifier, "sym-offset") == 0)
1973                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1974                 else if ((strcmp(modifier, "execname") == 0) &&
1975                          (strcmp(field_name, "common_pid") == 0))
1976                         *flags |= HIST_FIELD_FL_EXECNAME;
1977                 else if (strcmp(modifier, "syscall") == 0)
1978                         *flags |= HIST_FIELD_FL_SYSCALL;
1979                 else if (strcmp(modifier, "log2") == 0)
1980                         *flags |= HIST_FIELD_FL_LOG2;
1981                 else if (strcmp(modifier, "usecs") == 0)
1982                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1983                 else {
1984                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1985                         field = ERR_PTR(-EINVAL);
1986                         goto out;
1987                 }
1988         }
1989
1990         if (strcmp(field_name, "common_timestamp") == 0) {
1991                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1992                 hist_data->enable_timestamps = true;
1993                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1994                         hist_data->attrs->ts_in_usecs = true;
1995         } else if (strcmp(field_name, "common_cpu") == 0)
1996                 *flags |= HIST_FIELD_FL_CPU;
1997         else {
1998                 field = trace_find_event_field(file->event_call, field_name);
1999                 if (!field || !field->size) {
2000                         /*
2001                          * For backward compatibility, if field_name
2002                          * was "cpu", then we treat this the same as
2003                          * common_cpu.
2004                          */
2005                         if (strcmp(field_name, "cpu") == 0) {
2006                                 *flags |= HIST_FIELD_FL_CPU;
2007                         } else {
2008                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2009                                          errpos(field_name));
2010                                 field = ERR_PTR(-EINVAL);
2011                                 goto out;
2012                         }
2013                 }
2014         }
2015  out:
2016         kfree(str);
2017
2018         return field;
2019 }
2020
2021 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2022                                        struct hist_field *var_ref,
2023                                        char *var_name)
2024 {
2025         struct hist_field *alias = NULL;
2026         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2027
2028         alias = create_hist_field(hist_data, NULL, flags, var_name);
2029         if (!alias)
2030                 return NULL;
2031
2032         alias->fn = var_ref->fn;
2033         alias->operands[0] = var_ref;
2034
2035         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2036                 destroy_hist_field(alias, 0);
2037                 return NULL;
2038         }
2039
2040         alias->var_ref_idx = var_ref->var_ref_idx;
2041
2042         return alias;
2043 }
2044
2045 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2046                                      struct trace_event_file *file, char *str,
2047                                      unsigned long *flags, char *var_name)
2048 {
2049         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2050         struct ftrace_event_field *field = NULL;
2051         struct hist_field *hist_field = NULL;
2052         int ret = 0;
2053
2054         s = strchr(str, '.');
2055         if (s) {
2056                 s = strchr(++s, '.');
2057                 if (s) {
2058                         ref_system = strsep(&str, ".");
2059                         if (!str) {
2060                                 ret = -EINVAL;
2061                                 goto out;
2062                         }
2063                         ref_event = strsep(&str, ".");
2064                         if (!str) {
2065                                 ret = -EINVAL;
2066                                 goto out;
2067                         }
2068                         ref_var = str;
2069                 }
2070         }
2071
2072         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2073         if (!s) {
2074                 hist_field = parse_var_ref(hist_data, ref_system,
2075                                            ref_event, ref_var);
2076                 if (hist_field) {
2077                         if (var_name) {
2078                                 hist_field = create_alias(hist_data, hist_field, var_name);
2079                                 if (!hist_field) {
2080                                         ret = -ENOMEM;
2081                                         goto out;
2082                                 }
2083                         }
2084                         return hist_field;
2085                 }
2086         } else
2087                 str = s;
2088
2089         field = parse_field(hist_data, file, str, flags);
2090         if (IS_ERR(field)) {
2091                 ret = PTR_ERR(field);
2092                 goto out;
2093         }
2094
2095         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2096         if (!hist_field) {
2097                 ret = -ENOMEM;
2098                 goto out;
2099         }
2100
2101         return hist_field;
2102  out:
2103         return ERR_PTR(ret);
2104 }
2105
2106 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2107                                      struct trace_event_file *file,
2108                                      char *str, unsigned long flags,
2109                                      char *var_name, unsigned int level);
2110
2111 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2112                                       struct trace_event_file *file,
2113                                       char *str, unsigned long flags,
2114                                       char *var_name, unsigned int level)
2115 {
2116         struct hist_field *operand1, *expr = NULL;
2117         unsigned long operand_flags;
2118         int ret = 0;
2119         char *s;
2120
2121         /* we support only -(xxx) i.e. explicit parens required */
2122
2123         if (level > 3) {
2124                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2125                 ret = -EINVAL;
2126                 goto free;
2127         }
2128
2129         str++; /* skip leading '-' */
2130
2131         s = strchr(str, '(');
2132         if (s)
2133                 str++;
2134         else {
2135                 ret = -EINVAL;
2136                 goto free;
2137         }
2138
2139         s = strrchr(str, ')');
2140         if (s)
2141                 *s = '\0';
2142         else {
2143                 ret = -EINVAL; /* no closing ')' */
2144                 goto free;
2145         }
2146
2147         flags |= HIST_FIELD_FL_EXPR;
2148         expr = create_hist_field(hist_data, NULL, flags, var_name);
2149         if (!expr) {
2150                 ret = -ENOMEM;
2151                 goto free;
2152         }
2153
2154         operand_flags = 0;
2155         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2156         if (IS_ERR(operand1)) {
2157                 ret = PTR_ERR(operand1);
2158                 goto free;
2159         }
2160         if (operand1->flags & HIST_FIELD_FL_STRING) {
2161                 /* String type can not be the operand of unary operator. */
2162                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2163                 destroy_hist_field(operand1, 0);
2164                 ret = -EINVAL;
2165                 goto free;
2166         }
2167
2168         expr->flags |= operand1->flags &
2169                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2170         expr->fn = hist_field_unary_minus;
2171         expr->operands[0] = operand1;
2172         expr->operator = FIELD_OP_UNARY_MINUS;
2173         expr->name = expr_str(expr, 0);
2174         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2175         if (!expr->type) {
2176                 ret = -ENOMEM;
2177                 goto free;
2178         }
2179
2180         return expr;
2181  free:
2182         destroy_hist_field(expr, 0);
2183         return ERR_PTR(ret);
2184 }
2185
2186 static int check_expr_operands(struct trace_array *tr,
2187                                struct hist_field *operand1,
2188                                struct hist_field *operand2)
2189 {
2190         unsigned long operand1_flags = operand1->flags;
2191         unsigned long operand2_flags = operand2->flags;
2192
2193         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2194             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2195                 struct hist_field *var;
2196
2197                 var = find_var_field(operand1->var.hist_data, operand1->name);
2198                 if (!var)
2199                         return -EINVAL;
2200                 operand1_flags = var->flags;
2201         }
2202
2203         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2204             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2205                 struct hist_field *var;
2206
2207                 var = find_var_field(operand2->var.hist_data, operand2->name);
2208                 if (!var)
2209                         return -EINVAL;
2210                 operand2_flags = var->flags;
2211         }
2212
2213         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2214             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2215                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2216                 return -EINVAL;
2217         }
2218
2219         return 0;
2220 }
2221
2222 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2223                                      struct trace_event_file *file,
2224                                      char *str, unsigned long flags,
2225                                      char *var_name, unsigned int level)
2226 {
2227         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2228         unsigned long operand_flags;
2229         int field_op, ret = -EINVAL;
2230         char *sep, *operand1_str;
2231
2232         if (level > 3) {
2233                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2234                 return ERR_PTR(-EINVAL);
2235         }
2236
2237         field_op = contains_operator(str);
2238
2239         if (field_op == FIELD_OP_NONE)
2240                 return parse_atom(hist_data, file, str, &flags, var_name);
2241
2242         if (field_op == FIELD_OP_UNARY_MINUS)
2243                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2244
2245         switch (field_op) {
2246         case FIELD_OP_MINUS:
2247                 sep = "-";
2248                 break;
2249         case FIELD_OP_PLUS:
2250                 sep = "+";
2251                 break;
2252         default:
2253                 goto free;
2254         }
2255
2256         operand1_str = strsep(&str, sep);
2257         if (!operand1_str || !str)
2258                 goto free;
2259
2260         operand_flags = 0;
2261         operand1 = parse_atom(hist_data, file, operand1_str,
2262                               &operand_flags, NULL);
2263         if (IS_ERR(operand1)) {
2264                 ret = PTR_ERR(operand1);
2265                 operand1 = NULL;
2266                 goto free;
2267         }
2268         if (operand1->flags & HIST_FIELD_FL_STRING) {
2269                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2270                 ret = -EINVAL;
2271                 goto free;
2272         }
2273
2274         /* rest of string could be another expression e.g. b+c in a+b+c */
2275         operand_flags = 0;
2276         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2277         if (IS_ERR(operand2)) {
2278                 ret = PTR_ERR(operand2);
2279                 operand2 = NULL;
2280                 goto free;
2281         }
2282         if (operand2->flags & HIST_FIELD_FL_STRING) {
2283                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2284                 ret = -EINVAL;
2285                 goto free;
2286         }
2287
2288         ret = check_expr_operands(file->tr, operand1, operand2);
2289         if (ret)
2290                 goto free;
2291
2292         flags |= HIST_FIELD_FL_EXPR;
2293
2294         flags |= operand1->flags &
2295                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2296
2297         expr = create_hist_field(hist_data, NULL, flags, var_name);
2298         if (!expr) {
2299                 ret = -ENOMEM;
2300                 goto free;
2301         }
2302
2303         operand1->read_once = true;
2304         operand2->read_once = true;
2305
2306         expr->operands[0] = operand1;
2307         expr->operands[1] = operand2;
2308
2309         /* The operand sizes should be the same, so just pick one */
2310         expr->size = operand1->size;
2311
2312         expr->operator = field_op;
2313         expr->name = expr_str(expr, 0);
2314         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2315         if (!expr->type) {
2316                 ret = -ENOMEM;
2317                 goto free;
2318         }
2319
2320         switch (field_op) {
2321         case FIELD_OP_MINUS:
2322                 expr->fn = hist_field_minus;
2323                 break;
2324         case FIELD_OP_PLUS:
2325                 expr->fn = hist_field_plus;
2326                 break;
2327         default:
2328                 ret = -EINVAL;
2329                 goto free;
2330         }
2331
2332         return expr;
2333  free:
2334         destroy_hist_field(operand1, 0);
2335         destroy_hist_field(operand2, 0);
2336         destroy_hist_field(expr, 0);
2337
2338         return ERR_PTR(ret);
2339 }
2340
2341 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2342                                  struct trace_event_file *file)
2343 {
2344         struct event_trigger_data *test;
2345
2346         lockdep_assert_held(&event_mutex);
2347
2348         list_for_each_entry(test, &file->triggers, list) {
2349                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2350                         if (test->private_data == hist_data)
2351                                 return test->filter_str;
2352                 }
2353         }
2354
2355         return NULL;
2356 }
2357
2358 static struct event_command trigger_hist_cmd;
2359 static int event_hist_trigger_func(struct event_command *cmd_ops,
2360                                    struct trace_event_file *file,
2361                                    char *glob, char *cmd, char *param);
2362
2363 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2364                             struct hist_trigger_data *hist_data,
2365                             unsigned int n_keys)
2366 {
2367         struct hist_field *target_hist_field, *hist_field;
2368         unsigned int n, i, j;
2369
2370         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2371                 return false;
2372
2373         i = hist_data->n_vals;
2374         j = target_hist_data->n_vals;
2375
2376         for (n = 0; n < n_keys; n++) {
2377                 hist_field = hist_data->fields[i + n];
2378                 target_hist_field = target_hist_data->fields[j + n];
2379
2380                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2381                         return false;
2382                 if (hist_field->size != target_hist_field->size)
2383                         return false;
2384                 if (hist_field->is_signed != target_hist_field->is_signed)
2385                         return false;
2386         }
2387
2388         return true;
2389 }
2390
2391 static struct hist_trigger_data *
2392 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2393                      struct trace_event_file *file)
2394 {
2395         struct hist_trigger_data *hist_data;
2396         struct event_trigger_data *test;
2397         unsigned int n_keys;
2398
2399         lockdep_assert_held(&event_mutex);
2400
2401         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2402
2403         list_for_each_entry(test, &file->triggers, list) {
2404                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2405                         hist_data = test->private_data;
2406
2407                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2408                                 return hist_data;
2409                 }
2410         }
2411
2412         return NULL;
2413 }
2414
2415 static struct trace_event_file *event_file(struct trace_array *tr,
2416                                            char *system, char *event_name)
2417 {
2418         struct trace_event_file *file;
2419
2420         file = __find_event_file(tr, system, event_name);
2421         if (!file)
2422                 return ERR_PTR(-EINVAL);
2423
2424         return file;
2425 }
2426
2427 static struct hist_field *
2428 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2429                          char *system, char *event_name, char *field_name)
2430 {
2431         struct hist_field *event_var;
2432         char *synthetic_name;
2433
2434         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2435         if (!synthetic_name)
2436                 return ERR_PTR(-ENOMEM);
2437
2438         strcpy(synthetic_name, "synthetic_");
2439         strcat(synthetic_name, field_name);
2440
2441         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2442
2443         kfree(synthetic_name);
2444
2445         return event_var;
2446 }
2447
2448 /**
2449  * create_field_var_hist - Automatically create a histogram and var for a field
2450  * @target_hist_data: The target hist trigger
2451  * @subsys_name: Optional subsystem name
2452  * @event_name: Optional event name
2453  * @field_name: The name of the field (and the resulting variable)
2454  *
2455  * Hist trigger actions fetch data from variables, not directly from
2456  * events.  However, for convenience, users are allowed to directly
2457  * specify an event field in an action, which will be automatically
2458  * converted into a variable on their behalf.
2459
2460  * If a user specifies a field on an event that isn't the event the
2461  * histogram currently being defined (the target event histogram), the
2462  * only way that can be accomplished is if a new hist trigger is
2463  * created and the field variable defined on that.
2464  *
2465  * This function creates a new histogram compatible with the target
2466  * event (meaning a histogram with the same key as the target
2467  * histogram), and creates a variable for the specified field, but
2468  * with 'synthetic_' prepended to the variable name in order to avoid
2469  * collision with normal field variables.
2470  *
2471  * Return: The variable created for the field.
2472  */
2473 static struct hist_field *
2474 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2475                       char *subsys_name, char *event_name, char *field_name)
2476 {
2477         struct trace_array *tr = target_hist_data->event_file->tr;
2478         struct hist_trigger_data *hist_data;
2479         unsigned int i, n, first = true;
2480         struct field_var_hist *var_hist;
2481         struct trace_event_file *file;
2482         struct hist_field *key_field;
2483         struct hist_field *event_var;
2484         char *saved_filter;
2485         char *cmd;
2486         int ret;
2487
2488         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2489                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2490                 return ERR_PTR(-EINVAL);
2491         }
2492
2493         file = event_file(tr, subsys_name, event_name);
2494
2495         if (IS_ERR(file)) {
2496                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2497                 ret = PTR_ERR(file);
2498                 return ERR_PTR(ret);
2499         }
2500
2501         /*
2502          * Look for a histogram compatible with target.  We'll use the
2503          * found histogram specification to create a new matching
2504          * histogram with our variable on it.  target_hist_data is not
2505          * yet a registered histogram so we can't use that.
2506          */
2507         hist_data = find_compatible_hist(target_hist_data, file);
2508         if (!hist_data) {
2509                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2510                 return ERR_PTR(-EINVAL);
2511         }
2512
2513         /* See if a synthetic field variable has already been created */
2514         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2515                                              event_name, field_name);
2516         if (!IS_ERR_OR_NULL(event_var))
2517                 return event_var;
2518
2519         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2520         if (!var_hist)
2521                 return ERR_PTR(-ENOMEM);
2522
2523         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2524         if (!cmd) {
2525                 kfree(var_hist);
2526                 return ERR_PTR(-ENOMEM);
2527         }
2528
2529         /* Use the same keys as the compatible histogram */
2530         strcat(cmd, "keys=");
2531
2532         for_each_hist_key_field(i, hist_data) {
2533                 key_field = hist_data->fields[i];
2534                 if (!first)
2535                         strcat(cmd, ",");
2536                 strcat(cmd, key_field->field->name);
2537                 first = false;
2538         }
2539
2540         /* Create the synthetic field variable specification */
2541         strcat(cmd, ":synthetic_");
2542         strcat(cmd, field_name);
2543         strcat(cmd, "=");
2544         strcat(cmd, field_name);
2545
2546         /* Use the same filter as the compatible histogram */
2547         saved_filter = find_trigger_filter(hist_data, file);
2548         if (saved_filter) {
2549                 strcat(cmd, " if ");
2550                 strcat(cmd, saved_filter);
2551         }
2552
2553         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2554         if (!var_hist->cmd) {
2555                 kfree(cmd);
2556                 kfree(var_hist);
2557                 return ERR_PTR(-ENOMEM);
2558         }
2559
2560         /* Save the compatible histogram information */
2561         var_hist->hist_data = hist_data;
2562
2563         /* Create the new histogram with our variable */
2564         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2565                                       "", "hist", cmd);
2566         if (ret) {
2567                 kfree(cmd);
2568                 kfree(var_hist->cmd);
2569                 kfree(var_hist);
2570                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2571                 return ERR_PTR(ret);
2572         }
2573
2574         kfree(cmd);
2575
2576         /* If we can't find the variable, something went wrong */
2577         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2578                                              event_name, field_name);
2579         if (IS_ERR_OR_NULL(event_var)) {
2580                 kfree(var_hist->cmd);
2581                 kfree(var_hist);
2582                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2583                 return ERR_PTR(-EINVAL);
2584         }
2585
2586         n = target_hist_data->n_field_var_hists;
2587         target_hist_data->field_var_hists[n] = var_hist;
2588         target_hist_data->n_field_var_hists++;
2589
2590         return event_var;
2591 }
2592
2593 static struct hist_field *
2594 find_target_event_var(struct hist_trigger_data *hist_data,
2595                       char *subsys_name, char *event_name, char *var_name)
2596 {
2597         struct trace_event_file *file = hist_data->event_file;
2598         struct hist_field *hist_field = NULL;
2599
2600         if (subsys_name) {
2601                 struct trace_event_call *call;
2602
2603                 if (!event_name)
2604                         return NULL;
2605
2606                 call = file->event_call;
2607
2608                 if (strcmp(subsys_name, call->class->system) != 0)
2609                         return NULL;
2610
2611                 if (strcmp(event_name, trace_event_name(call)) != 0)
2612                         return NULL;
2613         }
2614
2615         hist_field = find_var_field(hist_data, var_name);
2616
2617         return hist_field;
2618 }
2619
2620 static inline void __update_field_vars(struct tracing_map_elt *elt,
2621                                        struct trace_buffer *buffer,
2622                                        struct ring_buffer_event *rbe,
2623                                        void *rec,
2624                                        struct field_var **field_vars,
2625                                        unsigned int n_field_vars,
2626                                        unsigned int field_var_str_start)
2627 {
2628         struct hist_elt_data *elt_data = elt->private_data;
2629         unsigned int i, j, var_idx;
2630         u64 var_val;
2631
2632         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2633                 struct field_var *field_var = field_vars[i];
2634                 struct hist_field *var = field_var->var;
2635                 struct hist_field *val = field_var->val;
2636
2637                 var_val = val->fn(val, elt, buffer, rbe, rec);
2638                 var_idx = var->var.idx;
2639
2640                 if (val->flags & HIST_FIELD_FL_STRING) {
2641                         char *str = elt_data->field_var_str[j++];
2642                         char *val_str = (char *)(uintptr_t)var_val;
2643
2644                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2645                         var_val = (u64)(uintptr_t)str;
2646                 }
2647                 tracing_map_set_var(elt, var_idx, var_val);
2648         }
2649 }
2650
2651 static void update_field_vars(struct hist_trigger_data *hist_data,
2652                               struct tracing_map_elt *elt,
2653                               struct trace_buffer *buffer,
2654                               struct ring_buffer_event *rbe,
2655                               void *rec)
2656 {
2657         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2658                             hist_data->n_field_vars, 0);
2659 }
2660
2661 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2662                                  struct tracing_map_elt *elt,
2663                                  struct trace_buffer *buffer,  void *rec,
2664                                  struct ring_buffer_event *rbe, void *key,
2665                                  struct action_data *data, u64 *var_ref_vals)
2666 {
2667         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2668                             hist_data->n_save_vars, hist_data->n_field_var_str);
2669 }
2670
2671 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2672                                      struct trace_event_file *file,
2673                                      char *name, int size, const char *type)
2674 {
2675         struct hist_field *var;
2676         int idx;
2677
2678         if (find_var(hist_data, file, name) && !hist_data->remove) {
2679                 var = ERR_PTR(-EINVAL);
2680                 goto out;
2681         }
2682
2683         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2684         if (!var) {
2685                 var = ERR_PTR(-ENOMEM);
2686                 goto out;
2687         }
2688
2689         idx = tracing_map_add_var(hist_data->map);
2690         if (idx < 0) {
2691                 kfree(var);
2692                 var = ERR_PTR(-EINVAL);
2693                 goto out;
2694         }
2695
2696         var->ref = 1;
2697         var->flags = HIST_FIELD_FL_VAR;
2698         var->var.idx = idx;
2699         var->var.hist_data = var->hist_data = hist_data;
2700         var->size = size;
2701         var->var.name = kstrdup(name, GFP_KERNEL);
2702         var->type = kstrdup(type, GFP_KERNEL);
2703         if (!var->var.name || !var->type) {
2704                 kfree(var->var.name);
2705                 kfree(var->type);
2706                 kfree(var);
2707                 var = ERR_PTR(-ENOMEM);
2708         }
2709  out:
2710         return var;
2711 }
2712
2713 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2714                                           struct trace_event_file *file,
2715                                           char *field_name)
2716 {
2717         struct hist_field *val = NULL, *var = NULL;
2718         unsigned long flags = HIST_FIELD_FL_VAR;
2719         struct trace_array *tr = file->tr;
2720         struct field_var *field_var;
2721         int ret = 0;
2722
2723         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2724                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2725                 ret = -EINVAL;
2726                 goto err;
2727         }
2728
2729         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2730         if (IS_ERR(val)) {
2731                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2732                 ret = PTR_ERR(val);
2733                 goto err;
2734         }
2735
2736         var = create_var(hist_data, file, field_name, val->size, val->type);
2737         if (IS_ERR(var)) {
2738                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2739                 kfree(val);
2740                 ret = PTR_ERR(var);
2741                 goto err;
2742         }
2743
2744         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2745         if (!field_var) {
2746                 kfree(val);
2747                 kfree(var);
2748                 ret =  -ENOMEM;
2749                 goto err;
2750         }
2751
2752         field_var->var = var;
2753         field_var->val = val;
2754  out:
2755         return field_var;
2756  err:
2757         field_var = ERR_PTR(ret);
2758         goto out;
2759 }
2760
2761 /**
2762  * create_target_field_var - Automatically create a variable for a field
2763  * @target_hist_data: The target hist trigger
2764  * @subsys_name: Optional subsystem name
2765  * @event_name: Optional event name
2766  * @var_name: The name of the field (and the resulting variable)
2767  *
2768  * Hist trigger actions fetch data from variables, not directly from
2769  * events.  However, for convenience, users are allowed to directly
2770  * specify an event field in an action, which will be automatically
2771  * converted into a variable on their behalf.
2772
2773  * This function creates a field variable with the name var_name on
2774  * the hist trigger currently being defined on the target event.  If
2775  * subsys_name and event_name are specified, this function simply
2776  * verifies that they do in fact match the target event subsystem and
2777  * event name.
2778  *
2779  * Return: The variable created for the field.
2780  */
2781 static struct field_var *
2782 create_target_field_var(struct hist_trigger_data *target_hist_data,
2783                         char *subsys_name, char *event_name, char *var_name)
2784 {
2785         struct trace_event_file *file = target_hist_data->event_file;
2786
2787         if (subsys_name) {
2788                 struct trace_event_call *call;
2789
2790                 if (!event_name)
2791                         return NULL;
2792
2793                 call = file->event_call;
2794
2795                 if (strcmp(subsys_name, call->class->system) != 0)
2796                         return NULL;
2797
2798                 if (strcmp(event_name, trace_event_name(call)) != 0)
2799                         return NULL;
2800         }
2801
2802         return create_field_var(target_hist_data, file, var_name);
2803 }
2804
2805 static bool check_track_val_max(u64 track_val, u64 var_val)
2806 {
2807         if (var_val <= track_val)
2808                 return false;
2809
2810         return true;
2811 }
2812
2813 static bool check_track_val_changed(u64 track_val, u64 var_val)
2814 {
2815         if (var_val == track_val)
2816                 return false;
2817
2818         return true;
2819 }
2820
2821 static u64 get_track_val(struct hist_trigger_data *hist_data,
2822                          struct tracing_map_elt *elt,
2823                          struct action_data *data)
2824 {
2825         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2826         u64 track_val;
2827
2828         track_val = tracing_map_read_var(elt, track_var_idx);
2829
2830         return track_val;
2831 }
2832
2833 static void save_track_val(struct hist_trigger_data *hist_data,
2834                            struct tracing_map_elt *elt,
2835                            struct action_data *data, u64 var_val)
2836 {
2837         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2838
2839         tracing_map_set_var(elt, track_var_idx, var_val);
2840 }
2841
2842 static void save_track_data(struct hist_trigger_data *hist_data,
2843                             struct tracing_map_elt *elt,
2844                             struct trace_buffer *buffer, void *rec,
2845                             struct ring_buffer_event *rbe, void *key,
2846                             struct action_data *data, u64 *var_ref_vals)
2847 {
2848         if (data->track_data.save_data)
2849                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2850                                            key, data, var_ref_vals);
2851 }
2852
2853 static bool check_track_val(struct tracing_map_elt *elt,
2854                             struct action_data *data,
2855                             u64 var_val)
2856 {
2857         struct hist_trigger_data *hist_data;
2858         u64 track_val;
2859
2860         hist_data = data->track_data.track_var->hist_data;
2861         track_val = get_track_val(hist_data, elt, data);
2862
2863         return data->track_data.check_val(track_val, var_val);
2864 }
2865
2866 #ifdef CONFIG_TRACER_SNAPSHOT
2867 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2868 {
2869         /* called with tr->max_lock held */
2870         struct track_data *track_data = tr->cond_snapshot->cond_data;
2871         struct hist_elt_data *elt_data, *track_elt_data;
2872         struct snapshot_context *context = cond_data;
2873         struct action_data *action;
2874         u64 track_val;
2875
2876         if (!track_data)
2877                 return false;
2878
2879         action = track_data->action_data;
2880
2881         track_val = get_track_val(track_data->hist_data, context->elt,
2882                                   track_data->action_data);
2883
2884         if (!action->track_data.check_val(track_data->track_val, track_val))
2885                 return false;
2886
2887         track_data->track_val = track_val;
2888         memcpy(track_data->key, context->key, track_data->key_len);
2889
2890         elt_data = context->elt->private_data;
2891         track_elt_data = track_data->elt.private_data;
2892         if (elt_data->comm)
2893                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2894
2895         track_data->updated = true;
2896
2897         return true;
2898 }
2899
2900 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2901                                      struct tracing_map_elt *elt,
2902                                      struct trace_buffer *buffer, void *rec,
2903                                      struct ring_buffer_event *rbe, void *key,
2904                                      struct action_data *data,
2905                                      u64 *var_ref_vals)
2906 {
2907         struct trace_event_file *file = hist_data->event_file;
2908         struct snapshot_context context;
2909
2910         context.elt = elt;
2911         context.key = key;
2912
2913         tracing_snapshot_cond(file->tr, &context);
2914 }
2915
2916 static void hist_trigger_print_key(struct seq_file *m,
2917                                    struct hist_trigger_data *hist_data,
2918                                    void *key,
2919                                    struct tracing_map_elt *elt);
2920
2921 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2922 {
2923         unsigned int i;
2924
2925         if (!hist_data->n_actions)
2926                 return NULL;
2927
2928         for (i = 0; i < hist_data->n_actions; i++) {
2929                 struct action_data *data = hist_data->actions[i];
2930
2931                 if (data->action == ACTION_SNAPSHOT)
2932                         return data;
2933         }
2934
2935         return NULL;
2936 }
2937
2938 static void track_data_snapshot_print(struct seq_file *m,
2939                                       struct hist_trigger_data *hist_data)
2940 {
2941         struct trace_event_file *file = hist_data->event_file;
2942         struct track_data *track_data;
2943         struct action_data *action;
2944
2945         track_data = tracing_cond_snapshot_data(file->tr);
2946         if (!track_data)
2947                 return;
2948
2949         if (!track_data->updated)
2950                 return;
2951
2952         action = snapshot_action(hist_data);
2953         if (!action)
2954                 return;
2955
2956         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2957         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2958                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2959                    action->track_data.var_str, track_data->track_val);
2960
2961         seq_puts(m, "\ttriggered by event with key: ");
2962         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2963         seq_putc(m, '\n');
2964 }
2965 #else
2966 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2967 {
2968         return false;
2969 }
2970 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2971                                      struct tracing_map_elt *elt,
2972                                      struct trace_buffer *buffer, void *rec,
2973                                      struct ring_buffer_event *rbe, void *key,
2974                                      struct action_data *data,
2975                                      u64 *var_ref_vals) {}
2976 static void track_data_snapshot_print(struct seq_file *m,
2977                                       struct hist_trigger_data *hist_data) {}
2978 #endif /* CONFIG_TRACER_SNAPSHOT */
2979
2980 static void track_data_print(struct seq_file *m,
2981                              struct hist_trigger_data *hist_data,
2982                              struct tracing_map_elt *elt,
2983                              struct action_data *data)
2984 {
2985         u64 track_val = get_track_val(hist_data, elt, data);
2986         unsigned int i, save_var_idx;
2987
2988         if (data->handler == HANDLER_ONMAX)
2989                 seq_printf(m, "\n\tmax: %10llu", track_val);
2990         else if (data->handler == HANDLER_ONCHANGE)
2991                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2992
2993         if (data->action == ACTION_SNAPSHOT)
2994                 return;
2995
2996         for (i = 0; i < hist_data->n_save_vars; i++) {
2997                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2998                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2999                 u64 val;
3000
3001                 save_var_idx = save_var->var.idx;
3002
3003                 val = tracing_map_read_var(elt, save_var_idx);
3004
3005                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3006                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3007                                    (char *)(uintptr_t)(val));
3008                 } else
3009                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3010         }
3011 }
3012
3013 static void ontrack_action(struct hist_trigger_data *hist_data,
3014                            struct tracing_map_elt *elt,
3015                            struct trace_buffer *buffer, void *rec,
3016                            struct ring_buffer_event *rbe, void *key,
3017                            struct action_data *data, u64 *var_ref_vals)
3018 {
3019         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3020
3021         if (check_track_val(elt, data, var_val)) {
3022                 save_track_val(hist_data, elt, data, var_val);
3023                 save_track_data(hist_data, elt, buffer, rec, rbe,
3024                                 key, data, var_ref_vals);
3025         }
3026 }
3027
3028 static void action_data_destroy(struct action_data *data)
3029 {
3030         unsigned int i;
3031
3032         lockdep_assert_held(&event_mutex);
3033
3034         kfree(data->action_name);
3035
3036         for (i = 0; i < data->n_params; i++)
3037                 kfree(data->params[i]);
3038
3039         if (data->synth_event)
3040                 data->synth_event->ref--;
3041
3042         kfree(data->synth_event_name);
3043
3044         kfree(data);
3045 }
3046
3047 static void track_data_destroy(struct hist_trigger_data *hist_data,
3048                                struct action_data *data)
3049 {
3050         struct trace_event_file *file = hist_data->event_file;
3051
3052         destroy_hist_field(data->track_data.track_var, 0);
3053
3054         if (data->action == ACTION_SNAPSHOT) {
3055                 struct track_data *track_data;
3056
3057                 track_data = tracing_cond_snapshot_data(file->tr);
3058                 if (track_data && track_data->hist_data == hist_data) {
3059                         tracing_snapshot_cond_disable(file->tr);
3060                         track_data_free(track_data);
3061                 }
3062         }
3063
3064         kfree(data->track_data.var_str);
3065
3066         action_data_destroy(data);
3067 }
3068
3069 static int action_create(struct hist_trigger_data *hist_data,
3070                          struct action_data *data);
3071
3072 static int track_data_create(struct hist_trigger_data *hist_data,
3073                              struct action_data *data)
3074 {
3075         struct hist_field *var_field, *ref_field, *track_var = NULL;
3076         struct trace_event_file *file = hist_data->event_file;
3077         struct trace_array *tr = file->tr;
3078         char *track_data_var_str;
3079         int ret = 0;
3080
3081         track_data_var_str = data->track_data.var_str;
3082         if (track_data_var_str[0] != '$') {
3083                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3084                 return -EINVAL;
3085         }
3086         track_data_var_str++;
3087
3088         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3089         if (!var_field) {
3090                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3091                 return -EINVAL;
3092         }
3093
3094         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3095         if (!ref_field)
3096                 return -ENOMEM;
3097
3098         data->track_data.var_ref = ref_field;
3099
3100         if (data->handler == HANDLER_ONMAX)
3101                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3102         if (IS_ERR(track_var)) {
3103                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3104                 ret = PTR_ERR(track_var);
3105                 goto out;
3106         }
3107
3108         if (data->handler == HANDLER_ONCHANGE)
3109                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3110         if (IS_ERR(track_var)) {
3111                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3112                 ret = PTR_ERR(track_var);
3113                 goto out;
3114         }
3115         data->track_data.track_var = track_var;
3116
3117         ret = action_create(hist_data, data);
3118  out:
3119         return ret;
3120 }
3121
3122 static int parse_action_params(struct trace_array *tr, char *params,
3123                                struct action_data *data)
3124 {
3125         char *param, *saved_param;
3126         bool first_param = true;
3127         int ret = 0;
3128
3129         while (params) {
3130                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3131                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3132                         goto out;
3133                 }
3134
3135                 param = strsep(&params, ",");
3136                 if (!param) {
3137                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3138                         ret = -EINVAL;
3139                         goto out;
3140                 }
3141
3142                 param = strstrip(param);
3143                 if (strlen(param) < 2) {
3144                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3145                         ret = -EINVAL;
3146                         goto out;
3147                 }
3148
3149                 saved_param = kstrdup(param, GFP_KERNEL);
3150                 if (!saved_param) {
3151                         ret = -ENOMEM;
3152                         goto out;
3153                 }
3154
3155                 if (first_param && data->use_trace_keyword) {
3156                         data->synth_event_name = saved_param;
3157                         first_param = false;
3158                         continue;
3159                 }
3160                 first_param = false;
3161
3162                 data->params[data->n_params++] = saved_param;
3163         }
3164  out:
3165         return ret;
3166 }
3167
3168 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3169                         enum handler_id handler)
3170 {
3171         char *action_name;
3172         int ret = 0;
3173
3174         strsep(&str, ".");
3175         if (!str) {
3176                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3177                 ret = -EINVAL;
3178                 goto out;
3179         }
3180
3181         action_name = strsep(&str, "(");
3182         if (!action_name || !str) {
3183                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3184                 ret = -EINVAL;
3185                 goto out;
3186         }
3187
3188         if (str_has_prefix(action_name, "save")) {
3189                 char *params = strsep(&str, ")");
3190
3191                 if (!params) {
3192                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3193                         ret = -EINVAL;
3194                         goto out;
3195                 }
3196
3197                 ret = parse_action_params(tr, params, data);
3198                 if (ret)
3199                         goto out;
3200
3201                 if (handler == HANDLER_ONMAX)
3202                         data->track_data.check_val = check_track_val_max;
3203                 else if (handler == HANDLER_ONCHANGE)
3204                         data->track_data.check_val = check_track_val_changed;
3205                 else {
3206                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3207                         ret = -EINVAL;
3208                         goto out;
3209                 }
3210
3211                 data->track_data.save_data = save_track_data_vars;
3212                 data->fn = ontrack_action;
3213                 data->action = ACTION_SAVE;
3214         } else if (str_has_prefix(action_name, "snapshot")) {
3215                 char *params = strsep(&str, ")");
3216
3217                 if (!str) {
3218                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3219                         ret = -EINVAL;
3220                         goto out;
3221                 }
3222
3223                 if (handler == HANDLER_ONMAX)
3224                         data->track_data.check_val = check_track_val_max;
3225                 else if (handler == HANDLER_ONCHANGE)
3226                         data->track_data.check_val = check_track_val_changed;
3227                 else {
3228                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3229                         ret = -EINVAL;
3230                         goto out;
3231                 }
3232
3233                 data->track_data.save_data = save_track_data_snapshot;
3234                 data->fn = ontrack_action;
3235                 data->action = ACTION_SNAPSHOT;
3236         } else {
3237                 char *params = strsep(&str, ")");
3238
3239                 if (str_has_prefix(action_name, "trace"))
3240                         data->use_trace_keyword = true;
3241
3242                 if (params) {
3243                         ret = parse_action_params(tr, params, data);
3244                         if (ret)
3245                                 goto out;
3246                 }
3247
3248                 if (handler == HANDLER_ONMAX)
3249                         data->track_data.check_val = check_track_val_max;
3250                 else if (handler == HANDLER_ONCHANGE)
3251                         data->track_data.check_val = check_track_val_changed;
3252
3253                 if (handler != HANDLER_ONMATCH) {
3254                         data->track_data.save_data = action_trace;
3255                         data->fn = ontrack_action;
3256                 } else
3257                         data->fn = action_trace;
3258
3259                 data->action = ACTION_TRACE;
3260         }
3261
3262         data->action_name = kstrdup(action_name, GFP_KERNEL);
3263         if (!data->action_name) {
3264                 ret = -ENOMEM;
3265                 goto out;
3266         }
3267
3268         data->handler = handler;
3269  out:
3270         return ret;
3271 }
3272
3273 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3274                                             char *str, enum handler_id handler)
3275 {
3276         struct action_data *data;
3277         int ret = -EINVAL;
3278         char *var_str;
3279
3280         data = kzalloc(sizeof(*data), GFP_KERNEL);
3281         if (!data)
3282                 return ERR_PTR(-ENOMEM);
3283
3284         var_str = strsep(&str, ")");
3285         if (!var_str || !str) {
3286                 ret = -EINVAL;
3287                 goto free;
3288         }
3289
3290         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3291         if (!data->track_data.var_str) {
3292                 ret = -ENOMEM;
3293                 goto free;
3294         }
3295
3296         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3297         if (ret)
3298                 goto free;
3299  out:
3300         return data;
3301  free:
3302         track_data_destroy(hist_data, data);
3303         data = ERR_PTR(ret);
3304         goto out;
3305 }
3306
3307 static void onmatch_destroy(struct action_data *data)
3308 {
3309         kfree(data->match_data.event);
3310         kfree(data->match_data.event_system);
3311
3312         action_data_destroy(data);
3313 }
3314
3315 static void destroy_field_var(struct field_var *field_var)
3316 {
3317         if (!field_var)
3318                 return;
3319
3320         destroy_hist_field(field_var->var, 0);
3321         destroy_hist_field(field_var->val, 0);
3322
3323         kfree(field_var);
3324 }
3325
3326 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3327 {
3328         unsigned int i;
3329
3330         for (i = 0; i < hist_data->n_field_vars; i++)
3331                 destroy_field_var(hist_data->field_vars[i]);
3332
3333         for (i = 0; i < hist_data->n_save_vars; i++)
3334                 destroy_field_var(hist_data->save_vars[i]);
3335 }
3336
3337 static void save_field_var(struct hist_trigger_data *hist_data,
3338                            struct field_var *field_var)
3339 {
3340         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3341
3342         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3343                 hist_data->n_field_var_str++;
3344 }
3345
3346
3347 static int check_synth_field(struct synth_event *event,
3348                              struct hist_field *hist_field,
3349                              unsigned int field_pos)
3350 {
3351         struct synth_field *field;
3352
3353         if (field_pos >= event->n_fields)
3354                 return -EINVAL;
3355
3356         field = event->fields[field_pos];
3357
3358         /*
3359          * A dynamic string synth field can accept static or
3360          * dynamic. A static string synth field can only accept a
3361          * same-sized static string, which is checked for later.
3362          */
3363         if (strstr(hist_field->type, "char[") && field->is_string
3364             && field->is_dynamic)
3365                 return 0;
3366
3367         if (strcmp(field->type, hist_field->type) != 0) {
3368                 if (field->size != hist_field->size ||
3369                     field->is_signed != hist_field->is_signed)
3370                         return -EINVAL;
3371         }
3372
3373         return 0;
3374 }
3375
3376 static struct hist_field *
3377 trace_action_find_var(struct hist_trigger_data *hist_data,
3378                       struct action_data *data,
3379                       char *system, char *event, char *var)
3380 {
3381         struct trace_array *tr = hist_data->event_file->tr;
3382         struct hist_field *hist_field;
3383
3384         var++; /* skip '$' */
3385
3386         hist_field = find_target_event_var(hist_data, system, event, var);
3387         if (!hist_field) {
3388                 if (!system && data->handler == HANDLER_ONMATCH) {
3389                         system = data->match_data.event_system;
3390                         event = data->match_data.event;
3391                 }
3392
3393                 hist_field = find_event_var(hist_data, system, event, var);
3394         }
3395
3396         if (!hist_field)
3397                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3398
3399         return hist_field;
3400 }
3401
3402 static struct hist_field *
3403 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3404                               struct action_data *data, char *system,
3405                               char *event, char *var)
3406 {
3407         struct hist_field *hist_field = NULL;
3408         struct field_var *field_var;
3409
3410         /*
3411          * First try to create a field var on the target event (the
3412          * currently being defined).  This will create a variable for
3413          * unqualified fields on the target event, or if qualified,
3414          * target fields that have qualified names matching the target.
3415          */
3416         field_var = create_target_field_var(hist_data, system, event, var);
3417
3418         if (field_var && !IS_ERR(field_var)) {
3419                 save_field_var(hist_data, field_var);
3420                 hist_field = field_var->var;
3421         } else {
3422                 field_var = NULL;
3423                 /*
3424                  * If no explicit system.event is specified, default to
3425                  * looking for fields on the onmatch(system.event.xxx)
3426                  * event.
3427                  */
3428                 if (!system && data->handler == HANDLER_ONMATCH) {
3429                         system = data->match_data.event_system;
3430                         event = data->match_data.event;
3431                 }
3432
3433                 /*
3434                  * At this point, we're looking at a field on another
3435                  * event.  Because we can't modify a hist trigger on
3436                  * another event to add a variable for a field, we need
3437                  * to create a new trigger on that event and create the
3438                  * variable at the same time.
3439                  */
3440                 hist_field = create_field_var_hist(hist_data, system, event, var);
3441                 if (IS_ERR(hist_field))
3442                         goto free;
3443         }
3444  out:
3445         return hist_field;
3446  free:
3447         destroy_field_var(field_var);
3448         hist_field = NULL;
3449         goto out;
3450 }
3451
3452 static int trace_action_create(struct hist_trigger_data *hist_data,
3453                                struct action_data *data)
3454 {
3455         struct trace_array *tr = hist_data->event_file->tr;
3456         char *event_name, *param, *system = NULL;
3457         struct hist_field *hist_field, *var_ref;
3458         unsigned int i;
3459         unsigned int field_pos = 0;
3460         struct synth_event *event;
3461         char *synth_event_name;
3462         int var_ref_idx, ret = 0;
3463
3464         lockdep_assert_held(&event_mutex);
3465
3466         if (data->use_trace_keyword)
3467                 synth_event_name = data->synth_event_name;
3468         else
3469                 synth_event_name = data->action_name;
3470
3471         event = find_synth_event(synth_event_name);
3472         if (!event) {
3473                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3474                 return -EINVAL;
3475         }
3476
3477         event->ref++;
3478
3479         for (i = 0; i < data->n_params; i++) {
3480                 char *p;
3481
3482                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3483                 if (!param) {
3484                         ret = -ENOMEM;
3485                         goto err;
3486                 }
3487
3488                 system = strsep(&param, ".");
3489                 if (!param) {
3490                         param = (char *)system;
3491                         system = event_name = NULL;
3492                 } else {
3493                         event_name = strsep(&param, ".");
3494                         if (!param) {
3495                                 kfree(p);
3496                                 ret = -EINVAL;
3497                                 goto err;
3498                         }
3499                 }
3500
3501                 if (param[0] == '$')
3502                         hist_field = trace_action_find_var(hist_data, data,
3503                                                            system, event_name,
3504                                                            param);
3505                 else
3506                         hist_field = trace_action_create_field_var(hist_data,
3507                                                                    data,
3508                                                                    system,
3509                                                                    event_name,
3510                                                                    param);
3511
3512                 if (!hist_field) {
3513                         kfree(p);
3514                         ret = -EINVAL;
3515                         goto err;
3516                 }
3517
3518                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3519                         var_ref = create_var_ref(hist_data, hist_field,
3520                                                  system, event_name);
3521                         if (!var_ref) {
3522                                 kfree(p);
3523                                 ret = -ENOMEM;
3524                                 goto err;
3525                         }
3526
3527                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3528                         if (WARN_ON(var_ref_idx < 0)) {
3529                                 ret = var_ref_idx;
3530                                 goto err;
3531                         }
3532
3533                         data->var_ref_idx[i] = var_ref_idx;
3534
3535                         field_pos++;
3536                         kfree(p);
3537                         continue;
3538                 }
3539
3540                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3541                 kfree(p);
3542                 ret = -EINVAL;
3543                 goto err;
3544         }
3545
3546         if (field_pos != event->n_fields) {
3547                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3548                 ret = -EINVAL;
3549                 goto err;
3550         }
3551
3552         data->synth_event = event;
3553  out:
3554         return ret;
3555  err:
3556         event->ref--;
3557
3558         goto out;
3559 }
3560
3561 static int action_create(struct hist_trigger_data *hist_data,
3562                          struct action_data *data)
3563 {
3564         struct trace_event_file *file = hist_data->event_file;
3565         struct trace_array *tr = file->tr;
3566         struct track_data *track_data;
3567         struct field_var *field_var;
3568         unsigned int i;
3569         char *param;
3570         int ret = 0;
3571
3572         if (data->action == ACTION_TRACE)
3573                 return trace_action_create(hist_data, data);
3574
3575         if (data->action == ACTION_SNAPSHOT) {
3576                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3577                 if (IS_ERR(track_data)) {
3578                         ret = PTR_ERR(track_data);
3579                         goto out;
3580                 }
3581
3582                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3583                                                    cond_snapshot_update);
3584                 if (ret)
3585                         track_data_free(track_data);
3586
3587                 goto out;
3588         }
3589
3590         if (data->action == ACTION_SAVE) {
3591                 if (hist_data->n_save_vars) {
3592                         ret = -EEXIST;
3593                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3594                         goto out;
3595                 }
3596
3597                 for (i = 0; i < data->n_params; i++) {
3598                         param = kstrdup(data->params[i], GFP_KERNEL);
3599                         if (!param) {
3600                                 ret = -ENOMEM;
3601                                 goto out;
3602                         }
3603
3604                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3605                         if (IS_ERR(field_var)) {
3606                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3607                                          errpos(param));
3608                                 ret = PTR_ERR(field_var);
3609                                 kfree(param);
3610                                 goto out;
3611                         }
3612
3613                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3614                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3615                                 hist_data->n_save_var_str++;
3616                         kfree(param);
3617                 }
3618         }
3619  out:
3620         return ret;
3621 }
3622
3623 static int onmatch_create(struct hist_trigger_data *hist_data,
3624                           struct action_data *data)
3625 {
3626         return action_create(hist_data, data);
3627 }
3628
3629 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3630 {
3631         char *match_event, *match_event_system;
3632         struct action_data *data;
3633         int ret = -EINVAL;
3634
3635         data = kzalloc(sizeof(*data), GFP_KERNEL);
3636         if (!data)
3637                 return ERR_PTR(-ENOMEM);
3638
3639         match_event = strsep(&str, ")");
3640         if (!match_event || !str) {
3641                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3642                 goto free;
3643         }
3644
3645         match_event_system = strsep(&match_event, ".");
3646         if (!match_event) {
3647                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3648                 goto free;
3649         }
3650
3651         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3652                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3653                 goto free;
3654         }
3655
3656         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3657         if (!data->match_data.event) {
3658                 ret = -ENOMEM;
3659                 goto free;
3660         }
3661
3662         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3663         if (!data->match_data.event_system) {
3664                 ret = -ENOMEM;
3665                 goto free;
3666         }
3667
3668         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3669         if (ret)
3670                 goto free;
3671  out:
3672         return data;
3673  free:
3674         onmatch_destroy(data);
3675         data = ERR_PTR(ret);
3676         goto out;
3677 }
3678
3679 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3680 {
3681         hist_data->fields[HITCOUNT_IDX] =
3682                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3683         if (!hist_data->fields[HITCOUNT_IDX])
3684                 return -ENOMEM;
3685
3686         hist_data->n_vals++;
3687         hist_data->n_fields++;
3688
3689         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3690                 return -EINVAL;
3691
3692         return 0;
3693 }
3694
3695 static int __create_val_field(struct hist_trigger_data *hist_data,
3696                               unsigned int val_idx,
3697                               struct trace_event_file *file,
3698                               char *var_name, char *field_str,
3699                               unsigned long flags)
3700 {
3701         struct hist_field *hist_field;
3702         int ret = 0;
3703
3704         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3705         if (IS_ERR(hist_field)) {
3706                 ret = PTR_ERR(hist_field);
3707                 goto out;
3708         }
3709
3710         hist_data->fields[val_idx] = hist_field;
3711
3712         ++hist_data->n_vals;
3713         ++hist_data->n_fields;
3714
3715         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3716                 ret = -EINVAL;
3717  out:
3718         return ret;
3719 }
3720
3721 static int create_val_field(struct hist_trigger_data *hist_data,
3722                             unsigned int val_idx,
3723                             struct trace_event_file *file,
3724                             char *field_str)
3725 {
3726         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3727                 return -EINVAL;
3728
3729         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3730 }
3731
3732 static int create_var_field(struct hist_trigger_data *hist_data,
3733                             unsigned int val_idx,
3734                             struct trace_event_file *file,
3735                             char *var_name, char *expr_str)
3736 {
3737         struct trace_array *tr = hist_data->event_file->tr;
3738         unsigned long flags = 0;
3739         int ret;
3740
3741         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3742                 return -EINVAL;
3743
3744         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3745                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3746                 return -EINVAL;
3747         }
3748
3749         flags |= HIST_FIELD_FL_VAR;
3750         hist_data->n_vars++;
3751         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3752                 return -EINVAL;
3753
3754         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3755
3756         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3757                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3758
3759         return ret;
3760 }
3761
3762 static int create_val_fields(struct hist_trigger_data *hist_data,
3763                              struct trace_event_file *file)
3764 {
3765         char *fields_str, *field_str;
3766         unsigned int i, j = 1;
3767         int ret;
3768
3769         ret = create_hitcount_val(hist_data);
3770         if (ret)
3771                 goto out;
3772
3773         fields_str = hist_data->attrs->vals_str;
3774         if (!fields_str)
3775                 goto out;
3776
3777         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3778                      j < TRACING_MAP_VALS_MAX; i++) {
3779                 field_str = strsep(&fields_str, ",");
3780                 if (!field_str)
3781                         break;
3782
3783                 if (strcmp(field_str, "hitcount") == 0)
3784                         continue;
3785
3786                 ret = create_val_field(hist_data, j++, file, field_str);
3787                 if (ret)
3788                         goto out;
3789         }
3790
3791         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3792                 ret = -EINVAL;
3793  out:
3794         return ret;
3795 }
3796
3797 static int create_key_field(struct hist_trigger_data *hist_data,
3798                             unsigned int key_idx,
3799                             unsigned int key_offset,
3800                             struct trace_event_file *file,
3801                             char *field_str)
3802 {
3803         struct trace_array *tr = hist_data->event_file->tr;
3804         struct hist_field *hist_field = NULL;
3805         unsigned long flags = 0;
3806         unsigned int key_size;
3807         int ret = 0;
3808
3809         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3810                 return -EINVAL;
3811
3812         flags |= HIST_FIELD_FL_KEY;
3813
3814         if (strcmp(field_str, "stacktrace") == 0) {
3815                 flags |= HIST_FIELD_FL_STACKTRACE;
3816                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3817                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3818         } else {
3819                 hist_field = parse_expr(hist_data, file, field_str, flags,
3820                                         NULL, 0);
3821                 if (IS_ERR(hist_field)) {
3822                         ret = PTR_ERR(hist_field);
3823                         goto out;
3824                 }
3825
3826                 if (field_has_hist_vars(hist_field, 0)) {
3827                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3828                         destroy_hist_field(hist_field, 0);
3829                         ret = -EINVAL;
3830                         goto out;
3831                 }
3832
3833                 key_size = hist_field->size;
3834         }
3835
3836         hist_data->fields[key_idx] = hist_field;
3837
3838         key_size = ALIGN(key_size, sizeof(u64));
3839         hist_data->fields[key_idx]->size = key_size;
3840         hist_data->fields[key_idx]->offset = key_offset;
3841
3842         hist_data->key_size += key_size;
3843
3844         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3845                 ret = -EINVAL;
3846                 goto out;
3847         }
3848
3849         hist_data->n_keys++;
3850         hist_data->n_fields++;
3851
3852         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3853                 return -EINVAL;
3854
3855         ret = key_size;
3856  out:
3857         return ret;
3858 }
3859
3860 static int create_key_fields(struct hist_trigger_data *hist_data,
3861                              struct trace_event_file *file)
3862 {
3863         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3864         char *fields_str, *field_str;
3865         int ret = -EINVAL;
3866
3867         fields_str = hist_data->attrs->keys_str;
3868         if (!fields_str)
3869                 goto out;
3870
3871         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3872                 field_str = strsep(&fields_str, ",");
3873                 if (!field_str)
3874                         break;
3875                 ret = create_key_field(hist_data, i, key_offset,
3876                                        file, field_str);
3877                 if (ret < 0)
3878                         goto out;
3879                 key_offset += ret;
3880         }
3881         if (fields_str) {
3882                 ret = -EINVAL;
3883                 goto out;
3884         }
3885         ret = 0;
3886  out:
3887         return ret;
3888 }
3889
3890 static int create_var_fields(struct hist_trigger_data *hist_data,
3891                              struct trace_event_file *file)
3892 {
3893         unsigned int i, j = hist_data->n_vals;
3894         int ret = 0;
3895
3896         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3897
3898         for (i = 0; i < n_vars; i++) {
3899                 char *var_name = hist_data->attrs->var_defs.name[i];
3900                 char *expr = hist_data->attrs->var_defs.expr[i];
3901
3902                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3903                 if (ret)
3904                         goto out;
3905         }
3906  out:
3907         return ret;
3908 }
3909
3910 static void free_var_defs(struct hist_trigger_data *hist_data)
3911 {
3912         unsigned int i;
3913
3914         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3915                 kfree(hist_data->attrs->var_defs.name[i]);
3916                 kfree(hist_data->attrs->var_defs.expr[i]);
3917         }
3918
3919         hist_data->attrs->var_defs.n_vars = 0;
3920 }
3921
3922 static int parse_var_defs(struct hist_trigger_data *hist_data)
3923 {
3924         struct trace_array *tr = hist_data->event_file->tr;
3925         char *s, *str, *var_name, *field_str;
3926         unsigned int i, j, n_vars = 0;
3927         int ret = 0;
3928
3929         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3930                 str = hist_data->attrs->assignment_str[i];
3931                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3932                         field_str = strsep(&str, ",");
3933                         if (!field_str)
3934                                 break;
3935
3936                         var_name = strsep(&field_str, "=");
3937                         if (!var_name || !field_str) {
3938                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3939                                          errpos(var_name));
3940                                 ret = -EINVAL;
3941                                 goto free;
3942                         }
3943
3944                         if (n_vars == TRACING_MAP_VARS_MAX) {
3945                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3946                                 ret = -EINVAL;
3947                                 goto free;
3948                         }
3949
3950                         s = kstrdup(var_name, GFP_KERNEL);
3951                         if (!s) {
3952                                 ret = -ENOMEM;
3953                                 goto free;
3954                         }
3955                         hist_data->attrs->var_defs.name[n_vars] = s;
3956
3957                         s = kstrdup(field_str, GFP_KERNEL);
3958                         if (!s) {
3959                                 ret = -ENOMEM;
3960                                 goto free;
3961                         }
3962                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3963
3964                         hist_data->attrs->var_defs.n_vars = n_vars;
3965                 }
3966         }
3967
3968         return ret;
3969  free:
3970         free_var_defs(hist_data);
3971
3972         return ret;
3973 }
3974
3975 static int create_hist_fields(struct hist_trigger_data *hist_data,
3976                               struct trace_event_file *file)
3977 {
3978         int ret;
3979
3980         ret = parse_var_defs(hist_data);
3981         if (ret)
3982                 goto out;
3983
3984         ret = create_val_fields(hist_data, file);
3985         if (ret)
3986                 goto out;
3987
3988         ret = create_var_fields(hist_data, file);
3989         if (ret)
3990                 goto out;
3991
3992         ret = create_key_fields(hist_data, file);
3993         if (ret)
3994                 goto out;
3995  out:
3996         free_var_defs(hist_data);
3997
3998         return ret;
3999 }
4000
4001 static int is_descending(struct trace_array *tr, const char *str)
4002 {
4003         if (!str)
4004                 return 0;
4005
4006         if (strcmp(str, "descending") == 0)
4007                 return 1;
4008
4009         if (strcmp(str, "ascending") == 0)
4010                 return 0;
4011
4012         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4013
4014         return -EINVAL;
4015 }
4016
4017 static int create_sort_keys(struct hist_trigger_data *hist_data)
4018 {
4019         struct trace_array *tr = hist_data->event_file->tr;
4020         char *fields_str = hist_data->attrs->sort_key_str;
4021         struct tracing_map_sort_key *sort_key;
4022         int descending, ret = 0;
4023         unsigned int i, j, k;
4024
4025         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4026
4027         if (!fields_str)
4028                 goto out;
4029
4030         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4031                 struct hist_field *hist_field;
4032                 char *field_str, *field_name;
4033                 const char *test_name;
4034
4035                 sort_key = &hist_data->sort_keys[i];
4036
4037                 field_str = strsep(&fields_str, ",");
4038                 if (!field_str)
4039                         break;
4040
4041                 if (!*field_str) {
4042                         ret = -EINVAL;
4043                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4044                         break;
4045                 }
4046
4047                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4048                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4049                         ret = -EINVAL;
4050                         break;
4051                 }
4052
4053                 field_name = strsep(&field_str, ".");
4054                 if (!field_name || !*field_name) {
4055                         ret = -EINVAL;
4056                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4057                         break;
4058                 }
4059
4060                 if (strcmp(field_name, "hitcount") == 0) {
4061                         descending = is_descending(tr, field_str);
4062                         if (descending < 0) {
4063                                 ret = descending;
4064                                 break;
4065                         }
4066                         sort_key->descending = descending;
4067                         continue;
4068                 }
4069
4070                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4071                         unsigned int idx;
4072
4073                         hist_field = hist_data->fields[j];
4074                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4075                                 continue;
4076
4077                         idx = k++;
4078
4079                         test_name = hist_field_name(hist_field, 0);
4080
4081                         if (strcmp(field_name, test_name) == 0) {
4082                                 sort_key->field_idx = idx;
4083                                 descending = is_descending(tr, field_str);
4084                                 if (descending < 0) {
4085                                         ret = descending;
4086                                         goto out;
4087                                 }
4088                                 sort_key->descending = descending;
4089                                 break;
4090                         }
4091                 }
4092                 if (j == hist_data->n_fields) {
4093                         ret = -EINVAL;
4094                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4095                         break;
4096                 }
4097         }
4098
4099         hist_data->n_sort_keys = i;
4100  out:
4101         return ret;
4102 }
4103
4104 static void destroy_actions(struct hist_trigger_data *hist_data)
4105 {
4106         unsigned int i;
4107
4108         for (i = 0; i < hist_data->n_actions; i++) {
4109                 struct action_data *data = hist_data->actions[i];
4110
4111                 if (data->handler == HANDLER_ONMATCH)
4112                         onmatch_destroy(data);
4113                 else if (data->handler == HANDLER_ONMAX ||
4114                          data->handler == HANDLER_ONCHANGE)
4115                         track_data_destroy(hist_data, data);
4116                 else
4117                         kfree(data);
4118         }
4119 }
4120
4121 static int parse_actions(struct hist_trigger_data *hist_data)
4122 {
4123         struct trace_array *tr = hist_data->event_file->tr;
4124         struct action_data *data;
4125         unsigned int i;
4126         int ret = 0;
4127         char *str;
4128         int len;
4129
4130         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4131                 str = hist_data->attrs->action_str[i];
4132
4133                 if ((len = str_has_prefix(str, "onmatch("))) {
4134                         char *action_str = str + len;
4135
4136                         data = onmatch_parse(tr, action_str);
4137                         if (IS_ERR(data)) {
4138                                 ret = PTR_ERR(data);
4139                                 break;
4140                         }
4141                 } else if ((len = str_has_prefix(str, "onmax("))) {
4142                         char *action_str = str + len;
4143
4144                         data = track_data_parse(hist_data, action_str,
4145                                                 HANDLER_ONMAX);
4146                         if (IS_ERR(data)) {
4147                                 ret = PTR_ERR(data);
4148                                 break;
4149                         }
4150                 } else if ((len = str_has_prefix(str, "onchange("))) {
4151                         char *action_str = str + len;
4152
4153                         data = track_data_parse(hist_data, action_str,
4154                                                 HANDLER_ONCHANGE);
4155                         if (IS_ERR(data)) {
4156                                 ret = PTR_ERR(data);
4157                                 break;
4158                         }
4159                 } else {
4160                         ret = -EINVAL;
4161                         break;
4162                 }
4163
4164                 hist_data->actions[hist_data->n_actions++] = data;
4165         }
4166
4167         return ret;
4168 }
4169
4170 static int create_actions(struct hist_trigger_data *hist_data)
4171 {
4172         struct action_data *data;
4173         unsigned int i;
4174         int ret = 0;
4175
4176         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4177                 data = hist_data->actions[i];
4178
4179                 if (data->handler == HANDLER_ONMATCH) {
4180                         ret = onmatch_create(hist_data, data);
4181                         if (ret)
4182                                 break;
4183                 } else if (data->handler == HANDLER_ONMAX ||
4184                            data->handler == HANDLER_ONCHANGE) {
4185                         ret = track_data_create(hist_data, data);
4186                         if (ret)
4187                                 break;
4188                 } else {
4189                         ret = -EINVAL;
4190                         break;
4191                 }
4192         }
4193
4194         return ret;
4195 }
4196
4197 static void print_actions(struct seq_file *m,
4198                           struct hist_trigger_data *hist_data,
4199                           struct tracing_map_elt *elt)
4200 {
4201         unsigned int i;
4202
4203         for (i = 0; i < hist_data->n_actions; i++) {
4204                 struct action_data *data = hist_data->actions[i];
4205
4206                 if (data->action == ACTION_SNAPSHOT)
4207                         continue;
4208
4209                 if (data->handler == HANDLER_ONMAX ||
4210                     data->handler == HANDLER_ONCHANGE)
4211                         track_data_print(m, hist_data, elt, data);
4212         }
4213 }
4214
4215 static void print_action_spec(struct seq_file *m,
4216                               struct hist_trigger_data *hist_data,
4217                               struct action_data *data)
4218 {
4219         unsigned int i;
4220
4221         if (data->action == ACTION_SAVE) {
4222                 for (i = 0; i < hist_data->n_save_vars; i++) {
4223                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4224                         if (i < hist_data->n_save_vars - 1)
4225                                 seq_puts(m, ",");
4226                 }
4227         } else if (data->action == ACTION_TRACE) {
4228                 if (data->use_trace_keyword)
4229                         seq_printf(m, "%s", data->synth_event_name);
4230                 for (i = 0; i < data->n_params; i++) {
4231                         if (i || data->use_trace_keyword)
4232                                 seq_puts(m, ",");
4233                         seq_printf(m, "%s", data->params[i]);
4234                 }
4235         }
4236 }
4237
4238 static void print_track_data_spec(struct seq_file *m,
4239                                   struct hist_trigger_data *hist_data,
4240                                   struct action_data *data)
4241 {
4242         if (data->handler == HANDLER_ONMAX)
4243                 seq_puts(m, ":onmax(");
4244         else if (data->handler == HANDLER_ONCHANGE)
4245                 seq_puts(m, ":onchange(");
4246         seq_printf(m, "%s", data->track_data.var_str);
4247         seq_printf(m, ").%s(", data->action_name);
4248
4249         print_action_spec(m, hist_data, data);
4250
4251         seq_puts(m, ")");
4252 }
4253
4254 static void print_onmatch_spec(struct seq_file *m,
4255                                struct hist_trigger_data *hist_data,
4256                                struct action_data *data)
4257 {
4258         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4259                    data->match_data.event);
4260
4261         seq_printf(m, "%s(", data->action_name);
4262
4263         print_action_spec(m, hist_data, data);
4264
4265         seq_puts(m, ")");
4266 }
4267
4268 static bool actions_match(struct hist_trigger_data *hist_data,
4269                           struct hist_trigger_data *hist_data_test)
4270 {
4271         unsigned int i, j;
4272
4273         if (hist_data->n_actions != hist_data_test->n_actions)
4274                 return false;
4275
4276         for (i = 0; i < hist_data->n_actions; i++) {
4277                 struct action_data *data = hist_data->actions[i];
4278                 struct action_data *data_test = hist_data_test->actions[i];
4279                 char *action_name, *action_name_test;
4280
4281                 if (data->handler != data_test->handler)
4282                         return false;
4283                 if (data->action != data_test->action)
4284                         return false;
4285
4286                 if (data->n_params != data_test->n_params)
4287                         return false;
4288
4289                 for (j = 0; j < data->n_params; j++) {
4290                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4291                                 return false;
4292                 }
4293
4294                 if (data->use_trace_keyword)
4295                         action_name = data->synth_event_name;
4296                 else
4297                         action_name = data->action_name;
4298
4299                 if (data_test->use_trace_keyword)
4300                         action_name_test = data_test->synth_event_name;
4301                 else
4302                         action_name_test = data_test->action_name;
4303
4304                 if (strcmp(action_name, action_name_test) != 0)
4305                         return false;
4306
4307                 if (data->handler == HANDLER_ONMATCH) {
4308                         if (strcmp(data->match_data.event_system,
4309                                    data_test->match_data.event_system) != 0)
4310                                 return false;
4311                         if (strcmp(data->match_data.event,
4312                                    data_test->match_data.event) != 0)
4313                                 return false;
4314                 } else if (data->handler == HANDLER_ONMAX ||
4315                            data->handler == HANDLER_ONCHANGE) {
4316                         if (strcmp(data->track_data.var_str,
4317                                    data_test->track_data.var_str) != 0)
4318                                 return false;
4319                 }
4320         }
4321
4322         return true;
4323 }
4324
4325
4326 static void print_actions_spec(struct seq_file *m,
4327                                struct hist_trigger_data *hist_data)
4328 {
4329         unsigned int i;
4330
4331         for (i = 0; i < hist_data->n_actions; i++) {
4332                 struct action_data *data = hist_data->actions[i];
4333
4334                 if (data->handler == HANDLER_ONMATCH)
4335                         print_onmatch_spec(m, hist_data, data);
4336                 else if (data->handler == HANDLER_ONMAX ||
4337                          data->handler == HANDLER_ONCHANGE)
4338                         print_track_data_spec(m, hist_data, data);
4339         }
4340 }
4341
4342 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4343 {
4344         unsigned int i;
4345
4346         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4347                 kfree(hist_data->field_var_hists[i]->cmd);
4348                 kfree(hist_data->field_var_hists[i]);
4349         }
4350 }
4351
4352 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4353 {
4354         if (!hist_data)
4355                 return;
4356
4357         destroy_hist_trigger_attrs(hist_data->attrs);
4358         destroy_hist_fields(hist_data);
4359         tracing_map_destroy(hist_data->map);
4360
4361         destroy_actions(hist_data);
4362         destroy_field_vars(hist_data);
4363         destroy_field_var_hists(hist_data);
4364
4365         kfree(hist_data);
4366 }
4367
4368 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4369 {
4370         struct tracing_map *map = hist_data->map;
4371         struct ftrace_event_field *field;
4372         struct hist_field *hist_field;
4373         int i, idx = 0;
4374
4375         for_each_hist_field(i, hist_data) {
4376                 hist_field = hist_data->fields[i];
4377                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4378                         tracing_map_cmp_fn_t cmp_fn;
4379
4380                         field = hist_field->field;
4381
4382                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4383                                 cmp_fn = tracing_map_cmp_none;
4384                         else if (!field)
4385                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4386                                                              hist_field->is_signed);
4387                         else if (is_string_field(field))
4388                                 cmp_fn = tracing_map_cmp_string;
4389                         else
4390                                 cmp_fn = tracing_map_cmp_num(field->size,
4391                                                              field->is_signed);
4392                         idx = tracing_map_add_key_field(map,
4393                                                         hist_field->offset,
4394                                                         cmp_fn);
4395                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4396                         idx = tracing_map_add_sum_field(map);
4397
4398                 if (idx < 0)
4399                         return idx;
4400
4401                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4402                         idx = tracing_map_add_var(map);
4403                         if (idx < 0)
4404                                 return idx;
4405                         hist_field->var.idx = idx;
4406                         hist_field->var.hist_data = hist_data;
4407                 }
4408         }
4409
4410         return 0;
4411 }
4412
4413 static struct hist_trigger_data *
4414 create_hist_data(unsigned int map_bits,
4415                  struct hist_trigger_attrs *attrs,
4416                  struct trace_event_file *file,
4417                  bool remove)
4418 {
4419         const struct tracing_map_ops *map_ops = NULL;
4420         struct hist_trigger_data *hist_data;
4421         int ret = 0;
4422
4423         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4424         if (!hist_data)
4425                 return ERR_PTR(-ENOMEM);
4426
4427         hist_data->attrs = attrs;
4428         hist_data->remove = remove;
4429         hist_data->event_file = file;
4430
4431         ret = parse_actions(hist_data);
4432         if (ret)
4433                 goto free;
4434
4435         ret = create_hist_fields(hist_data, file);
4436         if (ret)
4437                 goto free;
4438
4439         ret = create_sort_keys(hist_data);
4440         if (ret)
4441                 goto free;
4442
4443         map_ops = &hist_trigger_elt_data_ops;
4444
4445         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4446                                             map_ops, hist_data);
4447         if (IS_ERR(hist_data->map)) {
4448                 ret = PTR_ERR(hist_data->map);
4449                 hist_data->map = NULL;
4450                 goto free;
4451         }
4452
4453         ret = create_tracing_map_fields(hist_data);
4454         if (ret)
4455                 goto free;
4456  out:
4457         return hist_data;
4458  free:
4459         hist_data->attrs = NULL;
4460
4461         destroy_hist_data(hist_data);
4462
4463         hist_data = ERR_PTR(ret);
4464
4465         goto out;
4466 }
4467
4468 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4469                                     struct tracing_map_elt *elt,
4470                                     struct trace_buffer *buffer, void *rec,
4471                                     struct ring_buffer_event *rbe,
4472                                     u64 *var_ref_vals)
4473 {
4474         struct hist_elt_data *elt_data;
4475         struct hist_field *hist_field;
4476         unsigned int i, var_idx;
4477         u64 hist_val;
4478
4479         elt_data = elt->private_data;
4480         elt_data->var_ref_vals = var_ref_vals;
4481
4482         for_each_hist_val_field(i, hist_data) {
4483                 hist_field = hist_data->fields[i];
4484                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4485                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4486                         var_idx = hist_field->var.idx;
4487
4488                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4489                                 unsigned int str_start, var_str_idx, idx;
4490                                 char *str, *val_str;
4491
4492                                 str_start = hist_data->n_field_var_str +
4493                                         hist_data->n_save_var_str;
4494                                 var_str_idx = hist_field->var_str_idx;
4495                                 idx = str_start + var_str_idx;
4496
4497                                 str = elt_data->field_var_str[idx];
4498                                 val_str = (char *)(uintptr_t)hist_val;
4499                                 strscpy(str, val_str, STR_VAR_LEN_MAX);
4500
4501                                 hist_val = (u64)(uintptr_t)str;
4502                         }
4503                         tracing_map_set_var(elt, var_idx, hist_val);
4504                         continue;
4505                 }
4506                 tracing_map_update_sum(elt, i, hist_val);
4507         }
4508
4509         for_each_hist_key_field(i, hist_data) {
4510                 hist_field = hist_data->fields[i];
4511                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4512                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4513                         var_idx = hist_field->var.idx;
4514                         tracing_map_set_var(elt, var_idx, hist_val);
4515                 }
4516         }
4517
4518         update_field_vars(hist_data, elt, buffer, rbe, rec);
4519 }
4520
4521 static inline void add_to_key(char *compound_key, void *key,
4522                               struct hist_field *key_field, void *rec)
4523 {
4524         size_t size = key_field->size;
4525
4526         if (key_field->flags & HIST_FIELD_FL_STRING) {
4527                 struct ftrace_event_field *field;
4528
4529                 field = key_field->field;
4530                 if (field->filter_type == FILTER_DYN_STRING)
4531                         size = *(u32 *)(rec + field->offset) >> 16;
4532                 else if (field->filter_type == FILTER_STATIC_STRING)
4533                         size = field->size;
4534
4535                 /* ensure NULL-termination */
4536                 if (size > key_field->size - 1)
4537                         size = key_field->size - 1;
4538
4539                 strncpy(compound_key + key_field->offset, (char *)key, size);
4540         } else
4541                 memcpy(compound_key + key_field->offset, key, size);
4542 }
4543
4544 static void
4545 hist_trigger_actions(struct hist_trigger_data *hist_data,
4546                      struct tracing_map_elt *elt,
4547                      struct trace_buffer *buffer, void *rec,
4548                      struct ring_buffer_event *rbe, void *key,
4549                      u64 *var_ref_vals)
4550 {
4551         struct action_data *data;
4552         unsigned int i;
4553
4554         for (i = 0; i < hist_data->n_actions; i++) {
4555                 data = hist_data->actions[i];
4556                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4557         }
4558 }
4559
4560 static void event_hist_trigger(struct event_trigger_data *data,
4561                                struct trace_buffer *buffer, void *rec,
4562                                struct ring_buffer_event *rbe)
4563 {
4564         struct hist_trigger_data *hist_data = data->private_data;
4565         bool use_compound_key = (hist_data->n_keys > 1);
4566         unsigned long entries[HIST_STACKTRACE_DEPTH];
4567         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4568         char compound_key[HIST_KEY_SIZE_MAX];
4569         struct tracing_map_elt *elt = NULL;
4570         struct hist_field *key_field;
4571         u64 field_contents;
4572         void *key = NULL;
4573         unsigned int i;
4574
4575         memset(compound_key, 0, hist_data->key_size);
4576
4577         for_each_hist_key_field(i, hist_data) {
4578                 key_field = hist_data->fields[i];
4579
4580                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4581                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4582                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4583                                          HIST_STACKTRACE_SKIP);
4584                         key = entries;
4585                 } else {
4586                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4587                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4588                                 key = (void *)(unsigned long)field_contents;
4589                                 use_compound_key = true;
4590                         } else
4591                                 key = (void *)&field_contents;
4592                 }
4593
4594                 if (use_compound_key)
4595                         add_to_key(compound_key, key, key_field, rec);
4596         }
4597
4598         if (use_compound_key)
4599                 key = compound_key;
4600
4601         if (hist_data->n_var_refs &&
4602             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4603                 return;
4604
4605         elt = tracing_map_insert(hist_data->map, key);
4606         if (!elt)
4607                 return;
4608
4609         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4610
4611         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4612                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4613 }
4614
4615 static void hist_trigger_stacktrace_print(struct seq_file *m,
4616                                           unsigned long *stacktrace_entries,
4617                                           unsigned int max_entries)
4618 {
4619         char str[KSYM_SYMBOL_LEN];
4620         unsigned int spaces = 8;
4621         unsigned int i;
4622
4623         for (i = 0; i < max_entries; i++) {
4624                 if (!stacktrace_entries[i])
4625                         return;
4626
4627                 seq_printf(m, "%*c", 1 + spaces, ' ');
4628                 sprint_symbol(str, stacktrace_entries[i]);
4629                 seq_printf(m, "%s\n", str);
4630         }
4631 }
4632
4633 static void hist_trigger_print_key(struct seq_file *m,
4634                                    struct hist_trigger_data *hist_data,
4635                                    void *key,
4636                                    struct tracing_map_elt *elt)
4637 {
4638         struct hist_field *key_field;
4639         char str[KSYM_SYMBOL_LEN];
4640         bool multiline = false;
4641         const char *field_name;
4642         unsigned int i;
4643         u64 uval;
4644
4645         seq_puts(m, "{ ");
4646
4647         for_each_hist_key_field(i, hist_data) {
4648                 key_field = hist_data->fields[i];
4649
4650                 if (i > hist_data->n_vals)
4651                         seq_puts(m, ", ");
4652
4653                 field_name = hist_field_name(key_field, 0);
4654
4655                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4656                         uval = *(u64 *)(key + key_field->offset);
4657                         seq_printf(m, "%s: %llx", field_name, uval);
4658                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4659                         uval = *(u64 *)(key + key_field->offset);
4660                         sprint_symbol_no_offset(str, uval);
4661                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4662                                    uval, str);
4663                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4664                         uval = *(u64 *)(key + key_field->offset);
4665                         sprint_symbol(str, uval);
4666                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4667                                    uval, str);
4668                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4669                         struct hist_elt_data *elt_data = elt->private_data;
4670                         char *comm;
4671
4672                         if (WARN_ON_ONCE(!elt_data))
4673                                 return;
4674
4675                         comm = elt_data->comm;
4676
4677                         uval = *(u64 *)(key + key_field->offset);
4678                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4679                                    comm, uval);
4680                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4681                         const char *syscall_name;
4682
4683                         uval = *(u64 *)(key + key_field->offset);
4684                         syscall_name = get_syscall_name(uval);
4685                         if (!syscall_name)
4686                                 syscall_name = "unknown_syscall";
4687
4688                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4689                                    syscall_name, uval);
4690                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4691                         seq_puts(m, "stacktrace:\n");
4692                         hist_trigger_stacktrace_print(m,
4693                                                       key + key_field->offset,
4694                                                       HIST_STACKTRACE_DEPTH);
4695                         multiline = true;
4696                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4697                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4698                                    *(u64 *)(key + key_field->offset));
4699                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4700                         seq_printf(m, "%s: %-50s", field_name,
4701                                    (char *)(key + key_field->offset));
4702                 } else {
4703                         uval = *(u64 *)(key + key_field->offset);
4704                         seq_printf(m, "%s: %10llu", field_name, uval);
4705                 }
4706         }
4707
4708         if (!multiline)
4709                 seq_puts(m, " ");
4710
4711         seq_puts(m, "}");
4712 }
4713
4714 static void hist_trigger_entry_print(struct seq_file *m,
4715                                      struct hist_trigger_data *hist_data,
4716                                      void *key,
4717                                      struct tracing_map_elt *elt)
4718 {
4719         const char *field_name;
4720         unsigned int i;
4721
4722         hist_trigger_print_key(m, hist_data, key, elt);
4723
4724         seq_printf(m, " hitcount: %10llu",
4725                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4726
4727         for (i = 1; i < hist_data->n_vals; i++) {
4728                 field_name = hist_field_name(hist_data->fields[i], 0);
4729
4730                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4731                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4732                         continue;
4733
4734                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4735                         seq_printf(m, "  %s: %10llx", field_name,
4736                                    tracing_map_read_sum(elt, i));
4737                 } else {
4738                         seq_printf(m, "  %s: %10llu", field_name,
4739                                    tracing_map_read_sum(elt, i));
4740                 }
4741         }
4742
4743         print_actions(m, hist_data, elt);
4744
4745         seq_puts(m, "\n");
4746 }
4747
4748 static int print_entries(struct seq_file *m,
4749                          struct hist_trigger_data *hist_data)
4750 {
4751         struct tracing_map_sort_entry **sort_entries = NULL;
4752         struct tracing_map *map = hist_data->map;
4753         int i, n_entries;
4754
4755         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4756                                              hist_data->n_sort_keys,
4757                                              &sort_entries);
4758         if (n_entries < 0)
4759                 return n_entries;
4760
4761         for (i = 0; i < n_entries; i++)
4762                 hist_trigger_entry_print(m, hist_data,
4763                                          sort_entries[i]->key,
4764                                          sort_entries[i]->elt);
4765
4766         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4767
4768         return n_entries;
4769 }
4770
4771 static void hist_trigger_show(struct seq_file *m,
4772                               struct event_trigger_data *data, int n)
4773 {
4774         struct hist_trigger_data *hist_data;
4775         int n_entries;
4776
4777         if (n > 0)
4778                 seq_puts(m, "\n\n");
4779
4780         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4781         data->ops->print(m, data->ops, data);
4782         seq_puts(m, "#\n\n");
4783
4784         hist_data = data->private_data;
4785         n_entries = print_entries(m, hist_data);
4786         if (n_entries < 0)
4787                 n_entries = 0;
4788
4789         track_data_snapshot_print(m, hist_data);
4790
4791         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4792                    (u64)atomic64_read(&hist_data->map->hits),
4793                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4794 }
4795
4796 static int hist_show(struct seq_file *m, void *v)
4797 {
4798         struct event_trigger_data *data;
4799         struct trace_event_file *event_file;
4800         int n = 0, ret = 0;
4801
4802         mutex_lock(&event_mutex);
4803
4804         event_file = event_file_data(m->private);
4805         if (unlikely(!event_file)) {
4806                 ret = -ENODEV;
4807                 goto out_unlock;
4808         }
4809
4810         list_for_each_entry(data, &event_file->triggers, list) {
4811                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4812                         hist_trigger_show(m, data, n++);
4813         }
4814
4815  out_unlock:
4816         mutex_unlock(&event_mutex);
4817
4818         return ret;
4819 }
4820
4821 static int event_hist_open(struct inode *inode, struct file *file)
4822 {
4823         int ret;
4824
4825         ret = security_locked_down(LOCKDOWN_TRACEFS);
4826         if (ret)
4827                 return ret;
4828
4829         return single_open(file, hist_show, file);
4830 }
4831
4832 const struct file_operations event_hist_fops = {
4833         .open = event_hist_open,
4834         .read = seq_read,
4835         .llseek = seq_lseek,
4836         .release = single_release,
4837 };
4838
4839 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4840 static void hist_field_debug_show_flags(struct seq_file *m,
4841                                         unsigned long flags)
4842 {
4843         seq_puts(m, "      flags:\n");
4844
4845         if (flags & HIST_FIELD_FL_KEY)
4846                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4847         else if (flags & HIST_FIELD_FL_HITCOUNT)
4848                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4849         else if (flags & HIST_FIELD_FL_VAR)
4850                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4851         else if (flags & HIST_FIELD_FL_VAR_REF)
4852                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4853         else
4854                 seq_puts(m, "        VAL: normal u64 value\n");
4855
4856         if (flags & HIST_FIELD_FL_ALIAS)
4857                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4858 }
4859
4860 static int hist_field_debug_show(struct seq_file *m,
4861                                  struct hist_field *field, unsigned long flags)
4862 {
4863         if ((field->flags & flags) != flags) {
4864                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4865                 return -EINVAL;
4866         }
4867
4868         hist_field_debug_show_flags(m, field->flags);
4869         if (field->field)
4870                 seq_printf(m, "      ftrace_event_field name: %s\n",
4871                            field->field->name);
4872
4873         if (field->flags & HIST_FIELD_FL_VAR) {
4874                 seq_printf(m, "      var.name: %s\n", field->var.name);
4875                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4876                            field->var.idx);
4877         }
4878
4879         if (field->flags & HIST_FIELD_FL_ALIAS)
4880                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4881                            field->var_ref_idx);
4882
4883         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4884                 seq_printf(m, "      name: %s\n", field->name);
4885                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4886                            field->var.idx);
4887                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4888                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4889                            field->var_ref_idx);
4890                 if (field->system)
4891                         seq_printf(m, "      system: %s\n", field->system);
4892                 if (field->event_name)
4893                         seq_printf(m, "      event_name: %s\n", field->event_name);
4894         }
4895
4896         seq_printf(m, "      type: %s\n", field->type);
4897         seq_printf(m, "      size: %u\n", field->size);
4898         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4899
4900         return 0;
4901 }
4902
4903 static int field_var_debug_show(struct seq_file *m,
4904                                 struct field_var *field_var, unsigned int i,
4905                                 bool save_vars)
4906 {
4907         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4908         struct hist_field *field;
4909         int ret = 0;
4910
4911         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4912
4913         field = field_var->var;
4914
4915         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4916
4917         hist_field_debug_show_flags(m, field->flags);
4918         seq_printf(m, "      var.name: %s\n", field->var.name);
4919         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4920                    field->var.idx);
4921
4922         field = field_var->val;
4923
4924         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4925         if (field->field)
4926                 seq_printf(m, "      ftrace_event_field name: %s\n",
4927                            field->field->name);
4928         else {
4929                 ret = -EINVAL;
4930                 goto out;
4931         }
4932
4933         seq_printf(m, "      type: %s\n", field->type);
4934         seq_printf(m, "      size: %u\n", field->size);
4935         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4936 out:
4937         return ret;
4938 }
4939
4940 static int hist_action_debug_show(struct seq_file *m,
4941                                   struct action_data *data, int i)
4942 {
4943         int ret = 0;
4944
4945         if (data->handler == HANDLER_ONMAX ||
4946             data->handler == HANDLER_ONCHANGE) {
4947                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4948                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4949                                             HIST_FIELD_FL_VAR_REF);
4950                 if (ret)
4951                         goto out;
4952
4953                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4954                 ret = hist_field_debug_show(m, data->track_data.track_var,
4955                                             HIST_FIELD_FL_VAR);
4956                 if (ret)
4957                         goto out;
4958         }
4959
4960         if (data->handler == HANDLER_ONMATCH) {
4961                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4962                            i, data->match_data.event_system);
4963                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4964                            i, data->match_data.event);
4965         }
4966 out:
4967         return ret;
4968 }
4969
4970 static int hist_actions_debug_show(struct seq_file *m,
4971                                    struct hist_trigger_data *hist_data)
4972 {
4973         int i, ret = 0;
4974
4975         if (hist_data->n_actions)
4976                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4977
4978         for (i = 0; i < hist_data->n_actions; i++) {
4979                 struct action_data *action = hist_data->actions[i];
4980
4981                 ret = hist_action_debug_show(m, action, i);
4982                 if (ret)
4983                         goto out;
4984         }
4985
4986         if (hist_data->n_save_vars)
4987                 seq_puts(m, "\n  save action variables (save() params):\n");
4988
4989         for (i = 0; i < hist_data->n_save_vars; i++) {
4990                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4991                 if (ret)
4992                         goto out;
4993         }
4994 out:
4995         return ret;
4996 }
4997
4998 static void hist_trigger_debug_show(struct seq_file *m,
4999                                     struct event_trigger_data *data, int n)
5000 {
5001         struct hist_trigger_data *hist_data;
5002         int i, ret;
5003
5004         if (n > 0)
5005                 seq_puts(m, "\n\n");
5006
5007         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5008         data->ops->print(m, data->ops, data);
5009         seq_puts(m, "#\n\n");
5010
5011         hist_data = data->private_data;
5012
5013         seq_printf(m, "hist_data: %p\n\n", hist_data);
5014         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5015         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5016         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5017
5018         seq_puts(m, "\n  val fields:\n\n");
5019
5020         seq_puts(m, "    hist_data->fields[0]:\n");
5021         ret = hist_field_debug_show(m, hist_data->fields[0],
5022                                     HIST_FIELD_FL_HITCOUNT);
5023         if (ret)
5024                 return;
5025
5026         for (i = 1; i < hist_data->n_vals; i++) {
5027                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5028                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5029                 if (ret)
5030                         return;
5031         }
5032
5033         seq_puts(m, "\n  key fields:\n");
5034
5035         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5036                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5037                 ret = hist_field_debug_show(m, hist_data->fields[i],
5038                                             HIST_FIELD_FL_KEY);
5039                 if (ret)
5040                         return;
5041         }
5042
5043         if (hist_data->n_var_refs)
5044                 seq_puts(m, "\n  variable reference fields:\n");
5045
5046         for (i = 0; i < hist_data->n_var_refs; i++) {
5047                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5048                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5049                                             HIST_FIELD_FL_VAR_REF);
5050                 if (ret)
5051                         return;
5052         }
5053
5054         if (hist_data->n_field_vars)
5055                 seq_puts(m, "\n  field variables:\n");
5056
5057         for (i = 0; i < hist_data->n_field_vars; i++) {
5058                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5059                 if (ret)
5060                         return;
5061         }
5062
5063         ret = hist_actions_debug_show(m, hist_data);
5064         if (ret)
5065                 return;
5066 }
5067
5068 static int hist_debug_show(struct seq_file *m, void *v)
5069 {
5070         struct event_trigger_data *data;
5071         struct trace_event_file *event_file;
5072         int n = 0, ret = 0;
5073
5074         mutex_lock(&event_mutex);
5075
5076         event_file = event_file_data(m->private);
5077         if (unlikely(!event_file)) {
5078                 ret = -ENODEV;
5079                 goto out_unlock;
5080         }
5081
5082         list_for_each_entry(data, &event_file->triggers, list) {
5083                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5084                         hist_trigger_debug_show(m, data, n++);
5085         }
5086
5087  out_unlock:
5088         mutex_unlock(&event_mutex);
5089
5090         return ret;
5091 }
5092
5093 static int event_hist_debug_open(struct inode *inode, struct file *file)
5094 {
5095         int ret;
5096
5097         ret = security_locked_down(LOCKDOWN_TRACEFS);
5098         if (ret)
5099                 return ret;
5100
5101         return single_open(file, hist_debug_show, file);
5102 }
5103
5104 const struct file_operations event_hist_debug_fops = {
5105         .open = event_hist_debug_open,
5106         .read = seq_read,
5107         .llseek = seq_lseek,
5108         .release = single_release,
5109 };
5110 #endif
5111
5112 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5113 {
5114         const char *field_name = hist_field_name(hist_field, 0);
5115
5116         if (hist_field->var.name)
5117                 seq_printf(m, "%s=", hist_field->var.name);
5118
5119         if (hist_field->flags & HIST_FIELD_FL_CPU)
5120                 seq_puts(m, "common_cpu");
5121         else if (field_name) {
5122                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5123                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5124                         seq_putc(m, '$');
5125                 seq_printf(m, "%s", field_name);
5126         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5127                 seq_puts(m, "common_timestamp");
5128
5129         if (hist_field->flags) {
5130                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5131                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5132                         const char *flags = get_hist_field_flags(hist_field);
5133
5134                         if (flags)
5135                                 seq_printf(m, ".%s", flags);
5136                 }
5137         }
5138 }
5139
5140 static int event_hist_trigger_print(struct seq_file *m,
5141                                     struct event_trigger_ops *ops,
5142                                     struct event_trigger_data *data)
5143 {
5144         struct hist_trigger_data *hist_data = data->private_data;
5145         struct hist_field *field;
5146         bool have_var = false;
5147         unsigned int i;
5148
5149         seq_puts(m, "hist:");
5150
5151         if (data->name)
5152                 seq_printf(m, "%s:", data->name);
5153
5154         seq_puts(m, "keys=");
5155
5156         for_each_hist_key_field(i, hist_data) {
5157                 field = hist_data->fields[i];
5158
5159                 if (i > hist_data->n_vals)
5160                         seq_puts(m, ",");
5161
5162                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5163                         seq_puts(m, "stacktrace");
5164                 else
5165                         hist_field_print(m, field);
5166         }
5167
5168         seq_puts(m, ":vals=");
5169
5170         for_each_hist_val_field(i, hist_data) {
5171                 field = hist_data->fields[i];
5172                 if (field->flags & HIST_FIELD_FL_VAR) {
5173                         have_var = true;
5174                         continue;
5175                 }
5176
5177                 if (i == HITCOUNT_IDX)
5178                         seq_puts(m, "hitcount");
5179                 else {
5180                         seq_puts(m, ",");
5181                         hist_field_print(m, field);
5182                 }
5183         }
5184
5185         if (have_var) {
5186                 unsigned int n = 0;
5187
5188                 seq_puts(m, ":");
5189
5190                 for_each_hist_val_field(i, hist_data) {
5191                         field = hist_data->fields[i];
5192
5193                         if (field->flags & HIST_FIELD_FL_VAR) {
5194                                 if (n++)
5195                                         seq_puts(m, ",");
5196                                 hist_field_print(m, field);
5197                         }
5198                 }
5199         }
5200
5201         seq_puts(m, ":sort=");
5202
5203         for (i = 0; i < hist_data->n_sort_keys; i++) {
5204                 struct tracing_map_sort_key *sort_key;
5205                 unsigned int idx, first_key_idx;
5206
5207                 /* skip VAR vals */
5208                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5209
5210                 sort_key = &hist_data->sort_keys[i];
5211                 idx = sort_key->field_idx;
5212
5213                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5214                         return -EINVAL;
5215
5216                 if (i > 0)
5217                         seq_puts(m, ",");
5218
5219                 if (idx == HITCOUNT_IDX)
5220                         seq_puts(m, "hitcount");
5221                 else {
5222                         if (idx >= first_key_idx)
5223                                 idx += hist_data->n_vars;
5224                         hist_field_print(m, hist_data->fields[idx]);
5225                 }
5226
5227                 if (sort_key->descending)
5228                         seq_puts(m, ".descending");
5229         }
5230         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5231         if (hist_data->enable_timestamps)
5232                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5233
5234         print_actions_spec(m, hist_data);
5235
5236         if (data->filter_str)
5237                 seq_printf(m, " if %s", data->filter_str);
5238
5239         if (data->paused)
5240                 seq_puts(m, " [paused]");
5241         else
5242                 seq_puts(m, " [active]");
5243
5244         seq_putc(m, '\n');
5245
5246         return 0;
5247 }
5248
5249 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5250                                    struct event_trigger_data *data)
5251 {
5252         struct hist_trigger_data *hist_data = data->private_data;
5253
5254         if (!data->ref && hist_data->attrs->name)
5255                 save_named_trigger(hist_data->attrs->name, data);
5256
5257         data->ref++;
5258
5259         return 0;
5260 }
5261
5262 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5263 {
5264         struct trace_event_file *file;
5265         unsigned int i;
5266         char *cmd;
5267         int ret;
5268
5269         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5270                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5271                 cmd = hist_data->field_var_hists[i]->cmd;
5272                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5273                                               "!hist", "hist", cmd);
5274                 WARN_ON_ONCE(ret < 0);
5275         }
5276 }
5277
5278 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5279                                     struct event_trigger_data *data)
5280 {
5281         struct hist_trigger_data *hist_data = data->private_data;
5282
5283         if (WARN_ON_ONCE(data->ref <= 0))
5284                 return;
5285
5286         data->ref--;
5287         if (!data->ref) {
5288                 if (data->name)
5289                         del_named_trigger(data);
5290
5291                 trigger_data_free(data);
5292
5293                 remove_hist_vars(hist_data);
5294
5295                 unregister_field_var_hists(hist_data);
5296
5297                 destroy_hist_data(hist_data);
5298         }
5299 }
5300
5301 static struct event_trigger_ops event_hist_trigger_ops = {
5302         .func                   = event_hist_trigger,
5303         .print                  = event_hist_trigger_print,
5304         .init                   = event_hist_trigger_init,
5305         .free                   = event_hist_trigger_free,
5306 };
5307
5308 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5309                                          struct event_trigger_data *data)
5310 {
5311         data->ref++;
5312
5313         save_named_trigger(data->named_data->name, data);
5314
5315         event_hist_trigger_init(ops, data->named_data);
5316
5317         return 0;
5318 }
5319
5320 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5321                                           struct event_trigger_data *data)
5322 {
5323         if (WARN_ON_ONCE(data->ref <= 0))
5324                 return;
5325
5326         event_hist_trigger_free(ops, data->named_data);
5327
5328         data->ref--;
5329         if (!data->ref) {
5330                 del_named_trigger(data);
5331                 trigger_data_free(data);
5332         }
5333 }
5334
5335 static struct event_trigger_ops event_hist_trigger_named_ops = {
5336         .func                   = event_hist_trigger,
5337         .print                  = event_hist_trigger_print,
5338         .init                   = event_hist_trigger_named_init,
5339         .free                   = event_hist_trigger_named_free,
5340 };
5341
5342 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5343                                                             char *param)
5344 {
5345         return &event_hist_trigger_ops;
5346 }
5347
5348 static void hist_clear(struct event_trigger_data *data)
5349 {
5350         struct hist_trigger_data *hist_data = data->private_data;
5351
5352         if (data->name)
5353                 pause_named_trigger(data);
5354
5355         tracepoint_synchronize_unregister();
5356
5357         tracing_map_clear(hist_data->map);
5358
5359         if (data->name)
5360                 unpause_named_trigger(data);
5361 }
5362
5363 static bool compatible_field(struct ftrace_event_field *field,
5364                              struct ftrace_event_field *test_field)
5365 {
5366         if (field == test_field)
5367                 return true;
5368         if (field == NULL || test_field == NULL)
5369                 return false;
5370         if (strcmp(field->name, test_field->name) != 0)
5371                 return false;
5372         if (strcmp(field->type, test_field->type) != 0)
5373                 return false;
5374         if (field->size != test_field->size)
5375                 return false;
5376         if (field->is_signed != test_field->is_signed)
5377                 return false;
5378
5379         return true;
5380 }
5381
5382 static bool hist_trigger_match(struct event_trigger_data *data,
5383                                struct event_trigger_data *data_test,
5384                                struct event_trigger_data *named_data,
5385                                bool ignore_filter)
5386 {
5387         struct tracing_map_sort_key *sort_key, *sort_key_test;
5388         struct hist_trigger_data *hist_data, *hist_data_test;
5389         struct hist_field *key_field, *key_field_test;
5390         unsigned int i;
5391
5392         if (named_data && (named_data != data_test) &&
5393             (named_data != data_test->named_data))
5394                 return false;
5395
5396         if (!named_data && is_named_trigger(data_test))
5397                 return false;
5398
5399         hist_data = data->private_data;
5400         hist_data_test = data_test->private_data;
5401
5402         if (hist_data->n_vals != hist_data_test->n_vals ||
5403             hist_data->n_fields != hist_data_test->n_fields ||
5404             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5405                 return false;
5406
5407         if (!ignore_filter) {
5408                 if ((data->filter_str && !data_test->filter_str) ||
5409                    (!data->filter_str && data_test->filter_str))
5410                         return false;
5411         }
5412
5413         for_each_hist_field(i, hist_data) {
5414                 key_field = hist_data->fields[i];
5415                 key_field_test = hist_data_test->fields[i];
5416
5417                 if (key_field->flags != key_field_test->flags)
5418                         return false;
5419                 if (!compatible_field(key_field->field, key_field_test->field))
5420                         return false;
5421                 if (key_field->offset != key_field_test->offset)
5422                         return false;
5423                 if (key_field->size != key_field_test->size)
5424                         return false;
5425                 if (key_field->is_signed != key_field_test->is_signed)
5426                         return false;
5427                 if (!!key_field->var.name != !!key_field_test->var.name)
5428                         return false;
5429                 if (key_field->var.name &&
5430                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5431                         return false;
5432         }
5433
5434         for (i = 0; i < hist_data->n_sort_keys; i++) {
5435                 sort_key = &hist_data->sort_keys[i];
5436                 sort_key_test = &hist_data_test->sort_keys[i];
5437
5438                 if (sort_key->field_idx != sort_key_test->field_idx ||
5439                     sort_key->descending != sort_key_test->descending)
5440                         return false;
5441         }
5442
5443         if (!ignore_filter && data->filter_str &&
5444             (strcmp(data->filter_str, data_test->filter_str) != 0))
5445                 return false;
5446
5447         if (!actions_match(hist_data, hist_data_test))
5448                 return false;
5449
5450         return true;
5451 }
5452
5453 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5454                                  struct event_trigger_data *data,
5455                                  struct trace_event_file *file)
5456 {
5457         struct hist_trigger_data *hist_data = data->private_data;
5458         struct event_trigger_data *test, *named_data = NULL;
5459         struct trace_array *tr = file->tr;
5460         int ret = 0;
5461
5462         if (hist_data->attrs->name) {
5463                 named_data = find_named_trigger(hist_data->attrs->name);
5464                 if (named_data) {
5465                         if (!hist_trigger_match(data, named_data, named_data,
5466                                                 true)) {
5467                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5468                                 ret = -EINVAL;
5469                                 goto out;
5470                         }
5471                 }
5472         }
5473
5474         if (hist_data->attrs->name && !named_data)
5475                 goto new;
5476
5477         lockdep_assert_held(&event_mutex);
5478
5479         list_for_each_entry(test, &file->triggers, list) {
5480                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5481                         if (!hist_trigger_match(data, test, named_data, false))
5482                                 continue;
5483                         if (hist_data->attrs->pause)
5484                                 test->paused = true;
5485                         else if (hist_data->attrs->cont)
5486                                 test->paused = false;
5487                         else if (hist_data->attrs->clear)
5488                                 hist_clear(test);
5489                         else {
5490                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5491                                 ret = -EEXIST;
5492                         }
5493                         goto out;
5494                 }
5495         }
5496  new:
5497         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5498                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5499                 ret = -ENOENT;
5500                 goto out;
5501         }
5502
5503         if (hist_data->attrs->pause)
5504                 data->paused = true;
5505
5506         if (named_data) {
5507                 data->private_data = named_data->private_data;
5508                 set_named_trigger_data(data, named_data);
5509                 data->ops = &event_hist_trigger_named_ops;
5510         }
5511
5512         if (data->ops->init) {
5513                 ret = data->ops->init(data->ops, data);
5514                 if (ret < 0)
5515                         goto out;
5516         }
5517
5518         if (hist_data->enable_timestamps) {
5519                 char *clock = hist_data->attrs->clock;
5520
5521                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5522                 if (ret) {
5523                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5524                         goto out;
5525                 }
5526
5527                 tracing_set_filter_buffering(file->tr, true);
5528         }
5529
5530         if (named_data)
5531                 destroy_hist_data(hist_data);
5532
5533         ret++;
5534  out:
5535         return ret;
5536 }
5537
5538 static int hist_trigger_enable(struct event_trigger_data *data,
5539                                struct trace_event_file *file)
5540 {
5541         int ret = 0;
5542
5543         list_add_tail_rcu(&data->list, &file->triggers);
5544
5545         update_cond_flag(file);
5546
5547         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5548                 list_del_rcu(&data->list);
5549                 update_cond_flag(file);
5550                 ret--;
5551         }
5552
5553         return ret;
5554 }
5555
5556 static bool have_hist_trigger_match(struct event_trigger_data *data,
5557                                     struct trace_event_file *file)
5558 {
5559         struct hist_trigger_data *hist_data = data->private_data;
5560         struct event_trigger_data *test, *named_data = NULL;
5561         bool match = false;
5562
5563         lockdep_assert_held(&event_mutex);
5564
5565         if (hist_data->attrs->name)
5566                 named_data = find_named_trigger(hist_data->attrs->name);
5567
5568         list_for_each_entry(test, &file->triggers, list) {
5569                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5570                         if (hist_trigger_match(data, test, named_data, false)) {
5571                                 match = true;
5572                                 break;
5573                         }
5574                 }
5575         }
5576
5577         return match;
5578 }
5579
5580 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5581                                     struct trace_event_file *file)
5582 {
5583         struct hist_trigger_data *hist_data = data->private_data;
5584         struct event_trigger_data *test, *named_data = NULL;
5585
5586         lockdep_assert_held(&event_mutex);
5587
5588         if (hist_data->attrs->name)
5589                 named_data = find_named_trigger(hist_data->attrs->name);
5590
5591         list_for_each_entry(test, &file->triggers, list) {
5592                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5593                         if (!hist_trigger_match(data, test, named_data, false))
5594                                 continue;
5595                         hist_data = test->private_data;
5596                         if (check_var_refs(hist_data))
5597                                 return true;
5598                         break;
5599                 }
5600         }
5601
5602         return false;
5603 }
5604
5605 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5606                                     struct event_trigger_data *data,
5607                                     struct trace_event_file *file)
5608 {
5609         struct hist_trigger_data *hist_data = data->private_data;
5610         struct event_trigger_data *test, *named_data = NULL;
5611         bool unregistered = false;
5612
5613         lockdep_assert_held(&event_mutex);
5614
5615         if (hist_data->attrs->name)
5616                 named_data = find_named_trigger(hist_data->attrs->name);
5617
5618         list_for_each_entry(test, &file->triggers, list) {
5619                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5620                         if (!hist_trigger_match(data, test, named_data, false))
5621                                 continue;
5622                         unregistered = true;
5623                         list_del_rcu(&test->list);
5624                         trace_event_trigger_enable_disable(file, 0);
5625                         update_cond_flag(file);
5626                         break;
5627                 }
5628         }
5629
5630         if (unregistered && test->ops->free)
5631                 test->ops->free(test->ops, test);
5632
5633         if (hist_data->enable_timestamps) {
5634                 if (!hist_data->remove || unregistered)
5635                         tracing_set_filter_buffering(file->tr, false);
5636         }
5637 }
5638
5639 static bool hist_file_check_refs(struct trace_event_file *file)
5640 {
5641         struct hist_trigger_data *hist_data;
5642         struct event_trigger_data *test;
5643
5644         lockdep_assert_held(&event_mutex);
5645
5646         list_for_each_entry(test, &file->triggers, list) {
5647                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5648                         hist_data = test->private_data;
5649                         if (check_var_refs(hist_data))
5650                                 return true;
5651                 }
5652         }
5653
5654         return false;
5655 }
5656
5657 static void hist_unreg_all(struct trace_event_file *file)
5658 {
5659         struct event_trigger_data *test, *n;
5660         struct hist_trigger_data *hist_data;
5661         struct synth_event *se;
5662         const char *se_name;
5663
5664         lockdep_assert_held(&event_mutex);
5665
5666         if (hist_file_check_refs(file))
5667                 return;
5668
5669         list_for_each_entry_safe(test, n, &file->triggers, list) {
5670                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5671                         hist_data = test->private_data;
5672                         list_del_rcu(&test->list);
5673                         trace_event_trigger_enable_disable(file, 0);
5674
5675                         se_name = trace_event_name(file->event_call);
5676                         se = find_synth_event(se_name);
5677                         if (se)
5678                                 se->ref--;
5679
5680                         update_cond_flag(file);
5681                         if (hist_data->enable_timestamps)
5682                                 tracing_set_filter_buffering(file->tr, false);
5683                         if (test->ops->free)
5684                                 test->ops->free(test->ops, test);
5685                 }
5686         }
5687 }
5688
5689 static int event_hist_trigger_func(struct event_command *cmd_ops,
5690                                    struct trace_event_file *file,
5691                                    char *glob, char *cmd, char *param)
5692 {
5693         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5694         struct event_trigger_data *trigger_data;
5695         struct hist_trigger_attrs *attrs;
5696         struct event_trigger_ops *trigger_ops;
5697         struct hist_trigger_data *hist_data;
5698         struct synth_event *se;
5699         const char *se_name;
5700         bool remove = false;
5701         char *trigger, *p;
5702         int ret = 0;
5703
5704         lockdep_assert_held(&event_mutex);
5705
5706         if (glob && strlen(glob)) {
5707                 hist_err_clear();
5708                 last_cmd_set(file, param);
5709         }
5710
5711         if (!param)
5712                 return -EINVAL;
5713
5714         if (glob[0] == '!')
5715                 remove = true;
5716
5717         /*
5718          * separate the trigger from the filter (k:v [if filter])
5719          * allowing for whitespace in the trigger
5720          */
5721         p = trigger = param;
5722         do {
5723                 p = strstr(p, "if");
5724                 if (!p)
5725                         break;
5726                 if (p == param)
5727                         return -EINVAL;
5728                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5729                         p++;
5730                         continue;
5731                 }
5732                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5733                         return -EINVAL;
5734                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5735                         p++;
5736                         continue;
5737                 }
5738                 break;
5739         } while (p);
5740
5741         if (!p)
5742                 param = NULL;
5743         else {
5744                 *(p - 1) = '\0';
5745                 param = strstrip(p);
5746                 trigger = strstrip(trigger);
5747         }
5748
5749         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5750         if (IS_ERR(attrs))
5751                 return PTR_ERR(attrs);
5752
5753         if (attrs->map_bits)
5754                 hist_trigger_bits = attrs->map_bits;
5755
5756         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5757         if (IS_ERR(hist_data)) {
5758                 destroy_hist_trigger_attrs(attrs);
5759                 return PTR_ERR(hist_data);
5760         }
5761
5762         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5763
5764         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5765         if (!trigger_data) {
5766                 ret = -ENOMEM;
5767                 goto out_free;
5768         }
5769
5770         trigger_data->count = -1;
5771         trigger_data->ops = trigger_ops;
5772         trigger_data->cmd_ops = cmd_ops;
5773
5774         INIT_LIST_HEAD(&trigger_data->list);
5775         RCU_INIT_POINTER(trigger_data->filter, NULL);
5776
5777         trigger_data->private_data = hist_data;
5778
5779         /* if param is non-empty, it's supposed to be a filter */
5780         if (param && cmd_ops->set_filter) {
5781                 ret = cmd_ops->set_filter(param, trigger_data, file);
5782                 if (ret < 0)
5783                         goto out_free;
5784         }
5785
5786         if (remove) {
5787                 if (!have_hist_trigger_match(trigger_data, file))
5788                         goto out_free;
5789
5790                 if (hist_trigger_check_refs(trigger_data, file)) {
5791                         ret = -EBUSY;
5792                         goto out_free;
5793                 }
5794
5795                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5796                 se_name = trace_event_name(file->event_call);
5797                 se = find_synth_event(se_name);
5798                 if (se)
5799                         se->ref--;
5800                 ret = 0;
5801                 goto out_free;
5802         }
5803
5804         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5805         /*
5806          * The above returns on success the # of triggers registered,
5807          * but if it didn't register any it returns zero.  Consider no
5808          * triggers registered a failure too.
5809          */
5810         if (!ret) {
5811                 if (!(attrs->pause || attrs->cont || attrs->clear))
5812                         ret = -ENOENT;
5813                 goto out_free;
5814         } else if (ret < 0)
5815                 goto out_free;
5816
5817         if (get_named_trigger_data(trigger_data))
5818                 goto enable;
5819
5820         if (has_hist_vars(hist_data))
5821                 save_hist_vars(hist_data);
5822
5823         ret = create_actions(hist_data);
5824         if (ret)
5825                 goto out_unreg;
5826
5827         ret = tracing_map_init(hist_data->map);
5828         if (ret)
5829                 goto out_unreg;
5830 enable:
5831         ret = hist_trigger_enable(trigger_data, file);
5832         if (ret)
5833                 goto out_unreg;
5834
5835         se_name = trace_event_name(file->event_call);
5836         se = find_synth_event(se_name);
5837         if (se)
5838                 se->ref++;
5839         /* Just return zero, not the number of registered triggers */
5840         ret = 0;
5841  out:
5842         if (ret == 0)
5843                 hist_err_clear();
5844
5845         return ret;
5846  out_unreg:
5847         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5848  out_free:
5849         if (cmd_ops->set_filter)
5850                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5851
5852         remove_hist_vars(hist_data);
5853
5854         kfree(trigger_data);
5855
5856         destroy_hist_data(hist_data);
5857         goto out;
5858 }
5859
5860 static struct event_command trigger_hist_cmd = {
5861         .name                   = "hist",
5862         .trigger_type           = ETT_EVENT_HIST,
5863         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5864         .func                   = event_hist_trigger_func,
5865         .reg                    = hist_register_trigger,
5866         .unreg                  = hist_unregister_trigger,
5867         .unreg_all              = hist_unreg_all,
5868         .get_trigger_ops        = event_hist_get_trigger_ops,
5869         .set_filter             = set_trigger_filter,
5870 };
5871
5872 __init int register_trigger_hist_cmd(void)
5873 {
5874         int ret;
5875
5876         ret = register_event_command(&trigger_hist_cmd);
5877         WARN_ON(ret < 0);
5878
5879         return ret;
5880 }
5881
5882 static void
5883 hist_enable_trigger(struct event_trigger_data *data,
5884                     struct trace_buffer *buffer,  void *rec,
5885                     struct ring_buffer_event *event)
5886 {
5887         struct enable_trigger_data *enable_data = data->private_data;
5888         struct event_trigger_data *test;
5889
5890         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5891                                 lockdep_is_held(&event_mutex)) {
5892                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5893                         if (enable_data->enable)
5894                                 test->paused = false;
5895                         else
5896                                 test->paused = true;
5897                 }
5898         }
5899 }
5900
5901 static void
5902 hist_enable_count_trigger(struct event_trigger_data *data,
5903                           struct trace_buffer *buffer,  void *rec,
5904                           struct ring_buffer_event *event)
5905 {
5906         if (!data->count)
5907                 return;
5908
5909         if (data->count != -1)
5910                 (data->count)--;
5911
5912         hist_enable_trigger(data, buffer, rec, event);
5913 }
5914
5915 static struct event_trigger_ops hist_enable_trigger_ops = {
5916         .func                   = hist_enable_trigger,
5917         .print                  = event_enable_trigger_print,
5918         .init                   = event_trigger_init,
5919         .free                   = event_enable_trigger_free,
5920 };
5921
5922 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5923         .func                   = hist_enable_count_trigger,
5924         .print                  = event_enable_trigger_print,
5925         .init                   = event_trigger_init,
5926         .free                   = event_enable_trigger_free,
5927 };
5928
5929 static struct event_trigger_ops hist_disable_trigger_ops = {
5930         .func                   = hist_enable_trigger,
5931         .print                  = event_enable_trigger_print,
5932         .init                   = event_trigger_init,
5933         .free                   = event_enable_trigger_free,
5934 };
5935
5936 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5937         .func                   = hist_enable_count_trigger,
5938         .print                  = event_enable_trigger_print,
5939         .init                   = event_trigger_init,
5940         .free                   = event_enable_trigger_free,
5941 };
5942
5943 static struct event_trigger_ops *
5944 hist_enable_get_trigger_ops(char *cmd, char *param)
5945 {
5946         struct event_trigger_ops *ops;
5947         bool enable;
5948
5949         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5950
5951         if (enable)
5952                 ops = param ? &hist_enable_count_trigger_ops :
5953                         &hist_enable_trigger_ops;
5954         else
5955                 ops = param ? &hist_disable_count_trigger_ops :
5956                         &hist_disable_trigger_ops;
5957
5958         return ops;
5959 }
5960
5961 static void hist_enable_unreg_all(struct trace_event_file *file)
5962 {
5963         struct event_trigger_data *test, *n;
5964
5965         list_for_each_entry_safe(test, n, &file->triggers, list) {
5966                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5967                         list_del_rcu(&test->list);
5968                         update_cond_flag(file);
5969                         trace_event_trigger_enable_disable(file, 0);
5970                         if (test->ops->free)
5971                                 test->ops->free(test->ops, test);
5972                 }
5973         }
5974 }
5975
5976 static struct event_command trigger_hist_enable_cmd = {
5977         .name                   = ENABLE_HIST_STR,
5978         .trigger_type           = ETT_HIST_ENABLE,
5979         .func                   = event_enable_trigger_func,
5980         .reg                    = event_enable_register_trigger,
5981         .unreg                  = event_enable_unregister_trigger,
5982         .unreg_all              = hist_enable_unreg_all,
5983         .get_trigger_ops        = hist_enable_get_trigger_ops,
5984         .set_filter             = set_trigger_filter,
5985 };
5986
5987 static struct event_command trigger_hist_disable_cmd = {
5988         .name                   = DISABLE_HIST_STR,
5989         .trigger_type           = ETT_HIST_ENABLE,
5990         .func                   = event_enable_trigger_func,
5991         .reg                    = event_enable_register_trigger,
5992         .unreg                  = event_enable_unregister_trigger,
5993         .unreg_all              = hist_enable_unreg_all,
5994         .get_trigger_ops        = hist_enable_get_trigger_ops,
5995         .set_filter             = set_trigger_filter,
5996 };
5997
5998 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5999 {
6000         unregister_event_command(&trigger_hist_enable_cmd);
6001         unregister_event_command(&trigger_hist_disable_cmd);
6002 }
6003
6004 __init int register_trigger_hist_enable_disable_cmds(void)
6005 {
6006         int ret;
6007
6008         ret = register_event_command(&trigger_hist_enable_cmd);
6009         if (WARN_ON(ret < 0))
6010                 return ret;
6011         ret = register_event_command(&trigger_hist_disable_cmd);
6012         if (WARN_ON(ret < 0))
6013                 unregister_trigger_hist_enable_disable_cmds();
6014
6015         return ret;
6016 }