kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),
  73
  74 #undef C
  75 #define C(a, b)         HIST_ERR_##a
  76
  77 enum { ERRORS };
  78
  79 #undef C
  80 #define C(a, b)         b
  81
  82 static const char *err_text[] = { ERRORS };
  83
  84 struct hist_field;
  85
  86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  87                                 struct tracing_map_elt *elt,
  88                                 struct trace_buffer *buffer,
  89                                 struct ring_buffer_event *rbe,
  90                                 void *event);
  91
  92 #define HIST_FIELD_OPERANDS_MAX 2
  93 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  94 #define HIST_ACTIONS_MAX        8
  95 #define HIST_CONST_DIGITS_MAX   21
  96 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  97
  98 enum field_op_id {
  99         FIELD_OP_NONE,
 100         FIELD_OP_PLUS,
 101         FIELD_OP_MINUS,
 102         FIELD_OP_UNARY_MINUS,
 103         FIELD_OP_DIV,
 104         FIELD_OP_MULT,
 105 };
 106
 107 /*
 108  * A hist_var (histogram variable) contains variable information for
 109  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 110  * flag set.  A hist_var has a variable name e.g. ts0, and is
 111  * associated with a given histogram trigger, as specified by
 112  * hist_data.  The hist_var idx is the unique index assigned to the
 113  * variable by the hist trigger's tracing_map.  The idx is what is
 114  * used to set a variable's value and, by a variable reference, to
 115  * retrieve it.
 116  */
 117 struct hist_var {
 118         char                            *name;
 119         struct hist_trigger_data        *hist_data;
 120         unsigned int                    idx;
 121 };
 122
 123 struct hist_field {
 124         struct ftrace_event_field       *field;
 125         unsigned long                   flags;
 126         hist_field_fn_t                 fn;
 127         unsigned int                    ref;
 128         unsigned int                    size;
 129         unsigned int                    offset;
 130         unsigned int                    is_signed;
 131         unsigned long                   buckets;
 132         const char                      *type;
 133         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 134         struct hist_trigger_data        *hist_data;
 135
 136         /*
 137          * Variable fields contain variable-specific info in var.
 138          */
 139         struct hist_var                 var;
 140         enum field_op_id                operator;
 141         char                            *system;
 142         char                            *event_name;
 143
 144         /*
 145          * The name field is used for EXPR and VAR_REF fields.  VAR
 146          * fields contain the variable name in var.name.
 147          */
 148         char                            *name;
 149
 150         /*
 151          * When a histogram trigger is hit, if it has any references
 152          * to variables, the values of those variables are collected
 153          * into a var_ref_vals array by resolve_var_refs().  The
 154          * current value of each variable is read from the tracing_map
 155          * using the hist field's hist_var.idx and entered into the
 156          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 157          */
 158         unsigned int                    var_ref_idx;
 159         bool                            read_once;
 160
 161         unsigned int                    var_str_idx;
 162
 163         /* Numeric literals are represented as u64 */
 164         u64                             constant;
 165         /* Used to optimize division by constants */
 166         u64                             div_multiplier;
 167 };
 168
 169 static u64 hist_field_none(struct hist_field *field,
 170                            struct tracing_map_elt *elt,
 171                            struct trace_buffer *buffer,
 172                            struct ring_buffer_event *rbe,
 173                            void *event)
 174 {
 175         return 0;
 176 }
 177
 178 static u64 hist_field_const(struct hist_field *field,
 179                            struct tracing_map_elt *elt,
 180                            struct trace_buffer *buffer,
 181                            struct ring_buffer_event *rbe,
 182                            void *event)
 183 {
 184         return field->constant;
 185 }
 186
 187 static u64 hist_field_counter(struct hist_field *field,
 188                               struct tracing_map_elt *elt,
 189                               struct trace_buffer *buffer,
 190                               struct ring_buffer_event *rbe,
 191                               void *event)
 192 {
 193         return 1;
 194 }
 195
 196 static u64 hist_field_string(struct hist_field *hist_field,
 197                              struct tracing_map_elt *elt,
 198                              struct trace_buffer *buffer,
 199                              struct ring_buffer_event *rbe,
 200                              void *event)
 201 {
 202         char *addr = (char *)(event + hist_field->field->offset);
 203
 204         return (u64)(unsigned long)addr;
 205 }
 206
 207 static u64 hist_field_dynstring(struct hist_field *hist_field,
 208                                 struct tracing_map_elt *elt,
 209                                 struct trace_buffer *buffer,
 210                                 struct ring_buffer_event *rbe,
 211                                 void *event)
 212 {
 213         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 214         int str_loc = str_item & 0xffff;
 215         char *addr = (char *)(event + str_loc);
 216
 217         return (u64)(unsigned long)addr;
 218 }
 219
 220 static u64 hist_field_pstring(struct hist_field *hist_field,
 221                               struct tracing_map_elt *elt,
 222                               struct trace_buffer *buffer,
 223                               struct ring_buffer_event *rbe,
 224                               void *event)
 225 {
 226         char **addr = (char **)(event + hist_field->field->offset);
 227
 228         return (u64)(unsigned long)*addr;
 229 }
 230
 231 static u64 hist_field_log2(struct hist_field *hist_field,
 232                            struct tracing_map_elt *elt,
 233                            struct trace_buffer *buffer,
 234                            struct ring_buffer_event *rbe,
 235                            void *event)
 236 {
 237         struct hist_field *operand = hist_field->operands[0];
 238
 239         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 240
 241         return (u64) ilog2(roundup_pow_of_two(val));
 242 }
 243
 244 static u64 hist_field_bucket(struct hist_field *hist_field,
 245                              struct tracing_map_elt *elt,
 246                              struct trace_buffer *buffer,
 247                              struct ring_buffer_event *rbe,
 248                              void *event)
 249 {
 250         struct hist_field *operand = hist_field->operands[0];
 251         unsigned long buckets = hist_field->buckets;
 252
 253         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 254
 255         if (WARN_ON_ONCE(!buckets))
 256                 return val;
 257
 258         if (val >= LONG_MAX)
 259                 val = div64_ul(val, buckets);
 260         else
 261                 val = (u64)((unsigned long)val / buckets);
 262         return val * buckets;
 263 }
 264
 265 static u64 hist_field_plus(struct hist_field *hist_field,
 266                            struct tracing_map_elt *elt,
 267                            struct trace_buffer *buffer,
 268                            struct ring_buffer_event *rbe,
 269                            void *event)
 270 {
 271         struct hist_field *operand1 = hist_field->operands[0];
 272         struct hist_field *operand2 = hist_field->operands[1];
 273
 274         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 275         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 276
 277         return val1 + val2;
 278 }
 279
 280 static u64 hist_field_minus(struct hist_field *hist_field,
 281                             struct tracing_map_elt *elt,
 282                             struct trace_buffer *buffer,
 283                             struct ring_buffer_event *rbe,
 284                             void *event)
 285 {
 286         struct hist_field *operand1 = hist_field->operands[0];
 287         struct hist_field *operand2 = hist_field->operands[1];
 288
 289         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 290         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 291
 292         return val1 - val2;
 293 }
 294
 295 static u64 hist_field_div(struct hist_field *hist_field,
 296                            struct tracing_map_elt *elt,
 297                            struct trace_buffer *buffer,
 298                            struct ring_buffer_event *rbe,
 299                            void *event)
 300 {
 301         struct hist_field *operand1 = hist_field->operands[0];
 302         struct hist_field *operand2 = hist_field->operands[1];
 303
 304         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 305         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 306
 307         /* Return -1 for the undefined case */
 308         if (!val2)
 309                 return -1;
 310
 311         /* Use shift if the divisor is a power of 2 */
 312         if (!(val2 & (val2 - 1)))
 313                 return val1 >> __ffs64(val2);
 314
 315         return div64_u64(val1, val2);
 316 }
 317
 318 static u64 div_by_power_of_two(struct hist_field *hist_field,
 319                                 struct tracing_map_elt *elt,
 320                                 struct trace_buffer *buffer,
 321                                 struct ring_buffer_event *rbe,
 322                                 void *event)
 323 {
 324         struct hist_field *operand1 = hist_field->operands[0];
 325         struct hist_field *operand2 = hist_field->operands[1];
 326
 327         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 328
 329         return val1 >> __ffs64(operand2->constant);
 330 }
 331
 332 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 333                                 struct tracing_map_elt *elt,
 334                                 struct trace_buffer *buffer,
 335                                 struct ring_buffer_event *rbe,
 336                                 void *event)
 337 {
 338         struct hist_field *operand1 = hist_field->operands[0];
 339         struct hist_field *operand2 = hist_field->operands[1];
 340
 341         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 342
 343         return div64_u64(val1, operand2->constant);
 344 }
 345
 346 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 347                                 struct tracing_map_elt *elt,
 348                                 struct trace_buffer *buffer,
 349                                 struct ring_buffer_event *rbe,
 350                                 void *event)
 351 {
 352         struct hist_field *operand1 = hist_field->operands[0];
 353         struct hist_field *operand2 = hist_field->operands[1];
 354
 355         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 356
 357         /*
 358          * If the divisor is a constant, do a multiplication and shift instead.
 359          *
 360          * Choose Z = some power of 2. If Y <= Z, then:
 361          *     X / Y = (X * (Z / Y)) / Z
 362          *
 363          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 364          *     X / Y = (X * mult) / Z
 365          *
 366          * The division by Z can be replaced by a shift since Z is a power of 2:
 367          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 368          *
 369          * As long, as X < Z the results will not be off by more than 1.
 370          */
 371         if (val1 < (1 << HIST_DIV_SHIFT)) {
 372                 u64 mult = operand2->div_multiplier;
 373
 374                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 375         }
 376
 377         return div64_u64(val1, operand2->constant);
 378 }
 379
 380 static u64 hist_field_mult(struct hist_field *hist_field,
 381                            struct tracing_map_elt *elt,
 382                            struct trace_buffer *buffer,
 383                            struct ring_buffer_event *rbe,
 384                            void *event)
 385 {
 386         struct hist_field *operand1 = hist_field->operands[0];
 387         struct hist_field *operand2 = hist_field->operands[1];
 388
 389         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 390         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 391
 392         return val1 * val2;
 393 }
 394
 395 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 396                                   struct tracing_map_elt *elt,
 397                                   struct trace_buffer *buffer,
 398                                   struct ring_buffer_event *rbe,
 399                                   void *event)
 400 {
 401         struct hist_field *operand = hist_field->operands[0];
 402
 403         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 404         u64 val = (u64)-sval;
 405
 406         return val;
 407 }
 408
 409 #define DEFINE_HIST_FIELD_FN(type)                                      \
 410         static u64 hist_field_##type(struct hist_field *hist_field,     \
 411                                      struct tracing_map_elt *elt,       \
 412                                      struct trace_buffer *buffer,       \
 413                                      struct ring_buffer_event *rbe,     \
 414                                      void *event)                       \
 415 {                                                                       \
 416         type *addr = (type *)(event + hist_field->field->offset);       \
 417                                                                         \
 418         return (u64)(unsigned long)*addr;                               \
 419 }
 420
 421 DEFINE_HIST_FIELD_FN(s64);
 422 DEFINE_HIST_FIELD_FN(u64);
 423 DEFINE_HIST_FIELD_FN(s32);
 424 DEFINE_HIST_FIELD_FN(u32);
 425 DEFINE_HIST_FIELD_FN(s16);
 426 DEFINE_HIST_FIELD_FN(u16);
 427 DEFINE_HIST_FIELD_FN(s8);
 428 DEFINE_HIST_FIELD_FN(u8);
 429
 430 #define for_each_hist_field(i, hist_data)       \
 431         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 432
 433 #define for_each_hist_val_field(i, hist_data)   \
 434         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 435
 436 #define for_each_hist_key_field(i, hist_data)   \
 437         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 438
 439 #define HIST_STACKTRACE_DEPTH   16
 440 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 441 #define HIST_STACKTRACE_SKIP    5
 442
 443 #define HITCOUNT_IDX            0
 444 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 445
 446 enum hist_field_flags {
 447         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 448         HIST_FIELD_FL_KEY               = 1 << 1,
 449         HIST_FIELD_FL_STRING            = 1 << 2,
 450         HIST_FIELD_FL_HEX               = 1 << 3,
 451         HIST_FIELD_FL_SYM               = 1 << 4,
 452         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 453         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 454         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 455         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 456         HIST_FIELD_FL_LOG2              = 1 << 9,
 457         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 458         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 459         HIST_FIELD_FL_VAR               = 1 << 12,
 460         HIST_FIELD_FL_EXPR              = 1 << 13,
 461         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 462         HIST_FIELD_FL_CPU               = 1 << 15,
 463         HIST_FIELD_FL_ALIAS             = 1 << 16,
 464         HIST_FIELD_FL_BUCKET            = 1 << 17,
 465         HIST_FIELD_FL_CONST             = 1 << 18,
 466 };
 467
 468 struct var_defs {
 469         unsigned int    n_vars;
 470         char            *name[TRACING_MAP_VARS_MAX];
 471         char            *expr[TRACING_MAP_VARS_MAX];
 472 };
 473
 474 struct hist_trigger_attrs {
 475         char            *keys_str;
 476         char            *vals_str;
 477         char            *sort_key_str;
 478         char            *name;
 479         char            *clock;
 480         bool            pause;
 481         bool            cont;
 482         bool            clear;
 483         bool            ts_in_usecs;
 484         unsigned int    map_bits;
 485
 486         char            *assignment_str[TRACING_MAP_VARS_MAX];
 487         unsigned int    n_assignments;
 488
 489         char            *action_str[HIST_ACTIONS_MAX];
 490         unsigned int    n_actions;
 491
 492         struct var_defs var_defs;
 493 };
 494
 495 struct field_var {
 496         struct hist_field       *var;
 497         struct hist_field       *val;
 498 };
 499
 500 struct field_var_hist {
 501         struct hist_trigger_data        *hist_data;
 502         char                            *cmd;
 503 };
 504
 505 struct hist_trigger_data {
 506         struct hist_field               *fields[HIST_FIELDS_MAX];
 507         unsigned int                    n_vals;
 508         unsigned int                    n_keys;
 509         unsigned int                    n_fields;
 510         unsigned int                    n_vars;
 511         unsigned int                    n_var_str;
 512         unsigned int                    key_size;
 513         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 514         unsigned int                    n_sort_keys;
 515         struct trace_event_file         *event_file;
 516         struct hist_trigger_attrs       *attrs;
 517         struct tracing_map              *map;
 518         bool                            enable_timestamps;
 519         bool                            remove;
 520         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 521         unsigned int                    n_var_refs;
 522
 523         struct action_data              *actions[HIST_ACTIONS_MAX];
 524         unsigned int                    n_actions;
 525
 526         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 527         unsigned int                    n_field_vars;
 528         unsigned int                    n_field_var_str;
 529         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 530         unsigned int                    n_field_var_hists;
 531
 532         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 533         unsigned int                    n_save_vars;
 534         unsigned int                    n_save_var_str;
 535 };
 536
 537 struct action_data;
 538
 539 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 540                              struct tracing_map_elt *elt,
 541                              struct trace_buffer *buffer, void *rec,
 542                              struct ring_buffer_event *rbe, void *key,
 543                              struct action_data *data, u64 *var_ref_vals);
 544
 545 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 546
 547 enum handler_id {
 548         HANDLER_ONMATCH = 1,
 549         HANDLER_ONMAX,
 550         HANDLER_ONCHANGE,
 551 };
 552
 553 enum action_id {
 554         ACTION_SAVE = 1,
 555         ACTION_TRACE,
 556         ACTION_SNAPSHOT,
 557 };
 558
 559 struct action_data {
 560         enum handler_id         handler;
 561         enum action_id          action;
 562         char                    *action_name;
 563         action_fn_t             fn;
 564
 565         unsigned int            n_params;
 566         char                    *params[SYNTH_FIELDS_MAX];
 567
 568         /*
 569          * When a histogram trigger is hit, the values of any
 570          * references to variables, including variables being passed
 571          * as parameters to synthetic events, are collected into a
 572          * var_ref_vals array.  This var_ref_idx array is an array of
 573          * indices into the var_ref_vals array, one for each synthetic
 574          * event param, and is passed to the synthetic event
 575          * invocation.
 576          */
 577         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 578         struct synth_event      *synth_event;
 579         bool                    use_trace_keyword;
 580         char                    *synth_event_name;
 581
 582         union {
 583                 struct {
 584                         char                    *event;
 585                         char                    *event_system;
 586                 } match_data;
 587
 588                 struct {
 589                         /*
 590                          * var_str contains the $-unstripped variable
 591                          * name referenced by var_ref, and used when
 592                          * printing the action.  Because var_ref
 593                          * creation is deferred to create_actions(),
 594                          * we need a per-action way to save it until
 595                          * then, thus var_str.
 596                          */
 597                         char                    *var_str;
 598
 599                         /*
 600                          * var_ref refers to the variable being
 601                          * tracked e.g onmax($var).
 602                          */
 603                         struct hist_field       *var_ref;
 604
 605                         /*
 606                          * track_var contains the 'invisible' tracking
 607                          * variable created to keep the current
 608                          * e.g. max value.
 609                          */
 610                         struct hist_field       *track_var;
 611
 612                         check_track_val_fn_t    check_val;
 613                         action_fn_t             save_data;
 614                 } track_data;
 615         };
 616 };
 617
 618 struct track_data {
 619         u64                             track_val;
 620         bool                            updated;
 621
 622         unsigned int                    key_len;
 623         void                            *key;
 624         struct tracing_map_elt          elt;
 625
 626         struct action_data              *action_data;
 627         struct hist_trigger_data        *hist_data;
 628 };
 629
 630 struct hist_elt_data {
 631         char *comm;
 632         u64 *var_ref_vals;
 633         char **field_var_str;
 634         int n_field_var_str;
 635 };
 636
 637 struct snapshot_context {
 638         struct tracing_map_elt  *elt;
 639         void                    *key;
 640 };
 641
 642 /*
 643  * Returns the specific division function to use if the divisor
 644  * is constant. This avoids extra branches when the trigger is hit.
 645  */
 646 static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
 647 {
 648         u64 div = divisor->constant;
 649
 650         if (!(div & (div - 1)))
 651                 return div_by_power_of_two;
 652
 653         /* If the divisor is too large, do a regular division */
 654         if (div > (1 << HIST_DIV_SHIFT))
 655                 return div_by_not_power_of_two;
 656
 657         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 658         return div_by_mult_and_shift;
 659 }
 660
 661 static void track_data_free(struct track_data *track_data)
 662 {
 663         struct hist_elt_data *elt_data;
 664
 665         if (!track_data)
 666                 return;
 667
 668         kfree(track_data->key);
 669
 670         elt_data = track_data->elt.private_data;
 671         if (elt_data) {
 672                 kfree(elt_data->comm);
 673                 kfree(elt_data);
 674         }
 675
 676         kfree(track_data);
 677 }
 678
 679 static struct track_data *track_data_alloc(unsigned int key_len,
 680                                            struct action_data *action_data,
 681                                            struct hist_trigger_data *hist_data)
 682 {
 683         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 684         struct hist_elt_data *elt_data;
 685
 686         if (!data)
 687                 return ERR_PTR(-ENOMEM);
 688
 689         data->key = kzalloc(key_len, GFP_KERNEL);
 690         if (!data->key) {
 691                 track_data_free(data);
 692                 return ERR_PTR(-ENOMEM);
 693         }
 694
 695         data->key_len = key_len;
 696         data->action_data = action_data;
 697         data->hist_data = hist_data;
 698
 699         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 700         if (!elt_data) {
 701                 track_data_free(data);
 702                 return ERR_PTR(-ENOMEM);
 703         }
 704
 705         data->elt.private_data = elt_data;
 706
 707         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 708         if (!elt_data->comm) {
 709                 track_data_free(data);
 710                 return ERR_PTR(-ENOMEM);
 711         }
 712
 713         return data;
 714 }
 715
 716 static char last_cmd[MAX_FILTER_STR_VAL];
 717 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 718
 719 static int errpos(char *str)
 720 {
 721         return err_pos(last_cmd, str);
 722 }
 723
 724 static void last_cmd_set(struct trace_event_file *file, char *str)
 725 {
 726         const char *system = NULL, *name = NULL;
 727         struct trace_event_call *call;
 728
 729         if (!str)
 730                 return;
 731
 732         strcpy(last_cmd, "hist:");
 733         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 734
 735         if (file) {
 736                 call = file->event_call;
 737                 system = call->class->system;
 738                 if (system) {
 739                         name = trace_event_name(call);
 740                         if (!name)
 741                                 system = NULL;
 742                 }
 743         }
 744
 745         if (system)
 746                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 747 }
 748
 749 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 750 {
 751         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 752                         err_type, err_pos);
 753 }
 754
 755 static void hist_err_clear(void)
 756 {
 757         last_cmd[0] = '\0';
 758         last_cmd_loc[0] = '\0';
 759 }
 760
 761 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 762                                     unsigned int *var_ref_idx);
 763
 764 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 765                                unsigned int *var_ref_idx)
 766 {
 767         struct tracepoint *tp = event->tp;
 768
 769         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 770                 struct tracepoint_func *probe_func_ptr;
 771                 synth_probe_func_t probe_func;
 772                 void *__data;
 773
 774                 if (!(cpu_online(raw_smp_processor_id())))
 775                         return;
 776
 777                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 778                 if (probe_func_ptr) {
 779                         do {
 780                                 probe_func = probe_func_ptr->func;
 781                                 __data = probe_func_ptr->data;
 782                                 probe_func(__data, var_ref_vals, var_ref_idx);
 783                         } while ((++probe_func_ptr)->func);
 784                 }
 785         }
 786 }
 787
 788 static void action_trace(struct hist_trigger_data *hist_data,
 789                          struct tracing_map_elt *elt,
 790                          struct trace_buffer *buffer, void *rec,
 791                          struct ring_buffer_event *rbe, void *key,
 792                          struct action_data *data, u64 *var_ref_vals)
 793 {
 794         struct synth_event *event = data->synth_event;
 795
 796         trace_synth(event, var_ref_vals, data->var_ref_idx);
 797 }
 798
 799 struct hist_var_data {
 800         struct list_head list;
 801         struct hist_trigger_data *hist_data;
 802 };
 803
 804 static u64 hist_field_timestamp(struct hist_field *hist_field,
 805                                 struct tracing_map_elt *elt,
 806                                 struct trace_buffer *buffer,
 807                                 struct ring_buffer_event *rbe,
 808                                 void *event)
 809 {
 810         struct hist_trigger_data *hist_data = hist_field->hist_data;
 811         struct trace_array *tr = hist_data->event_file->tr;
 812
 813         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 814
 815         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 816                 ts = ns2usecs(ts);
 817
 818         return ts;
 819 }
 820
 821 static u64 hist_field_cpu(struct hist_field *hist_field,
 822                           struct tracing_map_elt *elt,
 823                           struct trace_buffer *buffer,
 824                           struct ring_buffer_event *rbe,
 825                           void *event)
 826 {
 827         int cpu = smp_processor_id();
 828
 829         return cpu;
 830 }
 831
 832 /**
 833  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 834  * @hist_field: The VAR_REF field to check
 835  * @var_data: The hist trigger that owns the variable
 836  * @var_idx: The trigger variable identifier
 837  *
 838  * Check the given VAR_REF field to see whether or not it references
 839  * the given variable associated with the given trigger.
 840  *
 841  * Return: The VAR_REF field if it does reference the variable, NULL if not
 842  */
 843 static struct hist_field *
 844 check_field_for_var_ref(struct hist_field *hist_field,
 845                         struct hist_trigger_data *var_data,
 846                         unsigned int var_idx)
 847 {
 848         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 849
 850         if (hist_field && hist_field->var.idx == var_idx &&
 851             hist_field->var.hist_data == var_data)
 852                 return hist_field;
 853
 854         return NULL;
 855 }
 856
 857 /**
 858  * find_var_ref - Check if a trigger has a reference to a trigger variable
 859  * @hist_data: The hist trigger that might have a reference to the variable
 860  * @var_data: The hist trigger that owns the variable
 861  * @var_idx: The trigger variable identifier
 862  *
 863  * Check the list of var_refs[] on the first hist trigger to see
 864  * whether any of them are references to the variable on the second
 865  * trigger.
 866  *
 867  * Return: The VAR_REF field referencing the variable if so, NULL if not
 868  */
 869 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 870                                        struct hist_trigger_data *var_data,
 871                                        unsigned int var_idx)
 872 {
 873         struct hist_field *hist_field;
 874         unsigned int i;
 875
 876         for (i = 0; i < hist_data->n_var_refs; i++) {
 877                 hist_field = hist_data->var_refs[i];
 878                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 879                         return hist_field;
 880         }
 881
 882         return NULL;
 883 }
 884
 885 /**
 886  * find_any_var_ref - Check if there is a reference to a given trigger variable
 887  * @hist_data: The hist trigger
 888  * @var_idx: The trigger variable identifier
 889  *
 890  * Check to see whether the given variable is currently referenced by
 891  * any other trigger.
 892  *
 893  * The trigger the variable is defined on is explicitly excluded - the
 894  * assumption being that a self-reference doesn't prevent a trigger
 895  * from being removed.
 896  *
 897  * Return: The VAR_REF field referencing the variable if so, NULL if not
 898  */
 899 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 900                                            unsigned int var_idx)
 901 {
 902         struct trace_array *tr = hist_data->event_file->tr;
 903         struct hist_field *found = NULL;
 904         struct hist_var_data *var_data;
 905
 906         list_for_each_entry(var_data, &tr->hist_vars, list) {
 907                 if (var_data->hist_data == hist_data)
 908                         continue;
 909                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 910                 if (found)
 911                         break;
 912         }
 913
 914         return found;
 915 }
 916
 917 /**
 918  * check_var_refs - Check if there is a reference to any of trigger's variables
 919  * @hist_data: The hist trigger
 920  *
 921  * A trigger can define one or more variables.  If any one of them is
 922  * currently referenced by any other trigger, this function will
 923  * determine that.
 924
 925  * Typically used to determine whether or not a trigger can be removed
 926  * - if there are any references to a trigger's variables, it cannot.
 927  *
 928  * Return: True if there is a reference to any of trigger's variables
 929  */
 930 static bool check_var_refs(struct hist_trigger_data *hist_data)
 931 {
 932         struct hist_field *field;
 933         bool found = false;
 934         int i;
 935
 936         for_each_hist_field(i, hist_data) {
 937                 field = hist_data->fields[i];
 938                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 939                         if (find_any_var_ref(hist_data, field->var.idx)) {
 940                                 found = true;
 941                                 break;
 942                         }
 943                 }
 944         }
 945
 946         return found;
 947 }
 948
 949 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 950 {
 951         struct trace_array *tr = hist_data->event_file->tr;
 952         struct hist_var_data *var_data, *found = NULL;
 953
 954         list_for_each_entry(var_data, &tr->hist_vars, list) {
 955                 if (var_data->hist_data == hist_data) {
 956                         found = var_data;
 957                         break;
 958                 }
 959         }
 960
 961         return found;
 962 }
 963
 964 static bool field_has_hist_vars(struct hist_field *hist_field,
 965                                 unsigned int level)
 966 {
 967         int i;
 968
 969         if (level > 3)
 970                 return false;
 971
 972         if (!hist_field)
 973                 return false;
 974
 975         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 976             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 977                 return true;
 978
 979         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 980                 struct hist_field *operand;
 981
 982                 operand = hist_field->operands[i];
 983                 if (field_has_hist_vars(operand, level + 1))
 984                         return true;
 985         }
 986
 987         return false;
 988 }
 989
 990 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 991 {
 992         struct hist_field *hist_field;
 993         int i;
 994
 995         for_each_hist_field(i, hist_data) {
 996                 hist_field = hist_data->fields[i];
 997                 if (field_has_hist_vars(hist_field, 0))
 998                         return true;
 999         }
1000
1001         return false;
1002 }
1003
1004 static int save_hist_vars(struct hist_trigger_data *hist_data)
1005 {
1006         struct trace_array *tr = hist_data->event_file->tr;
1007         struct hist_var_data *var_data;
1008
1009         var_data = find_hist_vars(hist_data);
1010         if (var_data)
1011                 return 0;
1012
1013         if (tracing_check_open_get_tr(tr))
1014                 return -ENODEV;
1015
1016         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1017         if (!var_data) {
1018                 trace_array_put(tr);
1019                 return -ENOMEM;
1020         }
1021
1022         var_data->hist_data = hist_data;
1023         list_add(&var_data->list, &tr->hist_vars);
1024
1025         return 0;
1026 }
1027
1028 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1029 {
1030         struct trace_array *tr = hist_data->event_file->tr;
1031         struct hist_var_data *var_data;
1032
1033         var_data = find_hist_vars(hist_data);
1034         if (!var_data)
1035                 return;
1036
1037         if (WARN_ON(check_var_refs(hist_data)))
1038                 return;
1039
1040         list_del(&var_data->list);
1041
1042         kfree(var_data);
1043
1044         trace_array_put(tr);
1045 }
1046
1047 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1048                                          const char *var_name)
1049 {
1050         struct hist_field *hist_field, *found = NULL;
1051         int i;
1052
1053         for_each_hist_field(i, hist_data) {
1054                 hist_field = hist_data->fields[i];
1055                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1056                     strcmp(hist_field->var.name, var_name) == 0) {
1057                         found = hist_field;
1058                         break;
1059                 }
1060         }
1061
1062         return found;
1063 }
1064
1065 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1066                                    struct trace_event_file *file,
1067                                    const char *var_name)
1068 {
1069         struct hist_trigger_data *test_data;
1070         struct event_trigger_data *test;
1071         struct hist_field *hist_field;
1072
1073         lockdep_assert_held(&event_mutex);
1074
1075         hist_field = find_var_field(hist_data, var_name);
1076         if (hist_field)
1077                 return hist_field;
1078
1079         list_for_each_entry(test, &file->triggers, list) {
1080                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1081                         test_data = test->private_data;
1082                         hist_field = find_var_field(test_data, var_name);
1083                         if (hist_field)
1084                                 return hist_field;
1085                 }
1086         }
1087
1088         return NULL;
1089 }
1090
1091 static struct trace_event_file *find_var_file(struct trace_array *tr,
1092                                               char *system,
1093                                               char *event_name,
1094                                               char *var_name)
1095 {
1096         struct hist_trigger_data *var_hist_data;
1097         struct hist_var_data *var_data;
1098         struct trace_event_file *file, *found = NULL;
1099
1100         if (system)
1101                 return find_event_file(tr, system, event_name);
1102
1103         list_for_each_entry(var_data, &tr->hist_vars, list) {
1104                 var_hist_data = var_data->hist_data;
1105                 file = var_hist_data->event_file;
1106                 if (file == found)
1107                         continue;
1108
1109                 if (find_var_field(var_hist_data, var_name)) {
1110                         if (found) {
1111                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1112                                 return NULL;
1113                         }
1114
1115                         found = file;
1116                 }
1117         }
1118
1119         return found;
1120 }
1121
1122 static struct hist_field *find_file_var(struct trace_event_file *file,
1123                                         const char *var_name)
1124 {
1125         struct hist_trigger_data *test_data;
1126         struct event_trigger_data *test;
1127         struct hist_field *hist_field;
1128
1129         lockdep_assert_held(&event_mutex);
1130
1131         list_for_each_entry(test, &file->triggers, list) {
1132                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1133                         test_data = test->private_data;
1134                         hist_field = find_var_field(test_data, var_name);
1135                         if (hist_field)
1136                                 return hist_field;
1137                 }
1138         }
1139
1140         return NULL;
1141 }
1142
1143 static struct hist_field *
1144 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1145 {
1146         struct trace_array *tr = hist_data->event_file->tr;
1147         struct hist_field *hist_field, *found = NULL;
1148         struct trace_event_file *file;
1149         unsigned int i;
1150
1151         for (i = 0; i < hist_data->n_actions; i++) {
1152                 struct action_data *data = hist_data->actions[i];
1153
1154                 if (data->handler == HANDLER_ONMATCH) {
1155                         char *system = data->match_data.event_system;
1156                         char *event_name = data->match_data.event;
1157
1158                         file = find_var_file(tr, system, event_name, var_name);
1159                         if (!file)
1160                                 continue;
1161                         hist_field = find_file_var(file, var_name);
1162                         if (hist_field) {
1163                                 if (found) {
1164                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1165                                                  errpos(var_name));
1166                                         return ERR_PTR(-EINVAL);
1167                                 }
1168
1169                                 found = hist_field;
1170                         }
1171                 }
1172         }
1173         return found;
1174 }
1175
1176 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1177                                          char *system,
1178                                          char *event_name,
1179                                          char *var_name)
1180 {
1181         struct trace_array *tr = hist_data->event_file->tr;
1182         struct hist_field *hist_field = NULL;
1183         struct trace_event_file *file;
1184
1185         if (!system || !event_name) {
1186                 hist_field = find_match_var(hist_data, var_name);
1187                 if (IS_ERR(hist_field))
1188                         return NULL;
1189                 if (hist_field)
1190                         return hist_field;
1191         }
1192
1193         file = find_var_file(tr, system, event_name, var_name);
1194         if (!file)
1195                 return NULL;
1196
1197         hist_field = find_file_var(file, var_name);
1198
1199         return hist_field;
1200 }
1201
1202 static u64 hist_field_var_ref(struct hist_field *hist_field,
1203                               struct tracing_map_elt *elt,
1204                               struct trace_buffer *buffer,
1205                               struct ring_buffer_event *rbe,
1206                               void *event)
1207 {
1208         struct hist_elt_data *elt_data;
1209         u64 var_val = 0;
1210
1211         if (WARN_ON_ONCE(!elt))
1212                 return var_val;
1213
1214         elt_data = elt->private_data;
1215         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1216
1217         return var_val;
1218 }
1219
1220 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1221                              u64 *var_ref_vals, bool self)
1222 {
1223         struct hist_trigger_data *var_data;
1224         struct tracing_map_elt *var_elt;
1225         struct hist_field *hist_field;
1226         unsigned int i, var_idx;
1227         bool resolved = true;
1228         u64 var_val = 0;
1229
1230         for (i = 0; i < hist_data->n_var_refs; i++) {
1231                 hist_field = hist_data->var_refs[i];
1232                 var_idx = hist_field->var.idx;
1233                 var_data = hist_field->var.hist_data;
1234
1235                 if (var_data == NULL) {
1236                         resolved = false;
1237                         break;
1238                 }
1239
1240                 if ((self && var_data != hist_data) ||
1241                     (!self && var_data == hist_data))
1242                         continue;
1243
1244                 var_elt = tracing_map_lookup(var_data->map, key);
1245                 if (!var_elt) {
1246                         resolved = false;
1247                         break;
1248                 }
1249
1250                 if (!tracing_map_var_set(var_elt, var_idx)) {
1251                         resolved = false;
1252                         break;
1253                 }
1254
1255                 if (self || !hist_field->read_once)
1256                         var_val = tracing_map_read_var(var_elt, var_idx);
1257                 else
1258                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1259
1260                 var_ref_vals[i] = var_val;
1261         }
1262
1263         return resolved;
1264 }
1265
1266 static const char *hist_field_name(struct hist_field *field,
1267                                    unsigned int level)
1268 {
1269         const char *field_name = "";
1270
1271         if (level > 1)
1272                 return field_name;
1273
1274         if (field->field)
1275                 field_name = field->field->name;
1276         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1277                  field->flags & HIST_FIELD_FL_ALIAS ||
1278                  field->flags & HIST_FIELD_FL_BUCKET)
1279                 field_name = hist_field_name(field->operands[0], ++level);
1280         else if (field->flags & HIST_FIELD_FL_CPU)
1281                 field_name = "common_cpu";
1282         else if (field->flags & HIST_FIELD_FL_EXPR ||
1283                  field->flags & HIST_FIELD_FL_VAR_REF) {
1284                 if (field->system) {
1285                         static char full_name[MAX_FILTER_STR_VAL];
1286
1287                         strcat(full_name, field->system);
1288                         strcat(full_name, ".");
1289                         strcat(full_name, field->event_name);
1290                         strcat(full_name, ".");
1291                         strcat(full_name, field->name);
1292                         field_name = full_name;
1293                 } else
1294                         field_name = field->name;
1295         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1296                 field_name = "common_timestamp";
1297
1298         if (field_name == NULL)
1299                 field_name = "";
1300
1301         return field_name;
1302 }
1303
1304 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1305 {
1306         hist_field_fn_t fn = NULL;
1307
1308         switch (field_size) {
1309         case 8:
1310                 if (field_is_signed)
1311                         fn = hist_field_s64;
1312                 else
1313                         fn = hist_field_u64;
1314                 break;
1315         case 4:
1316                 if (field_is_signed)
1317                         fn = hist_field_s32;
1318                 else
1319                         fn = hist_field_u32;
1320                 break;
1321         case 2:
1322                 if (field_is_signed)
1323                         fn = hist_field_s16;
1324                 else
1325                         fn = hist_field_u16;
1326                 break;
1327         case 1:
1328                 if (field_is_signed)
1329                         fn = hist_field_s8;
1330                 else
1331                         fn = hist_field_u8;
1332                 break;
1333         }
1334
1335         return fn;
1336 }
1337
1338 static int parse_map_size(char *str)
1339 {
1340         unsigned long size, map_bits;
1341         int ret;
1342
1343         ret = kstrtoul(str, 0, &size);
1344         if (ret)
1345                 goto out;
1346
1347         map_bits = ilog2(roundup_pow_of_two(size));
1348         if (map_bits < TRACING_MAP_BITS_MIN ||
1349             map_bits > TRACING_MAP_BITS_MAX)
1350                 ret = -EINVAL;
1351         else
1352                 ret = map_bits;
1353  out:
1354         return ret;
1355 }
1356
1357 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1358 {
1359         unsigned int i;
1360
1361         if (!attrs)
1362                 return;
1363
1364         for (i = 0; i < attrs->n_assignments; i++)
1365                 kfree(attrs->assignment_str[i]);
1366
1367         for (i = 0; i < attrs->n_actions; i++)
1368                 kfree(attrs->action_str[i]);
1369
1370         kfree(attrs->name);
1371         kfree(attrs->sort_key_str);
1372         kfree(attrs->keys_str);
1373         kfree(attrs->vals_str);
1374         kfree(attrs->clock);
1375         kfree(attrs);
1376 }
1377
1378 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1379 {
1380         int ret = -EINVAL;
1381
1382         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1383                 return ret;
1384
1385         if ((str_has_prefix(str, "onmatch(")) ||
1386             (str_has_prefix(str, "onmax(")) ||
1387             (str_has_prefix(str, "onchange("))) {
1388                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1389                 if (!attrs->action_str[attrs->n_actions]) {
1390                         ret = -ENOMEM;
1391                         return ret;
1392                 }
1393                 attrs->n_actions++;
1394                 ret = 0;
1395         }
1396         return ret;
1397 }
1398
1399 static int parse_assignment(struct trace_array *tr,
1400                             char *str, struct hist_trigger_attrs *attrs)
1401 {
1402         int len, ret = 0;
1403
1404         if ((len = str_has_prefix(str, "key=")) ||
1405             (len = str_has_prefix(str, "keys="))) {
1406                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1407                 if (!attrs->keys_str) {
1408                         ret = -ENOMEM;
1409                         goto out;
1410                 }
1411         } else if ((len = str_has_prefix(str, "val=")) ||
1412                    (len = str_has_prefix(str, "vals=")) ||
1413                    (len = str_has_prefix(str, "values="))) {
1414                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1415                 if (!attrs->vals_str) {
1416                         ret = -ENOMEM;
1417                         goto out;
1418                 }
1419         } else if ((len = str_has_prefix(str, "sort="))) {
1420                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1421                 if (!attrs->sort_key_str) {
1422                         ret = -ENOMEM;
1423                         goto out;
1424                 }
1425         } else if (str_has_prefix(str, "name=")) {
1426                 attrs->name = kstrdup(str, GFP_KERNEL);
1427                 if (!attrs->name) {
1428                         ret = -ENOMEM;
1429                         goto out;
1430                 }
1431         } else if ((len = str_has_prefix(str, "clock="))) {
1432                 str += len;
1433
1434                 str = strstrip(str);
1435                 attrs->clock = kstrdup(str, GFP_KERNEL);
1436                 if (!attrs->clock) {
1437                         ret = -ENOMEM;
1438                         goto out;
1439                 }
1440         } else if ((len = str_has_prefix(str, "size="))) {
1441                 int map_bits = parse_map_size(str + len);
1442
1443                 if (map_bits < 0) {
1444                         ret = map_bits;
1445                         goto out;
1446                 }
1447                 attrs->map_bits = map_bits;
1448         } else {
1449                 char *assignment;
1450
1451                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1452                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1453                         ret = -EINVAL;
1454                         goto out;
1455                 }
1456
1457                 assignment = kstrdup(str, GFP_KERNEL);
1458                 if (!assignment) {
1459                         ret = -ENOMEM;
1460                         goto out;
1461                 }
1462
1463                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1464         }
1465  out:
1466         return ret;
1467 }
1468
1469 static struct hist_trigger_attrs *
1470 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1471 {
1472         struct hist_trigger_attrs *attrs;
1473         int ret = 0;
1474
1475         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1476         if (!attrs)
1477                 return ERR_PTR(-ENOMEM);
1478
1479         while (trigger_str) {
1480                 char *str = strsep(&trigger_str, ":");
1481                 char *rhs;
1482
1483                 rhs = strchr(str, '=');
1484                 if (rhs) {
1485                         if (!strlen(++rhs)) {
1486                                 ret = -EINVAL;
1487                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1488                                 goto free;
1489                         }
1490                         ret = parse_assignment(tr, str, attrs);
1491                         if (ret)
1492                                 goto free;
1493                 } else if (strcmp(str, "pause") == 0)
1494                         attrs->pause = true;
1495                 else if ((strcmp(str, "cont") == 0) ||
1496                          (strcmp(str, "continue") == 0))
1497                         attrs->cont = true;
1498                 else if (strcmp(str, "clear") == 0)
1499                         attrs->clear = true;
1500                 else {
1501                         ret = parse_action(str, attrs);
1502                         if (ret)
1503                                 goto free;
1504                 }
1505         }
1506
1507         if (!attrs->keys_str) {
1508                 ret = -EINVAL;
1509                 goto free;
1510         }
1511
1512         if (!attrs->clock) {
1513                 attrs->clock = kstrdup("global", GFP_KERNEL);
1514                 if (!attrs->clock) {
1515                         ret = -ENOMEM;
1516                         goto free;
1517                 }
1518         }
1519
1520         return attrs;
1521  free:
1522         destroy_hist_trigger_attrs(attrs);
1523
1524         return ERR_PTR(ret);
1525 }
1526
1527 static inline void save_comm(char *comm, struct task_struct *task)
1528 {
1529         if (!task->pid) {
1530                 strcpy(comm, "<idle>");
1531                 return;
1532         }
1533
1534         if (WARN_ON_ONCE(task->pid < 0)) {
1535                 strcpy(comm, "<XXX>");
1536                 return;
1537         }
1538
1539         strncpy(comm, task->comm, TASK_COMM_LEN);
1540 }
1541
1542 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1543 {
1544         unsigned int i;
1545
1546         for (i = 0; i < elt_data->n_field_var_str; i++)
1547                 kfree(elt_data->field_var_str[i]);
1548
1549         kfree(elt_data->field_var_str);
1550
1551         kfree(elt_data->comm);
1552         kfree(elt_data);
1553 }
1554
1555 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1556 {
1557         struct hist_elt_data *elt_data = elt->private_data;
1558
1559         hist_elt_data_free(elt_data);
1560 }
1561
1562 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1563 {
1564         struct hist_trigger_data *hist_data = elt->map->private_data;
1565         unsigned int size = TASK_COMM_LEN;
1566         struct hist_elt_data *elt_data;
1567         struct hist_field *hist_field;
1568         unsigned int i, n_str;
1569
1570         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1571         if (!elt_data)
1572                 return -ENOMEM;
1573
1574         for_each_hist_field(i, hist_data) {
1575                 hist_field = hist_data->fields[i];
1576
1577                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1578                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1579                         if (!elt_data->comm) {
1580                                 kfree(elt_data);
1581                                 return -ENOMEM;
1582                         }
1583                         break;
1584                 }
1585         }
1586
1587         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1588                 hist_data->n_var_str;
1589         if (n_str > SYNTH_FIELDS_MAX) {
1590                 hist_elt_data_free(elt_data);
1591                 return -EINVAL;
1592         }
1593
1594         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1595
1596         size = STR_VAR_LEN_MAX;
1597
1598         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1599         if (!elt_data->field_var_str) {
1600                 hist_elt_data_free(elt_data);
1601                 return -EINVAL;
1602         }
1603         elt_data->n_field_var_str = n_str;
1604
1605         for (i = 0; i < n_str; i++) {
1606                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1607                 if (!elt_data->field_var_str[i]) {
1608                         hist_elt_data_free(elt_data);
1609                         return -ENOMEM;
1610                 }
1611         }
1612
1613         elt->private_data = elt_data;
1614
1615         return 0;
1616 }
1617
1618 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1619 {
1620         struct hist_elt_data *elt_data = elt->private_data;
1621
1622         if (elt_data->comm)
1623                 save_comm(elt_data->comm, current);
1624 }
1625
1626 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1627         .elt_alloc      = hist_trigger_elt_data_alloc,
1628         .elt_free       = hist_trigger_elt_data_free,
1629         .elt_init       = hist_trigger_elt_data_init,
1630 };
1631
1632 static const char *get_hist_field_flags(struct hist_field *hist_field)
1633 {
1634         const char *flags_str = NULL;
1635
1636         if (hist_field->flags & HIST_FIELD_FL_HEX)
1637                 flags_str = "hex";
1638         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1639                 flags_str = "sym";
1640         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1641                 flags_str = "sym-offset";
1642         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1643                 flags_str = "execname";
1644         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1645                 flags_str = "syscall";
1646         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1647                 flags_str = "log2";
1648         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1649                 flags_str = "buckets";
1650         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1651                 flags_str = "usecs";
1652
1653         return flags_str;
1654 }
1655
1656 static void expr_field_str(struct hist_field *field, char *expr)
1657 {
1658         if (field->flags & HIST_FIELD_FL_VAR_REF)
1659                 strcat(expr, "$");
1660         else if (field->flags & HIST_FIELD_FL_CONST) {
1661                 char str[HIST_CONST_DIGITS_MAX];
1662
1663                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1664                 strcat(expr, str);
1665         }
1666
1667         strcat(expr, hist_field_name(field, 0));
1668
1669         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1670                 const char *flags_str = get_hist_field_flags(field);
1671
1672                 if (flags_str) {
1673                         strcat(expr, ".");
1674                         strcat(expr, flags_str);
1675                 }
1676         }
1677 }
1678
1679 static char *expr_str(struct hist_field *field, unsigned int level)
1680 {
1681         char *expr;
1682
1683         if (level > 1)
1684                 return NULL;
1685
1686         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1687         if (!expr)
1688                 return NULL;
1689
1690         if (!field->operands[0]) {
1691                 expr_field_str(field, expr);
1692                 return expr;
1693         }
1694
1695         if (field->operator == FIELD_OP_UNARY_MINUS) {
1696                 char *subexpr;
1697
1698                 strcat(expr, "-(");
1699                 subexpr = expr_str(field->operands[0], ++level);
1700                 if (!subexpr) {
1701                         kfree(expr);
1702                         return NULL;
1703                 }
1704                 strcat(expr, subexpr);
1705                 strcat(expr, ")");
1706
1707                 kfree(subexpr);
1708
1709                 return expr;
1710         }
1711
1712         expr_field_str(field->operands[0], expr);
1713
1714         switch (field->operator) {
1715         case FIELD_OP_MINUS:
1716                 strcat(expr, "-");
1717                 break;
1718         case FIELD_OP_PLUS:
1719                 strcat(expr, "+");
1720                 break;
1721         case FIELD_OP_DIV:
1722                 strcat(expr, "/");
1723                 break;
1724         case FIELD_OP_MULT:
1725                 strcat(expr, "*");
1726                 break;
1727         default:
1728                 kfree(expr);
1729                 return NULL;
1730         }
1731
1732         expr_field_str(field->operands[1], expr);
1733
1734         return expr;
1735 }
1736
1737 /*
1738  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1739  * of the expression tree to be evaluated.
1740  */
1741 static int contains_operator(char *str, char **sep)
1742 {
1743         enum field_op_id field_op = FIELD_OP_NONE;
1744         char *minus_op, *plus_op, *div_op, *mult_op;
1745
1746
1747         /*
1748          * Report the last occurrence of the operators first, so that the
1749          * expression is evaluated left to right. This is important since
1750          * subtraction and division are not associative.
1751          *
1752          *      e.g
1753          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1754          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1755          */
1756
1757         /*
1758          * First, find lower precedence addition and subtraction
1759          * since the expression will be evaluated recursively.
1760          */
1761         minus_op = strrchr(str, '-');
1762         if (minus_op) {
1763                 /*
1764                  * Unary minus is not supported in sub-expressions. If
1765                  * present, it is always the next root operator.
1766                  */
1767                 if (minus_op == str) {
1768                         field_op = FIELD_OP_UNARY_MINUS;
1769                         goto out;
1770                 }
1771
1772                 field_op = FIELD_OP_MINUS;
1773         }
1774
1775         plus_op = strrchr(str, '+');
1776         if (plus_op || minus_op) {
1777                 /*
1778                  * For operators of the same precedence use to rightmost as the
1779                  * root, so that the expression is evaluated left to right.
1780                  */
1781                 if (plus_op > minus_op)
1782                         field_op = FIELD_OP_PLUS;
1783                 goto out;
1784         }
1785
1786         /*
1787          * Multiplication and division have higher precedence than addition and
1788          * subtraction.
1789          */
1790         div_op = strrchr(str, '/');
1791         if (div_op)
1792                 field_op = FIELD_OP_DIV;
1793
1794         mult_op = strrchr(str, '*');
1795         /*
1796          * For operators of the same precedence use to rightmost as the
1797          * root, so that the expression is evaluated left to right.
1798          */
1799         if (mult_op > div_op)
1800                 field_op = FIELD_OP_MULT;
1801
1802 out:
1803         if (sep) {
1804                 switch (field_op) {
1805                 case FIELD_OP_UNARY_MINUS:
1806                 case FIELD_OP_MINUS:
1807                         *sep = minus_op;
1808                         break;
1809                 case FIELD_OP_PLUS:
1810                         *sep = plus_op;
1811                         break;
1812                 case FIELD_OP_DIV:
1813                         *sep = div_op;
1814                         break;
1815                 case FIELD_OP_MULT:
1816                         *sep = mult_op;
1817                         break;
1818                 case FIELD_OP_NONE:
1819                 default:
1820                         *sep = NULL;
1821                         break;
1822                 }
1823         }
1824
1825         return field_op;
1826 }
1827
1828 static void get_hist_field(struct hist_field *hist_field)
1829 {
1830         hist_field->ref++;
1831 }
1832
1833 static void __destroy_hist_field(struct hist_field *hist_field)
1834 {
1835         if (--hist_field->ref > 1)
1836                 return;
1837
1838         kfree(hist_field->var.name);
1839         kfree(hist_field->name);
1840
1841         /* Can likely be a const */
1842         kfree_const(hist_field->type);
1843
1844         kfree(hist_field->system);
1845         kfree(hist_field->event_name);
1846
1847         kfree(hist_field);
1848 }
1849
1850 static void destroy_hist_field(struct hist_field *hist_field,
1851                                unsigned int level)
1852 {
1853         unsigned int i;
1854
1855         if (level > 3)
1856                 return;
1857
1858         if (!hist_field)
1859                 return;
1860
1861         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1862                 return; /* var refs will be destroyed separately */
1863
1864         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1865                 destroy_hist_field(hist_field->operands[i], level + 1);
1866
1867         __destroy_hist_field(hist_field);
1868 }
1869
1870 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1871                                             struct ftrace_event_field *field,
1872                                             unsigned long flags,
1873                                             char *var_name)
1874 {
1875         struct hist_field *hist_field;
1876
1877         if (field && is_function_field(field))
1878                 return NULL;
1879
1880         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1881         if (!hist_field)
1882                 return NULL;
1883
1884         hist_field->ref = 1;
1885
1886         hist_field->hist_data = hist_data;
1887
1888         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1889                 goto out; /* caller will populate */
1890
1891         if (flags & HIST_FIELD_FL_VAR_REF) {
1892                 hist_field->fn = hist_field_var_ref;
1893                 goto out;
1894         }
1895
1896         if (flags & HIST_FIELD_FL_HITCOUNT) {
1897                 hist_field->fn = hist_field_counter;
1898                 hist_field->size = sizeof(u64);
1899                 hist_field->type = "u64";
1900                 goto out;
1901         }
1902
1903         if (flags & HIST_FIELD_FL_CONST) {
1904                 hist_field->fn = hist_field_const;
1905                 hist_field->size = sizeof(u64);
1906                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1907                 if (!hist_field->type)
1908                         goto free;
1909                 goto out;
1910         }
1911
1912         if (flags & HIST_FIELD_FL_STACKTRACE) {
1913                 hist_field->fn = hist_field_none;
1914                 goto out;
1915         }
1916
1917         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1918                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1919                 hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1920                         hist_field_bucket;
1921                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1922                 hist_field->size = hist_field->operands[0]->size;
1923                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1924                 if (!hist_field->type)
1925                         goto free;
1926                 goto out;
1927         }
1928
1929         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1930                 hist_field->fn = hist_field_timestamp;
1931                 hist_field->size = sizeof(u64);
1932                 hist_field->type = "u64";
1933                 goto out;
1934         }
1935
1936         if (flags & HIST_FIELD_FL_CPU) {
1937                 hist_field->fn = hist_field_cpu;
1938                 hist_field->size = sizeof(int);
1939                 hist_field->type = "unsigned int";
1940                 goto out;
1941         }
1942
1943         if (WARN_ON_ONCE(!field))
1944                 goto out;
1945
1946         /* Pointers to strings are just pointers and dangerous to dereference */
1947         if (is_string_field(field) &&
1948             (field->filter_type != FILTER_PTR_STRING)) {
1949                 flags |= HIST_FIELD_FL_STRING;
1950
1951                 hist_field->size = MAX_FILTER_STR_VAL;
1952                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1953                 if (!hist_field->type)
1954                         goto free;
1955
1956                 if (field->filter_type == FILTER_STATIC_STRING)
1957                         hist_field->fn = hist_field_string;
1958                 else if (field->filter_type == FILTER_DYN_STRING)
1959                         hist_field->fn = hist_field_dynstring;
1960                 else
1961                         hist_field->fn = hist_field_pstring;
1962         } else {
1963                 hist_field->size = field->size;
1964                 hist_field->is_signed = field->is_signed;
1965                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1966                 if (!hist_field->type)
1967                         goto free;
1968
1969                 hist_field->fn = select_value_fn(field->size,
1970                                                  field->is_signed);
1971                 if (!hist_field->fn) {
1972                         destroy_hist_field(hist_field, 0);
1973                         return NULL;
1974                 }
1975         }
1976  out:
1977         hist_field->field = field;
1978         hist_field->flags = flags;
1979
1980         if (var_name) {
1981                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1982                 if (!hist_field->var.name)
1983                         goto free;
1984         }
1985
1986         return hist_field;
1987  free:
1988         destroy_hist_field(hist_field, 0);
1989         return NULL;
1990 }
1991
1992 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1993 {
1994         unsigned int i;
1995
1996         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1997                 if (hist_data->fields[i]) {
1998                         destroy_hist_field(hist_data->fields[i], 0);
1999                         hist_data->fields[i] = NULL;
2000                 }
2001         }
2002
2003         for (i = 0; i < hist_data->n_var_refs; i++) {
2004                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2005                 __destroy_hist_field(hist_data->var_refs[i]);
2006                 hist_data->var_refs[i] = NULL;
2007         }
2008 }
2009
2010 static int init_var_ref(struct hist_field *ref_field,
2011                         struct hist_field *var_field,
2012                         char *system, char *event_name)
2013 {
2014         int err = 0;
2015
2016         ref_field->var.idx = var_field->var.idx;
2017         ref_field->var.hist_data = var_field->hist_data;
2018         ref_field->size = var_field->size;
2019         ref_field->is_signed = var_field->is_signed;
2020         ref_field->flags |= var_field->flags &
2021                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2022
2023         if (system) {
2024                 ref_field->system = kstrdup(system, GFP_KERNEL);
2025                 if (!ref_field->system)
2026                         return -ENOMEM;
2027         }
2028
2029         if (event_name) {
2030                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2031                 if (!ref_field->event_name) {
2032                         err = -ENOMEM;
2033                         goto free;
2034                 }
2035         }
2036
2037         if (var_field->var.name) {
2038                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2039                 if (!ref_field->name) {
2040                         err = -ENOMEM;
2041                         goto free;
2042                 }
2043         } else if (var_field->name) {
2044                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2045                 if (!ref_field->name) {
2046                         err = -ENOMEM;
2047                         goto free;
2048                 }
2049         }
2050
2051         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2052         if (!ref_field->type) {
2053                 err = -ENOMEM;
2054                 goto free;
2055         }
2056  out:
2057         return err;
2058  free:
2059         kfree(ref_field->system);
2060         kfree(ref_field->event_name);
2061         kfree(ref_field->name);
2062
2063         goto out;
2064 }
2065
2066 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2067                             struct hist_field *var_field)
2068 {
2069         struct hist_field *ref_field;
2070         int i;
2071
2072         for (i = 0; i < hist_data->n_var_refs; i++) {
2073                 ref_field = hist_data->var_refs[i];
2074                 if (ref_field->var.idx == var_field->var.idx &&
2075                     ref_field->var.hist_data == var_field->hist_data)
2076                         return i;
2077         }
2078
2079         return -ENOENT;
2080 }
2081
2082 /**
2083  * create_var_ref - Create a variable reference and attach it to trigger
2084  * @hist_data: The trigger that will be referencing the variable
2085  * @var_field: The VAR field to create a reference to
2086  * @system: The optional system string
2087  * @event_name: The optional event_name string
2088  *
2089  * Given a variable hist_field, create a VAR_REF hist_field that
2090  * represents a reference to it.
2091  *
2092  * This function also adds the reference to the trigger that
2093  * now references the variable.
2094  *
2095  * Return: The VAR_REF field if successful, NULL if not
2096  */
2097 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2098                                          struct hist_field *var_field,
2099                                          char *system, char *event_name)
2100 {
2101         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2102         struct hist_field *ref_field;
2103         int i;
2104
2105         /* Check if the variable already exists */
2106         for (i = 0; i < hist_data->n_var_refs; i++) {
2107                 ref_field = hist_data->var_refs[i];
2108                 if (ref_field->var.idx == var_field->var.idx &&
2109                     ref_field->var.hist_data == var_field->hist_data) {
2110                         get_hist_field(ref_field);
2111                         return ref_field;
2112                 }
2113         }
2114
2115         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2116         if (ref_field) {
2117                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2118                         destroy_hist_field(ref_field, 0);
2119                         return NULL;
2120                 }
2121
2122                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2123                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2124         }
2125
2126         return ref_field;
2127 }
2128
2129 static bool is_var_ref(char *var_name)
2130 {
2131         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2132                 return false;
2133
2134         return true;
2135 }
2136
2137 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2138                                  char *var_name)
2139 {
2140         char *name, *field;
2141         unsigned int i;
2142
2143         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2144                 name = hist_data->attrs->var_defs.name[i];
2145
2146                 if (strcmp(var_name, name) == 0) {
2147                         field = hist_data->attrs->var_defs.expr[i];
2148                         if (contains_operator(field, NULL) || is_var_ref(field))
2149                                 continue;
2150                         return field;
2151                 }
2152         }
2153
2154         return NULL;
2155 }
2156
2157 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2158                                  char *system, char *event_name,
2159                                  char *var_name)
2160 {
2161         struct trace_event_call *call;
2162
2163         if (system && event_name) {
2164                 call = hist_data->event_file->event_call;
2165
2166                 if (strcmp(system, call->class->system) != 0)
2167                         return NULL;
2168
2169                 if (strcmp(event_name, trace_event_name(call)) != 0)
2170                         return NULL;
2171         }
2172
2173         if (!!system != !!event_name)
2174                 return NULL;
2175
2176         if (!is_var_ref(var_name))
2177                 return NULL;
2178
2179         var_name++;
2180
2181         return field_name_from_var(hist_data, var_name);
2182 }
2183
2184 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2185                                         char *system, char *event_name,
2186                                         char *var_name)
2187 {
2188         struct hist_field *var_field = NULL, *ref_field = NULL;
2189         struct trace_array *tr = hist_data->event_file->tr;
2190
2191         if (!is_var_ref(var_name))
2192                 return NULL;
2193
2194         var_name++;
2195
2196         var_field = find_event_var(hist_data, system, event_name, var_name);
2197         if (var_field)
2198                 ref_field = create_var_ref(hist_data, var_field,
2199                                            system, event_name);
2200
2201         if (!ref_field)
2202                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2203
2204         return ref_field;
2205 }
2206
2207 static struct ftrace_event_field *
2208 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2209             char *field_str, unsigned long *flags, unsigned long *buckets)
2210 {
2211         struct ftrace_event_field *field = NULL;
2212         char *field_name, *modifier, *str;
2213         struct trace_array *tr = file->tr;
2214
2215         modifier = str = kstrdup(field_str, GFP_KERNEL);
2216         if (!modifier)
2217                 return ERR_PTR(-ENOMEM);
2218
2219         field_name = strsep(&modifier, ".");
2220         if (modifier) {
2221                 if (strcmp(modifier, "hex") == 0)
2222                         *flags |= HIST_FIELD_FL_HEX;
2223                 else if (strcmp(modifier, "sym") == 0)
2224                         *flags |= HIST_FIELD_FL_SYM;
2225                 /*
2226                  * 'sym-offset' occurrences in the trigger string are modified
2227                  * to 'symXoffset' to simplify arithmetic expression parsing.
2228                  */
2229                 else if (strcmp(modifier, "symXoffset") == 0)
2230                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2231                 else if ((strcmp(modifier, "execname") == 0) &&
2232                          (strcmp(field_name, "common_pid") == 0))
2233                         *flags |= HIST_FIELD_FL_EXECNAME;
2234                 else if (strcmp(modifier, "syscall") == 0)
2235                         *flags |= HIST_FIELD_FL_SYSCALL;
2236                 else if (strcmp(modifier, "log2") == 0)
2237                         *flags |= HIST_FIELD_FL_LOG2;
2238                 else if (strcmp(modifier, "usecs") == 0)
2239                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2240                 else if (strncmp(modifier, "bucket", 6) == 0) {
2241                         int ret;
2242
2243                         modifier += 6;
2244
2245                         if (*modifier == 's')
2246                                 modifier++;
2247                         if (*modifier != '=')
2248                                 goto error;
2249                         modifier++;
2250                         ret = kstrtoul(modifier, 0, buckets);
2251                         if (ret || !(*buckets))
2252                                 goto error;
2253                         *flags |= HIST_FIELD_FL_BUCKET;
2254                 } else {
2255  error:
2256                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2257                         field = ERR_PTR(-EINVAL);
2258                         goto out;
2259                 }
2260         }
2261
2262         if (strcmp(field_name, "common_timestamp") == 0) {
2263                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2264                 hist_data->enable_timestamps = true;
2265                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2266                         hist_data->attrs->ts_in_usecs = true;
2267         } else if (strcmp(field_name, "common_cpu") == 0)
2268                 *flags |= HIST_FIELD_FL_CPU;
2269         else {
2270                 field = trace_find_event_field(file->event_call, field_name);
2271                 if (!field || !field->size) {
2272                         /*
2273                          * For backward compatibility, if field_name
2274                          * was "cpu", then we treat this the same as
2275                          * common_cpu.
2276                          */
2277                         if (strcmp(field_name, "cpu") == 0) {
2278                                 *flags |= HIST_FIELD_FL_CPU;
2279                         } else {
2280                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2281                                          errpos(field_name));
2282                                 field = ERR_PTR(-EINVAL);
2283                                 goto out;
2284                         }
2285                 }
2286         }
2287  out:
2288         kfree(str);
2289
2290         return field;
2291 }
2292
2293 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2294                                        struct hist_field *var_ref,
2295                                        char *var_name)
2296 {
2297         struct hist_field *alias = NULL;
2298         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2299
2300         alias = create_hist_field(hist_data, NULL, flags, var_name);
2301         if (!alias)
2302                 return NULL;
2303
2304         alias->fn = var_ref->fn;
2305         alias->operands[0] = var_ref;
2306
2307         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2308                 destroy_hist_field(alias, 0);
2309                 return NULL;
2310         }
2311
2312         alias->var_ref_idx = var_ref->var_ref_idx;
2313
2314         return alias;
2315 }
2316
2317 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2318                                       char *str, char *var_name,
2319                                       unsigned long *flags)
2320 {
2321         struct trace_array *tr = hist_data->event_file->tr;
2322         struct hist_field *field = NULL;
2323         u64 constant;
2324
2325         if (kstrtoull(str, 0, &constant)) {
2326                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2327                 return NULL;
2328         }
2329
2330         *flags |= HIST_FIELD_FL_CONST;
2331         field = create_hist_field(hist_data, NULL, *flags, var_name);
2332         if (!field)
2333                 return NULL;
2334
2335         field->constant = constant;
2336
2337         return field;
2338 }
2339
2340 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2341                                      struct trace_event_file *file, char *str,
2342                                      unsigned long *flags, char *var_name)
2343 {
2344         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2345         struct ftrace_event_field *field = NULL;
2346         struct hist_field *hist_field = NULL;
2347         unsigned long buckets = 0;
2348         int ret = 0;
2349
2350         if (isdigit(str[0])) {
2351                 hist_field = parse_const(hist_data, str, var_name, flags);
2352                 if (!hist_field) {
2353                         ret = -EINVAL;
2354                         goto out;
2355                 }
2356                 return hist_field;
2357         }
2358
2359         s = strchr(str, '.');
2360         if (s) {
2361                 s = strchr(++s, '.');
2362                 if (s) {
2363                         ref_system = strsep(&str, ".");
2364                         if (!str) {
2365                                 ret = -EINVAL;
2366                                 goto out;
2367                         }
2368                         ref_event = strsep(&str, ".");
2369                         if (!str) {
2370                                 ret = -EINVAL;
2371                                 goto out;
2372                         }
2373                         ref_var = str;
2374                 }
2375         }
2376
2377         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2378         if (!s) {
2379                 hist_field = parse_var_ref(hist_data, ref_system,
2380                                            ref_event, ref_var);
2381                 if (hist_field) {
2382                         if (var_name) {
2383                                 hist_field = create_alias(hist_data, hist_field, var_name);
2384                                 if (!hist_field) {
2385                                         ret = -ENOMEM;
2386                                         goto out;
2387                                 }
2388                         }
2389                         return hist_field;
2390                 }
2391         } else
2392                 str = s;
2393
2394         field = parse_field(hist_data, file, str, flags, &buckets);
2395         if (IS_ERR(field)) {
2396                 ret = PTR_ERR(field);
2397                 goto out;
2398         }
2399
2400         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2401         if (!hist_field) {
2402                 ret = -ENOMEM;
2403                 goto out;
2404         }
2405         hist_field->buckets = buckets;
2406
2407         return hist_field;
2408  out:
2409         return ERR_PTR(ret);
2410 }
2411
2412 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2413                                      struct trace_event_file *file,
2414                                      char *str, unsigned long flags,
2415                                      char *var_name, unsigned int *n_subexprs);
2416
2417 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2418                                       struct trace_event_file *file,
2419                                       char *str, unsigned long flags,
2420                                       char *var_name, unsigned int *n_subexprs)
2421 {
2422         struct hist_field *operand1, *expr = NULL;
2423         unsigned long operand_flags;
2424         int ret = 0;
2425         char *s;
2426
2427         /* Unary minus operator, increment n_subexprs */
2428         ++*n_subexprs;
2429
2430         /* we support only -(xxx) i.e. explicit parens required */
2431
2432         if (*n_subexprs > 3) {
2433                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2434                 ret = -EINVAL;
2435                 goto free;
2436         }
2437
2438         str++; /* skip leading '-' */
2439
2440         s = strchr(str, '(');
2441         if (s)
2442                 str++;
2443         else {
2444                 ret = -EINVAL;
2445                 goto free;
2446         }
2447
2448         s = strrchr(str, ')');
2449         if (s) {
2450                  /* unary minus not supported in sub-expressions */
2451                 if (*(s+1) != '\0') {
2452                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2453                                  errpos(str));
2454                         ret = -EINVAL;
2455                         goto free;
2456                 }
2457                 *s = '\0';
2458         }
2459         else {
2460                 ret = -EINVAL; /* no closing ')' */
2461                 goto free;
2462         }
2463
2464         flags |= HIST_FIELD_FL_EXPR;
2465         expr = create_hist_field(hist_data, NULL, flags, var_name);
2466         if (!expr) {
2467                 ret = -ENOMEM;
2468                 goto free;
2469         }
2470
2471         operand_flags = 0;
2472         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2473         if (IS_ERR(operand1)) {
2474                 ret = PTR_ERR(operand1);
2475                 goto free;
2476         }
2477         if (operand1->flags & HIST_FIELD_FL_STRING) {
2478                 /* String type can not be the operand of unary operator. */
2479                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2480                 destroy_hist_field(operand1, 0);
2481                 ret = -EINVAL;
2482                 goto free;
2483         }
2484
2485         expr->flags |= operand1->flags &
2486                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2487         expr->fn = hist_field_unary_minus;
2488         expr->operands[0] = operand1;
2489         expr->operator = FIELD_OP_UNARY_MINUS;
2490         expr->name = expr_str(expr, 0);
2491         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2492         if (!expr->type) {
2493                 ret = -ENOMEM;
2494                 goto free;
2495         }
2496
2497         return expr;
2498  free:
2499         destroy_hist_field(expr, 0);
2500         return ERR_PTR(ret);
2501 }
2502
2503 /*
2504  * If the operands are var refs, return pointers the
2505  * variable(s) referenced in var1 and var2, else NULL.
2506  */
2507 static int check_expr_operands(struct trace_array *tr,
2508                                struct hist_field *operand1,
2509                                struct hist_field *operand2,
2510                                struct hist_field **var1,
2511                                struct hist_field **var2)
2512 {
2513         unsigned long operand1_flags = operand1->flags;
2514         unsigned long operand2_flags = operand2->flags;
2515
2516         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2517             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2518                 struct hist_field *var;
2519
2520                 var = find_var_field(operand1->var.hist_data, operand1->name);
2521                 if (!var)
2522                         return -EINVAL;
2523                 operand1_flags = var->flags;
2524                 *var1 = var;
2525         }
2526
2527         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2528             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2529                 struct hist_field *var;
2530
2531                 var = find_var_field(operand2->var.hist_data, operand2->name);
2532                 if (!var)
2533                         return -EINVAL;
2534                 operand2_flags = var->flags;
2535                 *var2 = var;
2536         }
2537
2538         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2539             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2540                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2541                 return -EINVAL;
2542         }
2543
2544         return 0;
2545 }
2546
2547 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2548                                      struct trace_event_file *file,
2549                                      char *str, unsigned long flags,
2550                                      char *var_name, unsigned int *n_subexprs)
2551 {
2552         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2553         struct hist_field *var1 = NULL, *var2 = NULL;
2554         unsigned long operand_flags, operand2_flags;
2555         int field_op, ret = -EINVAL;
2556         char *sep, *operand1_str;
2557         hist_field_fn_t op_fn;
2558         bool combine_consts;
2559
2560         if (*n_subexprs > 3) {
2561                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2562                 return ERR_PTR(-EINVAL);
2563         }
2564
2565         field_op = contains_operator(str, &sep);
2566
2567         if (field_op == FIELD_OP_NONE)
2568                 return parse_atom(hist_data, file, str, &flags, var_name);
2569
2570         if (field_op == FIELD_OP_UNARY_MINUS)
2571                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2572
2573         /* Binary operator found, increment n_subexprs */
2574         ++*n_subexprs;
2575
2576         /* Split the expression string at the root operator */
2577         if (!sep)
2578                 goto free;
2579         *sep = '\0';
2580         operand1_str = str;
2581         str = sep+1;
2582
2583         if (!operand1_str || !str)
2584                 goto free;
2585
2586         operand_flags = 0;
2587
2588         /* LHS of string is an expression e.g. a+b in a+b+c */
2589         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2590         if (IS_ERR(operand1)) {
2591                 ret = PTR_ERR(operand1);
2592                 operand1 = NULL;
2593                 goto free;
2594         }
2595         if (operand1->flags & HIST_FIELD_FL_STRING) {
2596                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2597                 ret = -EINVAL;
2598                 goto free;
2599         }
2600
2601         /* RHS of string is another expression e.g. c in a+b+c */
2602         operand_flags = 0;
2603         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2604         if (IS_ERR(operand2)) {
2605                 ret = PTR_ERR(operand2);
2606                 operand2 = NULL;
2607                 goto free;
2608         }
2609         if (operand2->flags & HIST_FIELD_FL_STRING) {
2610                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2611                 ret = -EINVAL;
2612                 goto free;
2613         }
2614
2615         switch (field_op) {
2616         case FIELD_OP_MINUS:
2617                 op_fn = hist_field_minus;
2618                 break;
2619         case FIELD_OP_PLUS:
2620                 op_fn = hist_field_plus;
2621                 break;
2622         case FIELD_OP_DIV:
2623                 op_fn = hist_field_div;
2624                 break;
2625         case FIELD_OP_MULT:
2626                 op_fn = hist_field_mult;
2627                 break;
2628         default:
2629                 ret = -EINVAL;
2630                 goto free;
2631         }
2632
2633         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2634         if (ret)
2635                 goto free;
2636
2637         operand_flags = var1 ? var1->flags : operand1->flags;
2638         operand2_flags = var2 ? var2->flags : operand2->flags;
2639
2640         /*
2641          * If both operands are constant, the expression can be
2642          * collapsed to a single constant.
2643          */
2644         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2645
2646         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2647
2648         flags |= operand1->flags &
2649                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2650
2651         expr = create_hist_field(hist_data, NULL, flags, var_name);
2652         if (!expr) {
2653                 ret = -ENOMEM;
2654                 goto free;
2655         }
2656
2657         operand1->read_once = true;
2658         operand2->read_once = true;
2659
2660         expr->operands[0] = operand1;
2661         expr->operands[1] = operand2;
2662
2663         if (field_op == FIELD_OP_DIV &&
2664                         operand2_flags & HIST_FIELD_FL_CONST) {
2665                 u64 divisor = var2 ? var2->constant : operand2->constant;
2666
2667                 if (!divisor) {
2668                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2669                         ret = -EDOM;
2670                         goto free;
2671                 }
2672
2673                 /*
2674                  * Copy the divisor here so we don't have to look it up
2675                  * later if this is a var ref
2676                  */
2677                 operand2->constant = divisor;
2678                 op_fn = hist_field_get_div_fn(operand2);
2679         }
2680
2681         if (combine_consts) {
2682                 if (var1)
2683                         expr->operands[0] = var1;
2684                 if (var2)
2685                         expr->operands[1] = var2;
2686
2687                 expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2688
2689                 expr->operands[0] = NULL;
2690                 expr->operands[1] = NULL;
2691
2692                 /*
2693                  * var refs won't be destroyed immediately
2694                  * See: destroy_hist_field()
2695                  */
2696                 destroy_hist_field(operand2, 0);
2697                 destroy_hist_field(operand1, 0);
2698
2699                 expr->name = expr_str(expr, 0);
2700         } else {
2701                 expr->fn = op_fn;
2702
2703                 /* The operand sizes should be the same, so just pick one */
2704                 expr->size = operand1->size;
2705
2706                 expr->operator = field_op;
2707                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2708                 if (!expr->type) {
2709                         ret = -ENOMEM;
2710                         goto free;
2711                 }
2712
2713                 expr->name = expr_str(expr, 0);
2714         }
2715
2716         return expr;
2717 free:
2718         destroy_hist_field(operand1, 0);
2719         destroy_hist_field(operand2, 0);
2720         destroy_hist_field(expr, 0);
2721
2722         return ERR_PTR(ret);
2723 }
2724
2725 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2726                                  struct trace_event_file *file)
2727 {
2728         struct event_trigger_data *test;
2729
2730         lockdep_assert_held(&event_mutex);
2731
2732         list_for_each_entry(test, &file->triggers, list) {
2733                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2734                         if (test->private_data == hist_data)
2735                                 return test->filter_str;
2736                 }
2737         }
2738
2739         return NULL;
2740 }
2741
2742 static struct event_command trigger_hist_cmd;
2743 static int event_hist_trigger_func(struct event_command *cmd_ops,
2744                                    struct trace_event_file *file,
2745                                    char *glob, char *cmd, char *param);
2746
2747 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2748                             struct hist_trigger_data *hist_data,
2749                             unsigned int n_keys)
2750 {
2751         struct hist_field *target_hist_field, *hist_field;
2752         unsigned int n, i, j;
2753
2754         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2755                 return false;
2756
2757         i = hist_data->n_vals;
2758         j = target_hist_data->n_vals;
2759
2760         for (n = 0; n < n_keys; n++) {
2761                 hist_field = hist_data->fields[i + n];
2762                 target_hist_field = target_hist_data->fields[j + n];
2763
2764                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2765                         return false;
2766                 if (hist_field->size != target_hist_field->size)
2767                         return false;
2768                 if (hist_field->is_signed != target_hist_field->is_signed)
2769                         return false;
2770         }
2771
2772         return true;
2773 }
2774
2775 static struct hist_trigger_data *
2776 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2777                      struct trace_event_file *file)
2778 {
2779         struct hist_trigger_data *hist_data;
2780         struct event_trigger_data *test;
2781         unsigned int n_keys;
2782
2783         lockdep_assert_held(&event_mutex);
2784
2785         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2786
2787         list_for_each_entry(test, &file->triggers, list) {
2788                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2789                         hist_data = test->private_data;
2790
2791                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2792                                 return hist_data;
2793                 }
2794         }
2795
2796         return NULL;
2797 }
2798
2799 static struct trace_event_file *event_file(struct trace_array *tr,
2800                                            char *system, char *event_name)
2801 {
2802         struct trace_event_file *file;
2803
2804         file = __find_event_file(tr, system, event_name);
2805         if (!file)
2806                 return ERR_PTR(-EINVAL);
2807
2808         return file;
2809 }
2810
2811 static struct hist_field *
2812 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2813                          char *system, char *event_name, char *field_name)
2814 {
2815         struct hist_field *event_var;
2816         char *synthetic_name;
2817
2818         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2819         if (!synthetic_name)
2820                 return ERR_PTR(-ENOMEM);
2821
2822         strcpy(synthetic_name, "synthetic_");
2823         strcat(synthetic_name, field_name);
2824
2825         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2826
2827         kfree(synthetic_name);
2828
2829         return event_var;
2830 }
2831
2832 /**
2833  * create_field_var_hist - Automatically create a histogram and var for a field
2834  * @target_hist_data: The target hist trigger
2835  * @subsys_name: Optional subsystem name
2836  * @event_name: Optional event name
2837  * @field_name: The name of the field (and the resulting variable)
2838  *
2839  * Hist trigger actions fetch data from variables, not directly from
2840  * events.  However, for convenience, users are allowed to directly
2841  * specify an event field in an action, which will be automatically
2842  * converted into a variable on their behalf.
2843  *
2844  * If a user specifies a field on an event that isn't the event the
2845  * histogram currently being defined (the target event histogram), the
2846  * only way that can be accomplished is if a new hist trigger is
2847  * created and the field variable defined on that.
2848  *
2849  * This function creates a new histogram compatible with the target
2850  * event (meaning a histogram with the same key as the target
2851  * histogram), and creates a variable for the specified field, but
2852  * with 'synthetic_' prepended to the variable name in order to avoid
2853  * collision with normal field variables.
2854  *
2855  * Return: The variable created for the field.
2856  */
2857 static struct hist_field *
2858 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2859                       char *subsys_name, char *event_name, char *field_name)
2860 {
2861         struct trace_array *tr = target_hist_data->event_file->tr;
2862         struct hist_trigger_data *hist_data;
2863         unsigned int i, n, first = true;
2864         struct field_var_hist *var_hist;
2865         struct trace_event_file *file;
2866         struct hist_field *key_field;
2867         struct hist_field *event_var;
2868         char *saved_filter;
2869         char *cmd;
2870         int ret;
2871
2872         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2873                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2874                 return ERR_PTR(-EINVAL);
2875         }
2876
2877         file = event_file(tr, subsys_name, event_name);
2878
2879         if (IS_ERR(file)) {
2880                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2881                 ret = PTR_ERR(file);
2882                 return ERR_PTR(ret);
2883         }
2884
2885         /*
2886          * Look for a histogram compatible with target.  We'll use the
2887          * found histogram specification to create a new matching
2888          * histogram with our variable on it.  target_hist_data is not
2889          * yet a registered histogram so we can't use that.
2890          */
2891         hist_data = find_compatible_hist(target_hist_data, file);
2892         if (!hist_data) {
2893                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2894                 return ERR_PTR(-EINVAL);
2895         }
2896
2897         /* See if a synthetic field variable has already been created */
2898         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2899                                              event_name, field_name);
2900         if (!IS_ERR_OR_NULL(event_var))
2901                 return event_var;
2902
2903         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2904         if (!var_hist)
2905                 return ERR_PTR(-ENOMEM);
2906
2907         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2908         if (!cmd) {
2909                 kfree(var_hist);
2910                 return ERR_PTR(-ENOMEM);
2911         }
2912
2913         /* Use the same keys as the compatible histogram */
2914         strcat(cmd, "keys=");
2915
2916         for_each_hist_key_field(i, hist_data) {
2917                 key_field = hist_data->fields[i];
2918                 if (!first)
2919                         strcat(cmd, ",");
2920                 strcat(cmd, key_field->field->name);
2921                 first = false;
2922         }
2923
2924         /* Create the synthetic field variable specification */
2925         strcat(cmd, ":synthetic_");
2926         strcat(cmd, field_name);
2927         strcat(cmd, "=");
2928         strcat(cmd, field_name);
2929
2930         /* Use the same filter as the compatible histogram */
2931         saved_filter = find_trigger_filter(hist_data, file);
2932         if (saved_filter) {
2933                 strcat(cmd, " if ");
2934                 strcat(cmd, saved_filter);
2935         }
2936
2937         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2938         if (!var_hist->cmd) {
2939                 kfree(cmd);
2940                 kfree(var_hist);
2941                 return ERR_PTR(-ENOMEM);
2942         }
2943
2944         /* Save the compatible histogram information */
2945         var_hist->hist_data = hist_data;
2946
2947         /* Create the new histogram with our variable */
2948         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2949                                       "", "hist", cmd);
2950         if (ret) {
2951                 kfree(cmd);
2952                 kfree(var_hist->cmd);
2953                 kfree(var_hist);
2954                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2955                 return ERR_PTR(ret);
2956         }
2957
2958         kfree(cmd);
2959
2960         /* If we can't find the variable, something went wrong */
2961         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2962                                              event_name, field_name);
2963         if (IS_ERR_OR_NULL(event_var)) {
2964                 kfree(var_hist->cmd);
2965                 kfree(var_hist);
2966                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2967                 return ERR_PTR(-EINVAL);
2968         }
2969
2970         n = target_hist_data->n_field_var_hists;
2971         target_hist_data->field_var_hists[n] = var_hist;
2972         target_hist_data->n_field_var_hists++;
2973
2974         return event_var;
2975 }
2976
2977 static struct hist_field *
2978 find_target_event_var(struct hist_trigger_data *hist_data,
2979                       char *subsys_name, char *event_name, char *var_name)
2980 {
2981         struct trace_event_file *file = hist_data->event_file;
2982         struct hist_field *hist_field = NULL;
2983
2984         if (subsys_name) {
2985                 struct trace_event_call *call;
2986
2987                 if (!event_name)
2988                         return NULL;
2989
2990                 call = file->event_call;
2991
2992                 if (strcmp(subsys_name, call->class->system) != 0)
2993                         return NULL;
2994
2995                 if (strcmp(event_name, trace_event_name(call)) != 0)
2996                         return NULL;
2997         }
2998
2999         hist_field = find_var_field(hist_data, var_name);
3000
3001         return hist_field;
3002 }
3003
3004 static inline void __update_field_vars(struct tracing_map_elt *elt,
3005                                        struct trace_buffer *buffer,
3006                                        struct ring_buffer_event *rbe,
3007                                        void *rec,
3008                                        struct field_var **field_vars,
3009                                        unsigned int n_field_vars,
3010                                        unsigned int field_var_str_start)
3011 {
3012         struct hist_elt_data *elt_data = elt->private_data;
3013         unsigned int i, j, var_idx;
3014         u64 var_val;
3015
3016         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3017                 struct field_var *field_var = field_vars[i];
3018                 struct hist_field *var = field_var->var;
3019                 struct hist_field *val = field_var->val;
3020
3021                 var_val = val->fn(val, elt, buffer, rbe, rec);
3022                 var_idx = var->var.idx;
3023
3024                 if (val->flags & HIST_FIELD_FL_STRING) {
3025                         char *str = elt_data->field_var_str[j++];
3026                         char *val_str = (char *)(uintptr_t)var_val;
3027
3028                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3029                         var_val = (u64)(uintptr_t)str;
3030                 }
3031                 tracing_map_set_var(elt, var_idx, var_val);
3032         }
3033 }
3034
3035 static void update_field_vars(struct hist_trigger_data *hist_data,
3036                               struct tracing_map_elt *elt,
3037                               struct trace_buffer *buffer,
3038                               struct ring_buffer_event *rbe,
3039                               void *rec)
3040 {
3041         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3042                             hist_data->n_field_vars, 0);
3043 }
3044
3045 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3046                                  struct tracing_map_elt *elt,
3047                                  struct trace_buffer *buffer,  void *rec,
3048                                  struct ring_buffer_event *rbe, void *key,
3049                                  struct action_data *data, u64 *var_ref_vals)
3050 {
3051         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3052                             hist_data->n_save_vars, hist_data->n_field_var_str);
3053 }
3054
3055 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3056                                      struct trace_event_file *file,
3057                                      char *name, int size, const char *type)
3058 {
3059         struct hist_field *var;
3060         int idx;
3061
3062         if (find_var(hist_data, file, name) && !hist_data->remove) {
3063                 var = ERR_PTR(-EINVAL);
3064                 goto out;
3065         }
3066
3067         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3068         if (!var) {
3069                 var = ERR_PTR(-ENOMEM);
3070                 goto out;
3071         }
3072
3073         idx = tracing_map_add_var(hist_data->map);
3074         if (idx < 0) {
3075                 kfree(var);
3076                 var = ERR_PTR(-EINVAL);
3077                 goto out;
3078         }
3079
3080         var->ref = 1;
3081         var->flags = HIST_FIELD_FL_VAR;
3082         var->var.idx = idx;
3083         var->var.hist_data = var->hist_data = hist_data;
3084         var->size = size;
3085         var->var.name = kstrdup(name, GFP_KERNEL);
3086         var->type = kstrdup_const(type, GFP_KERNEL);
3087         if (!var->var.name || !var->type) {
3088                 kfree_const(var->type);
3089                 kfree(var->var.name);
3090                 kfree(var);
3091                 var = ERR_PTR(-ENOMEM);
3092         }
3093  out:
3094         return var;
3095 }
3096
3097 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3098                                           struct trace_event_file *file,
3099                                           char *field_name)
3100 {
3101         struct hist_field *val = NULL, *var = NULL;
3102         unsigned long flags = HIST_FIELD_FL_VAR;
3103         struct trace_array *tr = file->tr;
3104         struct field_var *field_var;
3105         int ret = 0;
3106
3107         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3108                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3109                 ret = -EINVAL;
3110                 goto err;
3111         }
3112
3113         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3114         if (IS_ERR(val)) {
3115                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3116                 ret = PTR_ERR(val);
3117                 goto err;
3118         }
3119
3120         var = create_var(hist_data, file, field_name, val->size, val->type);
3121         if (IS_ERR(var)) {
3122                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3123                 kfree(val);
3124                 ret = PTR_ERR(var);
3125                 goto err;
3126         }
3127
3128         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3129         if (!field_var) {
3130                 kfree(val);
3131                 kfree(var);
3132                 ret =  -ENOMEM;
3133                 goto err;
3134         }
3135
3136         field_var->var = var;
3137         field_var->val = val;
3138  out:
3139         return field_var;
3140  err:
3141         field_var = ERR_PTR(ret);
3142         goto out;
3143 }
3144
3145 /**
3146  * create_target_field_var - Automatically create a variable for a field
3147  * @target_hist_data: The target hist trigger
3148  * @subsys_name: Optional subsystem name
3149  * @event_name: Optional event name
3150  * @var_name: The name of the field (and the resulting variable)
3151  *
3152  * Hist trigger actions fetch data from variables, not directly from
3153  * events.  However, for convenience, users are allowed to directly
3154  * specify an event field in an action, which will be automatically
3155  * converted into a variable on their behalf.
3156
3157  * This function creates a field variable with the name var_name on
3158  * the hist trigger currently being defined on the target event.  If
3159  * subsys_name and event_name are specified, this function simply
3160  * verifies that they do in fact match the target event subsystem and
3161  * event name.
3162  *
3163  * Return: The variable created for the field.
3164  */
3165 static struct field_var *
3166 create_target_field_var(struct hist_trigger_data *target_hist_data,
3167                         char *subsys_name, char *event_name, char *var_name)
3168 {
3169         struct trace_event_file *file = target_hist_data->event_file;
3170
3171         if (subsys_name) {
3172                 struct trace_event_call *call;
3173
3174                 if (!event_name)
3175                         return NULL;
3176
3177                 call = file->event_call;
3178
3179                 if (strcmp(subsys_name, call->class->system) != 0)
3180                         return NULL;
3181
3182                 if (strcmp(event_name, trace_event_name(call)) != 0)
3183                         return NULL;
3184         }
3185
3186         return create_field_var(target_hist_data, file, var_name);
3187 }
3188
3189 static bool check_track_val_max(u64 track_val, u64 var_val)
3190 {
3191         if (var_val <= track_val)
3192                 return false;
3193
3194         return true;
3195 }
3196
3197 static bool check_track_val_changed(u64 track_val, u64 var_val)
3198 {
3199         if (var_val == track_val)
3200                 return false;
3201
3202         return true;
3203 }
3204
3205 static u64 get_track_val(struct hist_trigger_data *hist_data,
3206                          struct tracing_map_elt *elt,
3207                          struct action_data *data)
3208 {
3209         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3210         u64 track_val;
3211
3212         track_val = tracing_map_read_var(elt, track_var_idx);
3213
3214         return track_val;
3215 }
3216
3217 static void save_track_val(struct hist_trigger_data *hist_data,
3218                            struct tracing_map_elt *elt,
3219                            struct action_data *data, u64 var_val)
3220 {
3221         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3222
3223         tracing_map_set_var(elt, track_var_idx, var_val);
3224 }
3225
3226 static void save_track_data(struct hist_trigger_data *hist_data,
3227                             struct tracing_map_elt *elt,
3228                             struct trace_buffer *buffer, void *rec,
3229                             struct ring_buffer_event *rbe, void *key,
3230                             struct action_data *data, u64 *var_ref_vals)
3231 {
3232         if (data->track_data.save_data)
3233                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3234                                            key, data, var_ref_vals);
3235 }
3236
3237 static bool check_track_val(struct tracing_map_elt *elt,
3238                             struct action_data *data,
3239                             u64 var_val)
3240 {
3241         struct hist_trigger_data *hist_data;
3242         u64 track_val;
3243
3244         hist_data = data->track_data.track_var->hist_data;
3245         track_val = get_track_val(hist_data, elt, data);
3246
3247         return data->track_data.check_val(track_val, var_val);
3248 }
3249
3250 #ifdef CONFIG_TRACER_SNAPSHOT
3251 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3252 {
3253         /* called with tr->max_lock held */
3254         struct track_data *track_data = tr->cond_snapshot->cond_data;
3255         struct hist_elt_data *elt_data, *track_elt_data;
3256         struct snapshot_context *context = cond_data;
3257         struct action_data *action;
3258         u64 track_val;
3259
3260         if (!track_data)
3261                 return false;
3262
3263         action = track_data->action_data;
3264
3265         track_val = get_track_val(track_data->hist_data, context->elt,
3266                                   track_data->action_data);
3267
3268         if (!action->track_data.check_val(track_data->track_val, track_val))
3269                 return false;
3270
3271         track_data->track_val = track_val;
3272         memcpy(track_data->key, context->key, track_data->key_len);
3273
3274         elt_data = context->elt->private_data;
3275         track_elt_data = track_data->elt.private_data;
3276         if (elt_data->comm)
3277                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3278
3279         track_data->updated = true;
3280
3281         return true;
3282 }
3283
3284 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3285                                      struct tracing_map_elt *elt,
3286                                      struct trace_buffer *buffer, void *rec,
3287                                      struct ring_buffer_event *rbe, void *key,
3288                                      struct action_data *data,
3289                                      u64 *var_ref_vals)
3290 {
3291         struct trace_event_file *file = hist_data->event_file;
3292         struct snapshot_context context;
3293
3294         context.elt = elt;
3295         context.key = key;
3296
3297         tracing_snapshot_cond(file->tr, &context);
3298 }
3299
3300 static void hist_trigger_print_key(struct seq_file *m,
3301                                    struct hist_trigger_data *hist_data,
3302                                    void *key,
3303                                    struct tracing_map_elt *elt);
3304
3305 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3306 {
3307         unsigned int i;
3308
3309         if (!hist_data->n_actions)
3310                 return NULL;
3311
3312         for (i = 0; i < hist_data->n_actions; i++) {
3313                 struct action_data *data = hist_data->actions[i];
3314
3315                 if (data->action == ACTION_SNAPSHOT)
3316                         return data;
3317         }
3318
3319         return NULL;
3320 }
3321
3322 static void track_data_snapshot_print(struct seq_file *m,
3323                                       struct hist_trigger_data *hist_data)
3324 {
3325         struct trace_event_file *file = hist_data->event_file;
3326         struct track_data *track_data;
3327         struct action_data *action;
3328
3329         track_data = tracing_cond_snapshot_data(file->tr);
3330         if (!track_data)
3331                 return;
3332
3333         if (!track_data->updated)
3334                 return;
3335
3336         action = snapshot_action(hist_data);
3337         if (!action)
3338                 return;
3339
3340         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3341         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3342                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3343                    action->track_data.var_str, track_data->track_val);
3344
3345         seq_puts(m, "\ttriggered by event with key: ");
3346         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3347         seq_putc(m, '\n');
3348 }
3349 #else
3350 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3351 {
3352         return false;
3353 }
3354 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3355                                      struct tracing_map_elt *elt,
3356                                      struct trace_buffer *buffer, void *rec,
3357                                      struct ring_buffer_event *rbe, void *key,
3358                                      struct action_data *data,
3359                                      u64 *var_ref_vals) {}
3360 static void track_data_snapshot_print(struct seq_file *m,
3361                                       struct hist_trigger_data *hist_data) {}
3362 #endif /* CONFIG_TRACER_SNAPSHOT */
3363
3364 static void track_data_print(struct seq_file *m,
3365                              struct hist_trigger_data *hist_data,
3366                              struct tracing_map_elt *elt,
3367                              struct action_data *data)
3368 {
3369         u64 track_val = get_track_val(hist_data, elt, data);
3370         unsigned int i, save_var_idx;
3371
3372         if (data->handler == HANDLER_ONMAX)
3373                 seq_printf(m, "\n\tmax: %10llu", track_val);
3374         else if (data->handler == HANDLER_ONCHANGE)
3375                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3376
3377         if (data->action == ACTION_SNAPSHOT)
3378                 return;
3379
3380         for (i = 0; i < hist_data->n_save_vars; i++) {
3381                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3382                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3383                 u64 val;
3384
3385                 save_var_idx = save_var->var.idx;
3386
3387                 val = tracing_map_read_var(elt, save_var_idx);
3388
3389                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3390                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3391                                    (char *)(uintptr_t)(val));
3392                 } else
3393                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3394         }
3395 }
3396
3397 static void ontrack_action(struct hist_trigger_data *hist_data,
3398                            struct tracing_map_elt *elt,
3399                            struct trace_buffer *buffer, void *rec,
3400                            struct ring_buffer_event *rbe, void *key,
3401                            struct action_data *data, u64 *var_ref_vals)
3402 {
3403         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3404
3405         if (check_track_val(elt, data, var_val)) {
3406                 save_track_val(hist_data, elt, data, var_val);
3407                 save_track_data(hist_data, elt, buffer, rec, rbe,
3408                                 key, data, var_ref_vals);
3409         }
3410 }
3411
3412 static void action_data_destroy(struct action_data *data)
3413 {
3414         unsigned int i;
3415
3416         lockdep_assert_held(&event_mutex);
3417
3418         kfree(data->action_name);
3419
3420         for (i = 0; i < data->n_params; i++)
3421                 kfree(data->params[i]);
3422
3423         if (data->synth_event)
3424                 data->synth_event->ref--;
3425
3426         kfree(data->synth_event_name);
3427
3428         kfree(data);
3429 }
3430
3431 static void track_data_destroy(struct hist_trigger_data *hist_data,
3432                                struct action_data *data)
3433 {
3434         struct trace_event_file *file = hist_data->event_file;
3435
3436         destroy_hist_field(data->track_data.track_var, 0);
3437
3438         if (data->action == ACTION_SNAPSHOT) {
3439                 struct track_data *track_data;
3440
3441                 track_data = tracing_cond_snapshot_data(file->tr);
3442                 if (track_data && track_data->hist_data == hist_data) {
3443                         tracing_snapshot_cond_disable(file->tr);
3444                         track_data_free(track_data);
3445                 }
3446         }
3447
3448         kfree(data->track_data.var_str);
3449
3450         action_data_destroy(data);
3451 }
3452
3453 static int action_create(struct hist_trigger_data *hist_data,
3454                          struct action_data *data);
3455
3456 static int track_data_create(struct hist_trigger_data *hist_data,
3457                              struct action_data *data)
3458 {
3459         struct hist_field *var_field, *ref_field, *track_var = NULL;
3460         struct trace_event_file *file = hist_data->event_file;
3461         struct trace_array *tr = file->tr;
3462         char *track_data_var_str;
3463         int ret = 0;
3464
3465         track_data_var_str = data->track_data.var_str;
3466         if (track_data_var_str[0] != '$') {
3467                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3468                 return -EINVAL;
3469         }
3470         track_data_var_str++;
3471
3472         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3473         if (!var_field) {
3474                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3475                 return -EINVAL;
3476         }
3477
3478         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3479         if (!ref_field)
3480                 return -ENOMEM;
3481
3482         data->track_data.var_ref = ref_field;
3483
3484         if (data->handler == HANDLER_ONMAX)
3485                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3486         if (IS_ERR(track_var)) {
3487                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3488                 ret = PTR_ERR(track_var);
3489                 goto out;
3490         }
3491
3492         if (data->handler == HANDLER_ONCHANGE)
3493                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3494         if (IS_ERR(track_var)) {
3495                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3496                 ret = PTR_ERR(track_var);
3497                 goto out;
3498         }
3499         data->track_data.track_var = track_var;
3500
3501         ret = action_create(hist_data, data);
3502  out:
3503         return ret;
3504 }
3505
3506 static int parse_action_params(struct trace_array *tr, char *params,
3507                                struct action_data *data)
3508 {
3509         char *param, *saved_param;
3510         bool first_param = true;
3511         int ret = 0;
3512
3513         while (params) {
3514                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3515                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3516                         goto out;
3517                 }
3518
3519                 param = strsep(&params, ",");
3520                 if (!param) {
3521                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3522                         ret = -EINVAL;
3523                         goto out;
3524                 }
3525
3526                 param = strstrip(param);
3527                 if (strlen(param) < 2) {
3528                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3529                         ret = -EINVAL;
3530                         goto out;
3531                 }
3532
3533                 saved_param = kstrdup(param, GFP_KERNEL);
3534                 if (!saved_param) {
3535                         ret = -ENOMEM;
3536                         goto out;
3537                 }
3538
3539                 if (first_param && data->use_trace_keyword) {
3540                         data->synth_event_name = saved_param;
3541                         first_param = false;
3542                         continue;
3543                 }
3544                 first_param = false;
3545
3546                 data->params[data->n_params++] = saved_param;
3547         }
3548  out:
3549         return ret;
3550 }
3551
3552 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3553                         enum handler_id handler)
3554 {
3555         char *action_name;
3556         int ret = 0;
3557
3558         strsep(&str, ".");
3559         if (!str) {
3560                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3561                 ret = -EINVAL;
3562                 goto out;
3563         }
3564
3565         action_name = strsep(&str, "(");
3566         if (!action_name || !str) {
3567                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3568                 ret = -EINVAL;
3569                 goto out;
3570         }
3571
3572         if (str_has_prefix(action_name, "save")) {
3573                 char *params = strsep(&str, ")");
3574
3575                 if (!params) {
3576                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3577                         ret = -EINVAL;
3578                         goto out;
3579                 }
3580
3581                 ret = parse_action_params(tr, params, data);
3582                 if (ret)
3583                         goto out;
3584
3585                 if (handler == HANDLER_ONMAX)
3586                         data->track_data.check_val = check_track_val_max;
3587                 else if (handler == HANDLER_ONCHANGE)
3588                         data->track_data.check_val = check_track_val_changed;
3589                 else {
3590                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3591                         ret = -EINVAL;
3592                         goto out;
3593                 }
3594
3595                 data->track_data.save_data = save_track_data_vars;
3596                 data->fn = ontrack_action;
3597                 data->action = ACTION_SAVE;
3598         } else if (str_has_prefix(action_name, "snapshot")) {
3599                 char *params = strsep(&str, ")");
3600
3601                 if (!str) {
3602                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3603                         ret = -EINVAL;
3604                         goto out;
3605                 }
3606
3607                 if (handler == HANDLER_ONMAX)
3608                         data->track_data.check_val = check_track_val_max;
3609                 else if (handler == HANDLER_ONCHANGE)
3610                         data->track_data.check_val = check_track_val_changed;
3611                 else {
3612                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3613                         ret = -EINVAL;
3614                         goto out;
3615                 }
3616
3617                 data->track_data.save_data = save_track_data_snapshot;
3618                 data->fn = ontrack_action;
3619                 data->action = ACTION_SNAPSHOT;
3620         } else {
3621                 char *params = strsep(&str, ")");
3622
3623                 if (str_has_prefix(action_name, "trace"))
3624                         data->use_trace_keyword = true;
3625
3626                 if (params) {
3627                         ret = parse_action_params(tr, params, data);
3628                         if (ret)
3629                                 goto out;
3630                 }
3631
3632                 if (handler == HANDLER_ONMAX)
3633                         data->track_data.check_val = check_track_val_max;
3634                 else if (handler == HANDLER_ONCHANGE)
3635                         data->track_data.check_val = check_track_val_changed;
3636
3637                 if (handler != HANDLER_ONMATCH) {
3638                         data->track_data.save_data = action_trace;
3639                         data->fn = ontrack_action;
3640                 } else
3641                         data->fn = action_trace;
3642
3643                 data->action = ACTION_TRACE;
3644         }
3645
3646         data->action_name = kstrdup(action_name, GFP_KERNEL);
3647         if (!data->action_name) {
3648                 ret = -ENOMEM;
3649                 goto out;
3650         }
3651
3652         data->handler = handler;
3653  out:
3654         return ret;
3655 }
3656
3657 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3658                                             char *str, enum handler_id handler)
3659 {
3660         struct action_data *data;
3661         int ret = -EINVAL;
3662         char *var_str;
3663
3664         data = kzalloc(sizeof(*data), GFP_KERNEL);
3665         if (!data)
3666                 return ERR_PTR(-ENOMEM);
3667
3668         var_str = strsep(&str, ")");
3669         if (!var_str || !str) {
3670                 ret = -EINVAL;
3671                 goto free;
3672         }
3673
3674         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3675         if (!data->track_data.var_str) {
3676                 ret = -ENOMEM;
3677                 goto free;
3678         }
3679
3680         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3681         if (ret)
3682                 goto free;
3683  out:
3684         return data;
3685  free:
3686         track_data_destroy(hist_data, data);
3687         data = ERR_PTR(ret);
3688         goto out;
3689 }
3690
3691 static void onmatch_destroy(struct action_data *data)
3692 {
3693         kfree(data->match_data.event);
3694         kfree(data->match_data.event_system);
3695
3696         action_data_destroy(data);
3697 }
3698
3699 static void destroy_field_var(struct field_var *field_var)
3700 {
3701         if (!field_var)
3702                 return;
3703
3704         destroy_hist_field(field_var->var, 0);
3705         destroy_hist_field(field_var->val, 0);
3706
3707         kfree(field_var);
3708 }
3709
3710 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3711 {
3712         unsigned int i;
3713
3714         for (i = 0; i < hist_data->n_field_vars; i++)
3715                 destroy_field_var(hist_data->field_vars[i]);
3716
3717         for (i = 0; i < hist_data->n_save_vars; i++)
3718                 destroy_field_var(hist_data->save_vars[i]);
3719 }
3720
3721 static void save_field_var(struct hist_trigger_data *hist_data,
3722                            struct field_var *field_var)
3723 {
3724         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3725
3726         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3727                 hist_data->n_field_var_str++;
3728 }
3729
3730
3731 static int check_synth_field(struct synth_event *event,
3732                              struct hist_field *hist_field,
3733                              unsigned int field_pos)
3734 {
3735         struct synth_field *field;
3736
3737         if (field_pos >= event->n_fields)
3738                 return -EINVAL;
3739
3740         field = event->fields[field_pos];
3741
3742         /*
3743          * A dynamic string synth field can accept static or
3744          * dynamic. A static string synth field can only accept a
3745          * same-sized static string, which is checked for later.
3746          */
3747         if (strstr(hist_field->type, "char[") && field->is_string
3748             && field->is_dynamic)
3749                 return 0;
3750
3751         if (strcmp(field->type, hist_field->type) != 0) {
3752                 if (field->size != hist_field->size ||
3753                     field->is_signed != hist_field->is_signed)
3754                         return -EINVAL;
3755         }
3756
3757         return 0;
3758 }
3759
3760 static struct hist_field *
3761 trace_action_find_var(struct hist_trigger_data *hist_data,
3762                       struct action_data *data,
3763                       char *system, char *event, char *var)
3764 {
3765         struct trace_array *tr = hist_data->event_file->tr;
3766         struct hist_field *hist_field;
3767
3768         var++; /* skip '$' */
3769
3770         hist_field = find_target_event_var(hist_data, system, event, var);
3771         if (!hist_field) {
3772                 if (!system && data->handler == HANDLER_ONMATCH) {
3773                         system = data->match_data.event_system;
3774                         event = data->match_data.event;
3775                 }
3776
3777                 hist_field = find_event_var(hist_data, system, event, var);
3778         }
3779
3780         if (!hist_field)
3781                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3782
3783         return hist_field;
3784 }
3785
3786 static struct hist_field *
3787 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3788                               struct action_data *data, char *system,
3789                               char *event, char *var)
3790 {
3791         struct hist_field *hist_field = NULL;
3792         struct field_var *field_var;
3793
3794         /*
3795          * First try to create a field var on the target event (the
3796          * currently being defined).  This will create a variable for
3797          * unqualified fields on the target event, or if qualified,
3798          * target fields that have qualified names matching the target.
3799          */
3800         field_var = create_target_field_var(hist_data, system, event, var);
3801
3802         if (field_var && !IS_ERR(field_var)) {
3803                 save_field_var(hist_data, field_var);
3804                 hist_field = field_var->var;
3805         } else {
3806                 field_var = NULL;
3807                 /*
3808                  * If no explicit system.event is specified, default to
3809                  * looking for fields on the onmatch(system.event.xxx)
3810                  * event.
3811                  */
3812                 if (!system && data->handler == HANDLER_ONMATCH) {
3813                         system = data->match_data.event_system;
3814                         event = data->match_data.event;
3815                 }
3816
3817                 if (!event)
3818                         goto free;
3819                 /*
3820                  * At this point, we're looking at a field on another
3821                  * event.  Because we can't modify a hist trigger on
3822                  * another event to add a variable for a field, we need
3823                  * to create a new trigger on that event and create the
3824                  * variable at the same time.
3825                  */
3826                 hist_field = create_field_var_hist(hist_data, system, event, var);
3827                 if (IS_ERR(hist_field))
3828                         goto free;
3829         }
3830  out:
3831         return hist_field;
3832  free:
3833         destroy_field_var(field_var);
3834         hist_field = NULL;
3835         goto out;
3836 }
3837
3838 static int trace_action_create(struct hist_trigger_data *hist_data,
3839                                struct action_data *data)
3840 {
3841         struct trace_array *tr = hist_data->event_file->tr;
3842         char *event_name, *param, *system = NULL;
3843         struct hist_field *hist_field, *var_ref;
3844         unsigned int i;
3845         unsigned int field_pos = 0;
3846         struct synth_event *event;
3847         char *synth_event_name;
3848         int var_ref_idx, ret = 0;
3849
3850         lockdep_assert_held(&event_mutex);
3851
3852         if (data->use_trace_keyword)
3853                 synth_event_name = data->synth_event_name;
3854         else
3855                 synth_event_name = data->action_name;
3856
3857         event = find_synth_event(synth_event_name);
3858         if (!event) {
3859                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3860                 return -EINVAL;
3861         }
3862
3863         event->ref++;
3864
3865         for (i = 0; i < data->n_params; i++) {
3866                 char *p;
3867
3868                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3869                 if (!param) {
3870                         ret = -ENOMEM;
3871                         goto err;
3872                 }
3873
3874                 system = strsep(&param, ".");
3875                 if (!param) {
3876                         param = (char *)system;
3877                         system = event_name = NULL;
3878                 } else {
3879                         event_name = strsep(&param, ".");
3880                         if (!param) {
3881                                 kfree(p);
3882                                 ret = -EINVAL;
3883                                 goto err;
3884                         }
3885                 }
3886
3887                 if (param[0] == '$')
3888                         hist_field = trace_action_find_var(hist_data, data,
3889                                                            system, event_name,
3890                                                            param);
3891                 else
3892                         hist_field = trace_action_create_field_var(hist_data,
3893                                                                    data,
3894                                                                    system,
3895                                                                    event_name,
3896                                                                    param);
3897
3898                 if (!hist_field) {
3899                         kfree(p);
3900                         ret = -EINVAL;
3901                         goto err;
3902                 }
3903
3904                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3905                         var_ref = create_var_ref(hist_data, hist_field,
3906                                                  system, event_name);
3907                         if (!var_ref) {
3908                                 kfree(p);
3909                                 ret = -ENOMEM;
3910                                 goto err;
3911                         }
3912
3913                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3914                         if (WARN_ON(var_ref_idx < 0)) {
3915                                 ret = var_ref_idx;
3916                                 goto err;
3917                         }
3918
3919                         data->var_ref_idx[i] = var_ref_idx;
3920
3921                         field_pos++;
3922                         kfree(p);
3923                         continue;
3924                 }
3925
3926                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3927                 kfree(p);
3928                 ret = -EINVAL;
3929                 goto err;
3930         }
3931
3932         if (field_pos != event->n_fields) {
3933                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3934                 ret = -EINVAL;
3935                 goto err;
3936         }
3937
3938         data->synth_event = event;
3939  out:
3940         return ret;
3941  err:
3942         event->ref--;
3943
3944         goto out;
3945 }
3946
3947 static int action_create(struct hist_trigger_data *hist_data,
3948                          struct action_data *data)
3949 {
3950         struct trace_event_file *file = hist_data->event_file;
3951         struct trace_array *tr = file->tr;
3952         struct track_data *track_data;
3953         struct field_var *field_var;
3954         unsigned int i;
3955         char *param;
3956         int ret = 0;
3957
3958         if (data->action == ACTION_TRACE)
3959                 return trace_action_create(hist_data, data);
3960
3961         if (data->action == ACTION_SNAPSHOT) {
3962                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3963                 if (IS_ERR(track_data)) {
3964                         ret = PTR_ERR(track_data);
3965                         goto out;
3966                 }
3967
3968                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3969                                                    cond_snapshot_update);
3970                 if (ret)
3971                         track_data_free(track_data);
3972
3973                 goto out;
3974         }
3975
3976         if (data->action == ACTION_SAVE) {
3977                 if (hist_data->n_save_vars) {
3978                         ret = -EEXIST;
3979                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3980                         goto out;
3981                 }
3982
3983                 for (i = 0; i < data->n_params; i++) {
3984                         param = kstrdup(data->params[i], GFP_KERNEL);
3985                         if (!param) {
3986                                 ret = -ENOMEM;
3987                                 goto out;
3988                         }
3989
3990                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3991                         if (IS_ERR(field_var)) {
3992                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3993                                          errpos(param));
3994                                 ret = PTR_ERR(field_var);
3995                                 kfree(param);
3996                                 goto out;
3997                         }
3998
3999                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4000                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4001                                 hist_data->n_save_var_str++;
4002                         kfree(param);
4003                 }
4004         }
4005  out:
4006         return ret;
4007 }
4008
4009 static int onmatch_create(struct hist_trigger_data *hist_data,
4010                           struct action_data *data)
4011 {
4012         return action_create(hist_data, data);
4013 }
4014
4015 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4016 {
4017         char *match_event, *match_event_system;
4018         struct action_data *data;
4019         int ret = -EINVAL;
4020
4021         data = kzalloc(sizeof(*data), GFP_KERNEL);
4022         if (!data)
4023                 return ERR_PTR(-ENOMEM);
4024
4025         match_event = strsep(&str, ")");
4026         if (!match_event || !str) {
4027                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4028                 goto free;
4029         }
4030
4031         match_event_system = strsep(&match_event, ".");
4032         if (!match_event) {
4033                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4034                 goto free;
4035         }
4036
4037         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4038                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4039                 goto free;
4040         }
4041
4042         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4043         if (!data->match_data.event) {
4044                 ret = -ENOMEM;
4045                 goto free;
4046         }
4047
4048         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4049         if (!data->match_data.event_system) {
4050                 ret = -ENOMEM;
4051                 goto free;
4052         }
4053
4054         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4055         if (ret)
4056                 goto free;
4057  out:
4058         return data;
4059  free:
4060         onmatch_destroy(data);
4061         data = ERR_PTR(ret);
4062         goto out;
4063 }
4064
4065 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4066 {
4067         hist_data->fields[HITCOUNT_IDX] =
4068                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4069         if (!hist_data->fields[HITCOUNT_IDX])
4070                 return -ENOMEM;
4071
4072         hist_data->n_vals++;
4073         hist_data->n_fields++;
4074
4075         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4076                 return -EINVAL;
4077
4078         return 0;
4079 }
4080
4081 static int __create_val_field(struct hist_trigger_data *hist_data,
4082                               unsigned int val_idx,
4083                               struct trace_event_file *file,
4084                               char *var_name, char *field_str,
4085                               unsigned long flags)
4086 {
4087         struct hist_field *hist_field;
4088         int ret = 0, n_subexprs = 0;
4089
4090         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4091         if (IS_ERR(hist_field)) {
4092                 ret = PTR_ERR(hist_field);
4093                 goto out;
4094         }
4095
4096         hist_data->fields[val_idx] = hist_field;
4097
4098         ++hist_data->n_vals;
4099         ++hist_data->n_fields;
4100
4101         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4102                 ret = -EINVAL;
4103  out:
4104         return ret;
4105 }
4106
4107 static int create_val_field(struct hist_trigger_data *hist_data,
4108                             unsigned int val_idx,
4109                             struct trace_event_file *file,
4110                             char *field_str)
4111 {
4112         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4113                 return -EINVAL;
4114
4115         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4116 }
4117
4118 static const char *no_comm = "(no comm)";
4119
4120 static u64 hist_field_execname(struct hist_field *hist_field,
4121                                struct tracing_map_elt *elt,
4122                                struct trace_buffer *buffer,
4123                                struct ring_buffer_event *rbe,
4124                                void *event)
4125 {
4126         struct hist_elt_data *elt_data;
4127
4128         if (WARN_ON_ONCE(!elt))
4129                 return (u64)(unsigned long)no_comm;
4130
4131         elt_data = elt->private_data;
4132
4133         if (WARN_ON_ONCE(!elt_data->comm))
4134                 return (u64)(unsigned long)no_comm;
4135
4136         return (u64)(unsigned long)(elt_data->comm);
4137 }
4138
4139 /* Convert a var that points to common_pid.execname to a string */
4140 static void update_var_execname(struct hist_field *hist_field)
4141 {
4142         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4143                 HIST_FIELD_FL_EXECNAME;
4144         hist_field->size = MAX_FILTER_STR_VAL;
4145         hist_field->is_signed = 0;
4146
4147         kfree_const(hist_field->type);
4148         hist_field->type = "char[]";
4149
4150         hist_field->fn = hist_field_execname;
4151 }
4152
4153 static int create_var_field(struct hist_trigger_data *hist_data,
4154                             unsigned int val_idx,
4155                             struct trace_event_file *file,
4156                             char *var_name, char *expr_str)
4157 {
4158         struct trace_array *tr = hist_data->event_file->tr;
4159         unsigned long flags = 0;
4160         int ret;
4161
4162         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4163                 return -EINVAL;
4164
4165         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4166                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4167                 return -EINVAL;
4168         }
4169
4170         flags |= HIST_FIELD_FL_VAR;
4171         hist_data->n_vars++;
4172         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4173                 return -EINVAL;
4174
4175         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4176
4177         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4178                 update_var_execname(hist_data->fields[val_idx]);
4179
4180         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4181                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4182
4183         return ret;
4184 }
4185
4186 static int create_val_fields(struct hist_trigger_data *hist_data,
4187                              struct trace_event_file *file)
4188 {
4189         char *fields_str, *field_str;
4190         unsigned int i, j = 1;
4191         int ret;
4192
4193         ret = create_hitcount_val(hist_data);
4194         if (ret)
4195                 goto out;
4196
4197         fields_str = hist_data->attrs->vals_str;
4198         if (!fields_str)
4199                 goto out;
4200
4201         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4202                      j < TRACING_MAP_VALS_MAX; i++) {
4203                 field_str = strsep(&fields_str, ",");
4204                 if (!field_str)
4205                         break;
4206
4207                 if (strcmp(field_str, "hitcount") == 0)
4208                         continue;
4209
4210                 ret = create_val_field(hist_data, j++, file, field_str);
4211                 if (ret)
4212                         goto out;
4213         }
4214
4215         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4216                 ret = -EINVAL;
4217  out:
4218         return ret;
4219 }
4220
4221 static int create_key_field(struct hist_trigger_data *hist_data,
4222                             unsigned int key_idx,
4223                             unsigned int key_offset,
4224                             struct trace_event_file *file,
4225                             char *field_str)
4226 {
4227         struct trace_array *tr = hist_data->event_file->tr;
4228         struct hist_field *hist_field = NULL;
4229         unsigned long flags = 0;
4230         unsigned int key_size;
4231         int ret = 0, n_subexprs = 0;
4232
4233         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4234                 return -EINVAL;
4235
4236         flags |= HIST_FIELD_FL_KEY;
4237
4238         if (strcmp(field_str, "stacktrace") == 0) {
4239                 flags |= HIST_FIELD_FL_STACKTRACE;
4240                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4241                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4242         } else {
4243                 hist_field = parse_expr(hist_data, file, field_str, flags,
4244                                         NULL, &n_subexprs);
4245                 if (IS_ERR(hist_field)) {
4246                         ret = PTR_ERR(hist_field);
4247                         goto out;
4248                 }
4249
4250                 if (field_has_hist_vars(hist_field, 0)) {
4251                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4252                         destroy_hist_field(hist_field, 0);
4253                         ret = -EINVAL;
4254                         goto out;
4255                 }
4256
4257                 key_size = hist_field->size;
4258         }
4259
4260         hist_data->fields[key_idx] = hist_field;
4261
4262         key_size = ALIGN(key_size, sizeof(u64));
4263         hist_data->fields[key_idx]->size = key_size;
4264         hist_data->fields[key_idx]->offset = key_offset;
4265
4266         hist_data->key_size += key_size;
4267
4268         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4269                 ret = -EINVAL;
4270                 goto out;
4271         }
4272
4273         hist_data->n_keys++;
4274         hist_data->n_fields++;
4275
4276         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4277                 return -EINVAL;
4278
4279         ret = key_size;
4280  out:
4281         return ret;
4282 }
4283
4284 static int create_key_fields(struct hist_trigger_data *hist_data,
4285                              struct trace_event_file *file)
4286 {
4287         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4288         char *fields_str, *field_str;
4289         int ret = -EINVAL;
4290
4291         fields_str = hist_data->attrs->keys_str;
4292         if (!fields_str)
4293                 goto out;
4294
4295         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4296                 field_str = strsep(&fields_str, ",");
4297                 if (!field_str)
4298                         break;
4299                 ret = create_key_field(hist_data, i, key_offset,
4300                                        file, field_str);
4301                 if (ret < 0)
4302                         goto out;
4303                 key_offset += ret;
4304         }
4305         if (fields_str) {
4306                 ret = -EINVAL;
4307                 goto out;
4308         }
4309         ret = 0;
4310  out:
4311         return ret;
4312 }
4313
4314 static int create_var_fields(struct hist_trigger_data *hist_data,
4315                              struct trace_event_file *file)
4316 {
4317         unsigned int i, j = hist_data->n_vals;
4318         int ret = 0;
4319
4320         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4321
4322         for (i = 0; i < n_vars; i++) {
4323                 char *var_name = hist_data->attrs->var_defs.name[i];
4324                 char *expr = hist_data->attrs->var_defs.expr[i];
4325
4326                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4327                 if (ret)
4328                         goto out;
4329         }
4330  out:
4331         return ret;
4332 }
4333
4334 static void free_var_defs(struct hist_trigger_data *hist_data)
4335 {
4336         unsigned int i;
4337
4338         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4339                 kfree(hist_data->attrs->var_defs.name[i]);
4340                 kfree(hist_data->attrs->var_defs.expr[i]);
4341         }
4342
4343         hist_data->attrs->var_defs.n_vars = 0;
4344 }
4345
4346 static int parse_var_defs(struct hist_trigger_data *hist_data)
4347 {
4348         struct trace_array *tr = hist_data->event_file->tr;
4349         char *s, *str, *var_name, *field_str;
4350         unsigned int i, j, n_vars = 0;
4351         int ret = 0;
4352
4353         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4354                 str = hist_data->attrs->assignment_str[i];
4355                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4356                         field_str = strsep(&str, ",");
4357                         if (!field_str)
4358                                 break;
4359
4360                         var_name = strsep(&field_str, "=");
4361                         if (!var_name || !field_str) {
4362                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4363                                          errpos(var_name));
4364                                 ret = -EINVAL;
4365                                 goto free;
4366                         }
4367
4368                         if (n_vars == TRACING_MAP_VARS_MAX) {
4369                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4370                                 ret = -EINVAL;
4371                                 goto free;
4372                         }
4373
4374                         s = kstrdup(var_name, GFP_KERNEL);
4375                         if (!s) {
4376                                 ret = -ENOMEM;
4377                                 goto free;
4378                         }
4379                         hist_data->attrs->var_defs.name[n_vars] = s;
4380
4381                         s = kstrdup(field_str, GFP_KERNEL);
4382                         if (!s) {
4383                                 ret = -ENOMEM;
4384                                 goto free;
4385                         }
4386                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4387
4388                         hist_data->attrs->var_defs.n_vars = n_vars;
4389                 }
4390         }
4391
4392         return ret;
4393  free:
4394         free_var_defs(hist_data);
4395
4396         return ret;
4397 }
4398
4399 static int create_hist_fields(struct hist_trigger_data *hist_data,
4400                               struct trace_event_file *file)
4401 {
4402         int ret;
4403
4404         ret = parse_var_defs(hist_data);
4405         if (ret)
4406                 goto out;
4407
4408         ret = create_val_fields(hist_data, file);
4409         if (ret)
4410                 goto out;
4411
4412         ret = create_var_fields(hist_data, file);
4413         if (ret)
4414                 goto out;
4415
4416         ret = create_key_fields(hist_data, file);
4417         if (ret)
4418                 goto out;
4419  out:
4420         free_var_defs(hist_data);
4421
4422         return ret;
4423 }
4424
4425 static int is_descending(struct trace_array *tr, const char *str)
4426 {
4427         if (!str)
4428                 return 0;
4429
4430         if (strcmp(str, "descending") == 0)
4431                 return 1;
4432
4433         if (strcmp(str, "ascending") == 0)
4434                 return 0;
4435
4436         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4437
4438         return -EINVAL;
4439 }
4440
4441 static int create_sort_keys(struct hist_trigger_data *hist_data)
4442 {
4443         struct trace_array *tr = hist_data->event_file->tr;
4444         char *fields_str = hist_data->attrs->sort_key_str;
4445         struct tracing_map_sort_key *sort_key;
4446         int descending, ret = 0;
4447         unsigned int i, j, k;
4448
4449         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4450
4451         if (!fields_str)
4452                 goto out;
4453
4454         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4455                 struct hist_field *hist_field;
4456                 char *field_str, *field_name;
4457                 const char *test_name;
4458
4459                 sort_key = &hist_data->sort_keys[i];
4460
4461                 field_str = strsep(&fields_str, ",");
4462                 if (!field_str)
4463                         break;
4464
4465                 if (!*field_str) {
4466                         ret = -EINVAL;
4467                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4468                         break;
4469                 }
4470
4471                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4472                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4473                         ret = -EINVAL;
4474                         break;
4475                 }
4476
4477                 field_name = strsep(&field_str, ".");
4478                 if (!field_name || !*field_name) {
4479                         ret = -EINVAL;
4480                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4481                         break;
4482                 }
4483
4484                 if (strcmp(field_name, "hitcount") == 0) {
4485                         descending = is_descending(tr, field_str);
4486                         if (descending < 0) {
4487                                 ret = descending;
4488                                 break;
4489                         }
4490                         sort_key->descending = descending;
4491                         continue;
4492                 }
4493
4494                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4495                         unsigned int idx;
4496
4497                         hist_field = hist_data->fields[j];
4498                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4499                                 continue;
4500
4501                         idx = k++;
4502
4503                         test_name = hist_field_name(hist_field, 0);
4504
4505                         if (strcmp(field_name, test_name) == 0) {
4506                                 sort_key->field_idx = idx;
4507                                 descending = is_descending(tr, field_str);
4508                                 if (descending < 0) {
4509                                         ret = descending;
4510                                         goto out;
4511                                 }
4512                                 sort_key->descending = descending;
4513                                 break;
4514                         }
4515                 }
4516                 if (j == hist_data->n_fields) {
4517                         ret = -EINVAL;
4518                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4519                         break;
4520                 }
4521         }
4522
4523         hist_data->n_sort_keys = i;
4524  out:
4525         return ret;
4526 }
4527
4528 static void destroy_actions(struct hist_trigger_data *hist_data)
4529 {
4530         unsigned int i;
4531
4532         for (i = 0; i < hist_data->n_actions; i++) {
4533                 struct action_data *data = hist_data->actions[i];
4534
4535                 if (data->handler == HANDLER_ONMATCH)
4536                         onmatch_destroy(data);
4537                 else if (data->handler == HANDLER_ONMAX ||
4538                          data->handler == HANDLER_ONCHANGE)
4539                         track_data_destroy(hist_data, data);
4540                 else
4541                         kfree(data);
4542         }
4543 }
4544
4545 static int parse_actions(struct hist_trigger_data *hist_data)
4546 {
4547         struct trace_array *tr = hist_data->event_file->tr;
4548         struct action_data *data;
4549         unsigned int i;
4550         int ret = 0;
4551         char *str;
4552         int len;
4553
4554         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4555                 str = hist_data->attrs->action_str[i];
4556
4557                 if ((len = str_has_prefix(str, "onmatch("))) {
4558                         char *action_str = str + len;
4559
4560                         data = onmatch_parse(tr, action_str);
4561                         if (IS_ERR(data)) {
4562                                 ret = PTR_ERR(data);
4563                                 break;
4564                         }
4565                 } else if ((len = str_has_prefix(str, "onmax("))) {
4566                         char *action_str = str + len;
4567
4568                         data = track_data_parse(hist_data, action_str,
4569                                                 HANDLER_ONMAX);
4570                         if (IS_ERR(data)) {
4571                                 ret = PTR_ERR(data);
4572                                 break;
4573                         }
4574                 } else if ((len = str_has_prefix(str, "onchange("))) {
4575                         char *action_str = str + len;
4576
4577                         data = track_data_parse(hist_data, action_str,
4578                                                 HANDLER_ONCHANGE);
4579                         if (IS_ERR(data)) {
4580                                 ret = PTR_ERR(data);
4581                                 break;
4582                         }
4583                 } else {
4584                         ret = -EINVAL;
4585                         break;
4586                 }
4587
4588                 hist_data->actions[hist_data->n_actions++] = data;
4589         }
4590
4591         return ret;
4592 }
4593
4594 static int create_actions(struct hist_trigger_data *hist_data)
4595 {
4596         struct action_data *data;
4597         unsigned int i;
4598         int ret = 0;
4599
4600         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4601                 data = hist_data->actions[i];
4602
4603                 if (data->handler == HANDLER_ONMATCH) {
4604                         ret = onmatch_create(hist_data, data);
4605                         if (ret)
4606                                 break;
4607                 } else if (data->handler == HANDLER_ONMAX ||
4608                            data->handler == HANDLER_ONCHANGE) {
4609                         ret = track_data_create(hist_data, data);
4610                         if (ret)
4611                                 break;
4612                 } else {
4613                         ret = -EINVAL;
4614                         break;
4615                 }
4616         }
4617
4618         return ret;
4619 }
4620
4621 static void print_actions(struct seq_file *m,
4622                           struct hist_trigger_data *hist_data,
4623                           struct tracing_map_elt *elt)
4624 {
4625         unsigned int i;
4626
4627         for (i = 0; i < hist_data->n_actions; i++) {
4628                 struct action_data *data = hist_data->actions[i];
4629
4630                 if (data->action == ACTION_SNAPSHOT)
4631                         continue;
4632
4633                 if (data->handler == HANDLER_ONMAX ||
4634                     data->handler == HANDLER_ONCHANGE)
4635                         track_data_print(m, hist_data, elt, data);
4636         }
4637 }
4638
4639 static void print_action_spec(struct seq_file *m,
4640                               struct hist_trigger_data *hist_data,
4641                               struct action_data *data)
4642 {
4643         unsigned int i;
4644
4645         if (data->action == ACTION_SAVE) {
4646                 for (i = 0; i < hist_data->n_save_vars; i++) {
4647                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4648                         if (i < hist_data->n_save_vars - 1)
4649                                 seq_puts(m, ",");
4650                 }
4651         } else if (data->action == ACTION_TRACE) {
4652                 if (data->use_trace_keyword)
4653                         seq_printf(m, "%s", data->synth_event_name);
4654                 for (i = 0; i < data->n_params; i++) {
4655                         if (i || data->use_trace_keyword)
4656                                 seq_puts(m, ",");
4657                         seq_printf(m, "%s", data->params[i]);
4658                 }
4659         }
4660 }
4661
4662 static void print_track_data_spec(struct seq_file *m,
4663                                   struct hist_trigger_data *hist_data,
4664                                   struct action_data *data)
4665 {
4666         if (data->handler == HANDLER_ONMAX)
4667                 seq_puts(m, ":onmax(");
4668         else if (data->handler == HANDLER_ONCHANGE)
4669                 seq_puts(m, ":onchange(");
4670         seq_printf(m, "%s", data->track_data.var_str);
4671         seq_printf(m, ").%s(", data->action_name);
4672
4673         print_action_spec(m, hist_data, data);
4674
4675         seq_puts(m, ")");
4676 }
4677
4678 static void print_onmatch_spec(struct seq_file *m,
4679                                struct hist_trigger_data *hist_data,
4680                                struct action_data *data)
4681 {
4682         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4683                    data->match_data.event);
4684
4685         seq_printf(m, "%s(", data->action_name);
4686
4687         print_action_spec(m, hist_data, data);
4688
4689         seq_puts(m, ")");
4690 }
4691
4692 static bool actions_match(struct hist_trigger_data *hist_data,
4693                           struct hist_trigger_data *hist_data_test)
4694 {
4695         unsigned int i, j;
4696
4697         if (hist_data->n_actions != hist_data_test->n_actions)
4698                 return false;
4699
4700         for (i = 0; i < hist_data->n_actions; i++) {
4701                 struct action_data *data = hist_data->actions[i];
4702                 struct action_data *data_test = hist_data_test->actions[i];
4703                 char *action_name, *action_name_test;
4704
4705                 if (data->handler != data_test->handler)
4706                         return false;
4707                 if (data->action != data_test->action)
4708                         return false;
4709
4710                 if (data->n_params != data_test->n_params)
4711                         return false;
4712
4713                 for (j = 0; j < data->n_params; j++) {
4714                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4715                                 return false;
4716                 }
4717
4718                 if (data->use_trace_keyword)
4719                         action_name = data->synth_event_name;
4720                 else
4721                         action_name = data->action_name;
4722
4723                 if (data_test->use_trace_keyword)
4724                         action_name_test = data_test->synth_event_name;
4725                 else
4726                         action_name_test = data_test->action_name;
4727
4728                 if (strcmp(action_name, action_name_test) != 0)
4729                         return false;
4730
4731                 if (data->handler == HANDLER_ONMATCH) {
4732                         if (strcmp(data->match_data.event_system,
4733                                    data_test->match_data.event_system) != 0)
4734                                 return false;
4735                         if (strcmp(data->match_data.event,
4736                                    data_test->match_data.event) != 0)
4737                                 return false;
4738                 } else if (data->handler == HANDLER_ONMAX ||
4739                            data->handler == HANDLER_ONCHANGE) {
4740                         if (strcmp(data->track_data.var_str,
4741                                    data_test->track_data.var_str) != 0)
4742                                 return false;
4743                 }
4744         }
4745
4746         return true;
4747 }
4748
4749
4750 static void print_actions_spec(struct seq_file *m,
4751                                struct hist_trigger_data *hist_data)
4752 {
4753         unsigned int i;
4754
4755         for (i = 0; i < hist_data->n_actions; i++) {
4756                 struct action_data *data = hist_data->actions[i];
4757
4758                 if (data->handler == HANDLER_ONMATCH)
4759                         print_onmatch_spec(m, hist_data, data);
4760                 else if (data->handler == HANDLER_ONMAX ||
4761                          data->handler == HANDLER_ONCHANGE)
4762                         print_track_data_spec(m, hist_data, data);
4763         }
4764 }
4765
4766 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4767 {
4768         unsigned int i;
4769
4770         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4771                 kfree(hist_data->field_var_hists[i]->cmd);
4772                 kfree(hist_data->field_var_hists[i]);
4773         }
4774 }
4775
4776 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4777 {
4778         if (!hist_data)
4779                 return;
4780
4781         destroy_hist_trigger_attrs(hist_data->attrs);
4782         destroy_hist_fields(hist_data);
4783         tracing_map_destroy(hist_data->map);
4784
4785         destroy_actions(hist_data);
4786         destroy_field_vars(hist_data);
4787         destroy_field_var_hists(hist_data);
4788
4789         kfree(hist_data);
4790 }
4791
4792 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4793 {
4794         struct tracing_map *map = hist_data->map;
4795         struct ftrace_event_field *field;
4796         struct hist_field *hist_field;
4797         int i, idx = 0;
4798
4799         for_each_hist_field(i, hist_data) {
4800                 hist_field = hist_data->fields[i];
4801                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4802                         tracing_map_cmp_fn_t cmp_fn;
4803
4804                         field = hist_field->field;
4805
4806                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4807                                 cmp_fn = tracing_map_cmp_none;
4808                         else if (!field)
4809                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4810                                                              hist_field->is_signed);
4811                         else if (is_string_field(field))
4812                                 cmp_fn = tracing_map_cmp_string;
4813                         else
4814                                 cmp_fn = tracing_map_cmp_num(field->size,
4815                                                              field->is_signed);
4816                         idx = tracing_map_add_key_field(map,
4817                                                         hist_field->offset,
4818                                                         cmp_fn);
4819                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4820                         idx = tracing_map_add_sum_field(map);
4821
4822                 if (idx < 0)
4823                         return idx;
4824
4825                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4826                         idx = tracing_map_add_var(map);
4827                         if (idx < 0)
4828                                 return idx;
4829                         hist_field->var.idx = idx;
4830                         hist_field->var.hist_data = hist_data;
4831                 }
4832         }
4833
4834         return 0;
4835 }
4836
4837 static struct hist_trigger_data *
4838 create_hist_data(unsigned int map_bits,
4839                  struct hist_trigger_attrs *attrs,
4840                  struct trace_event_file *file,
4841                  bool remove)
4842 {
4843         const struct tracing_map_ops *map_ops = NULL;
4844         struct hist_trigger_data *hist_data;
4845         int ret = 0;
4846
4847         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4848         if (!hist_data)
4849                 return ERR_PTR(-ENOMEM);
4850
4851         hist_data->attrs = attrs;
4852         hist_data->remove = remove;
4853         hist_data->event_file = file;
4854
4855         ret = parse_actions(hist_data);
4856         if (ret)
4857                 goto free;
4858
4859         ret = create_hist_fields(hist_data, file);
4860         if (ret)
4861                 goto free;
4862
4863         ret = create_sort_keys(hist_data);
4864         if (ret)
4865                 goto free;
4866
4867         map_ops = &hist_trigger_elt_data_ops;
4868
4869         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4870                                             map_ops, hist_data);
4871         if (IS_ERR(hist_data->map)) {
4872                 ret = PTR_ERR(hist_data->map);
4873                 hist_data->map = NULL;
4874                 goto free;
4875         }
4876
4877         ret = create_tracing_map_fields(hist_data);
4878         if (ret)
4879                 goto free;
4880  out:
4881         return hist_data;
4882  free:
4883         hist_data->attrs = NULL;
4884
4885         destroy_hist_data(hist_data);
4886
4887         hist_data = ERR_PTR(ret);
4888
4889         goto out;
4890 }
4891
4892 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4893                                     struct tracing_map_elt *elt,
4894                                     struct trace_buffer *buffer, void *rec,
4895                                     struct ring_buffer_event *rbe,
4896                                     u64 *var_ref_vals)
4897 {
4898         struct hist_elt_data *elt_data;
4899         struct hist_field *hist_field;
4900         unsigned int i, var_idx;
4901         u64 hist_val;
4902
4903         elt_data = elt->private_data;
4904         elt_data->var_ref_vals = var_ref_vals;
4905
4906         for_each_hist_val_field(i, hist_data) {
4907                 hist_field = hist_data->fields[i];
4908                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4909                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4910                         var_idx = hist_field->var.idx;
4911
4912                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4913                                 unsigned int str_start, var_str_idx, idx;
4914                                 char *str, *val_str;
4915
4916                                 str_start = hist_data->n_field_var_str +
4917                                         hist_data->n_save_var_str;
4918                                 var_str_idx = hist_field->var_str_idx;
4919                                 idx = str_start + var_str_idx;
4920
4921                                 str = elt_data->field_var_str[idx];
4922                                 val_str = (char *)(uintptr_t)hist_val;
4923                                 strscpy(str, val_str, STR_VAR_LEN_MAX);
4924
4925                                 hist_val = (u64)(uintptr_t)str;
4926                         }
4927                         tracing_map_set_var(elt, var_idx, hist_val);
4928                         continue;
4929                 }
4930                 tracing_map_update_sum(elt, i, hist_val);
4931         }
4932
4933         for_each_hist_key_field(i, hist_data) {
4934                 hist_field = hist_data->fields[i];
4935                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4936                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4937                         var_idx = hist_field->var.idx;
4938                         tracing_map_set_var(elt, var_idx, hist_val);
4939                 }
4940         }
4941
4942         update_field_vars(hist_data, elt, buffer, rbe, rec);
4943 }
4944
4945 static inline void add_to_key(char *compound_key, void *key,
4946                               struct hist_field *key_field, void *rec)
4947 {
4948         size_t size = key_field->size;
4949
4950         if (key_field->flags & HIST_FIELD_FL_STRING) {
4951                 struct ftrace_event_field *field;
4952
4953                 field = key_field->field;
4954                 if (field->filter_type == FILTER_DYN_STRING)
4955                         size = *(u32 *)(rec + field->offset) >> 16;
4956                 else if (field->filter_type == FILTER_STATIC_STRING)
4957                         size = field->size;
4958
4959                 /* ensure NULL-termination */
4960                 if (size > key_field->size - 1)
4961                         size = key_field->size - 1;
4962
4963                 strncpy(compound_key + key_field->offset, (char *)key, size);
4964         } else
4965                 memcpy(compound_key + key_field->offset, key, size);
4966 }
4967
4968 static void
4969 hist_trigger_actions(struct hist_trigger_data *hist_data,
4970                      struct tracing_map_elt *elt,
4971                      struct trace_buffer *buffer, void *rec,
4972                      struct ring_buffer_event *rbe, void *key,
4973                      u64 *var_ref_vals)
4974 {
4975         struct action_data *data;
4976         unsigned int i;
4977
4978         for (i = 0; i < hist_data->n_actions; i++) {
4979                 data = hist_data->actions[i];
4980                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4981         }
4982 }
4983
4984 static void event_hist_trigger(struct event_trigger_data *data,
4985                                struct trace_buffer *buffer, void *rec,
4986                                struct ring_buffer_event *rbe)
4987 {
4988         struct hist_trigger_data *hist_data = data->private_data;
4989         bool use_compound_key = (hist_data->n_keys > 1);
4990         unsigned long entries[HIST_STACKTRACE_DEPTH];
4991         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4992         char compound_key[HIST_KEY_SIZE_MAX];
4993         struct tracing_map_elt *elt = NULL;
4994         struct hist_field *key_field;
4995         u64 field_contents;
4996         void *key = NULL;
4997         unsigned int i;
4998
4999         memset(compound_key, 0, hist_data->key_size);
5000
5001         for_each_hist_key_field(i, hist_data) {
5002                 key_field = hist_data->fields[i];
5003
5004                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5005                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5006                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5007                                          HIST_STACKTRACE_SKIP);
5008                         key = entries;
5009                 } else {
5010                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5011                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5012                                 key = (void *)(unsigned long)field_contents;
5013                                 use_compound_key = true;
5014                         } else
5015                                 key = (void *)&field_contents;
5016                 }
5017
5018                 if (use_compound_key)
5019                         add_to_key(compound_key, key, key_field, rec);
5020         }
5021
5022         if (use_compound_key)
5023                 key = compound_key;
5024
5025         if (hist_data->n_var_refs &&
5026             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5027                 return;
5028
5029         elt = tracing_map_insert(hist_data->map, key);
5030         if (!elt)
5031                 return;
5032
5033         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5034
5035         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5036                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5037 }
5038
5039 static void hist_trigger_stacktrace_print(struct seq_file *m,
5040                                           unsigned long *stacktrace_entries,
5041                                           unsigned int max_entries)
5042 {
5043         unsigned int spaces = 8;
5044         unsigned int i;
5045
5046         for (i = 0; i < max_entries; i++) {
5047                 if (!stacktrace_entries[i])
5048                         return;
5049
5050                 seq_printf(m, "%*c", 1 + spaces, ' ');
5051                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5052         }
5053 }
5054
5055 static void hist_trigger_print_key(struct seq_file *m,
5056                                    struct hist_trigger_data *hist_data,
5057                                    void *key,
5058                                    struct tracing_map_elt *elt)
5059 {
5060         struct hist_field *key_field;
5061         bool multiline = false;
5062         const char *field_name;
5063         unsigned int i;
5064         u64 uval;
5065
5066         seq_puts(m, "{ ");
5067
5068         for_each_hist_key_field(i, hist_data) {
5069                 key_field = hist_data->fields[i];
5070
5071                 if (i > hist_data->n_vals)
5072                         seq_puts(m, ", ");
5073
5074                 field_name = hist_field_name(key_field, 0);
5075
5076                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5077                         uval = *(u64 *)(key + key_field->offset);
5078                         seq_printf(m, "%s: %llx", field_name, uval);
5079                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5080                         uval = *(u64 *)(key + key_field->offset);
5081                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5082                                    uval, (void *)(uintptr_t)uval);
5083                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5084                         uval = *(u64 *)(key + key_field->offset);
5085                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5086                                    uval, (void *)(uintptr_t)uval);
5087                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5088                         struct hist_elt_data *elt_data = elt->private_data;
5089                         char *comm;
5090
5091                         if (WARN_ON_ONCE(!elt_data))
5092                                 return;
5093
5094                         comm = elt_data->comm;
5095
5096                         uval = *(u64 *)(key + key_field->offset);
5097                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5098                                    comm, uval);
5099                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5100                         const char *syscall_name;
5101
5102                         uval = *(u64 *)(key + key_field->offset);
5103                         syscall_name = get_syscall_name(uval);
5104                         if (!syscall_name)
5105                                 syscall_name = "unknown_syscall";
5106
5107                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5108                                    syscall_name, uval);
5109                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5110                         seq_puts(m, "stacktrace:\n");
5111                         hist_trigger_stacktrace_print(m,
5112                                                       key + key_field->offset,
5113                                                       HIST_STACKTRACE_DEPTH);
5114                         multiline = true;
5115                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5116                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5117                                    *(u64 *)(key + key_field->offset));
5118                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5119                         unsigned long buckets = key_field->buckets;
5120                         uval = *(u64 *)(key + key_field->offset);
5121                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5122                                    uval, uval + buckets -1);
5123                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5124                         seq_printf(m, "%s: %-50s", field_name,
5125                                    (char *)(key + key_field->offset));
5126                 } else {
5127                         uval = *(u64 *)(key + key_field->offset);
5128                         seq_printf(m, "%s: %10llu", field_name, uval);
5129                 }
5130         }
5131
5132         if (!multiline)
5133                 seq_puts(m, " ");
5134
5135         seq_puts(m, "}");
5136 }
5137
5138 static void hist_trigger_entry_print(struct seq_file *m,
5139                                      struct hist_trigger_data *hist_data,
5140                                      void *key,
5141                                      struct tracing_map_elt *elt)
5142 {
5143         const char *field_name;
5144         unsigned int i;
5145
5146         hist_trigger_print_key(m, hist_data, key, elt);
5147
5148         seq_printf(m, " hitcount: %10llu",
5149                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5150
5151         for (i = 1; i < hist_data->n_vals; i++) {
5152                 field_name = hist_field_name(hist_data->fields[i], 0);
5153
5154                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5155                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5156                         continue;
5157
5158                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5159                         seq_printf(m, "  %s: %10llx", field_name,
5160                                    tracing_map_read_sum(elt, i));
5161                 } else {
5162                         seq_printf(m, "  %s: %10llu", field_name,
5163                                    tracing_map_read_sum(elt, i));
5164                 }
5165         }
5166
5167         print_actions(m, hist_data, elt);
5168
5169         seq_puts(m, "\n");
5170 }
5171
5172 static int print_entries(struct seq_file *m,
5173                          struct hist_trigger_data *hist_data)
5174 {
5175         struct tracing_map_sort_entry **sort_entries = NULL;
5176         struct tracing_map *map = hist_data->map;
5177         int i, n_entries;
5178
5179         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5180                                              hist_data->n_sort_keys,
5181                                              &sort_entries);
5182         if (n_entries < 0)
5183                 return n_entries;
5184
5185         for (i = 0; i < n_entries; i++)
5186                 hist_trigger_entry_print(m, hist_data,
5187                                          sort_entries[i]->key,
5188                                          sort_entries[i]->elt);
5189
5190         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5191
5192         return n_entries;
5193 }
5194
5195 static void hist_trigger_show(struct seq_file *m,
5196                               struct event_trigger_data *data, int n)
5197 {
5198         struct hist_trigger_data *hist_data;
5199         int n_entries;
5200
5201         if (n > 0)
5202                 seq_puts(m, "\n\n");
5203
5204         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5205         data->ops->print(m, data->ops, data);
5206         seq_puts(m, "#\n\n");
5207
5208         hist_data = data->private_data;
5209         n_entries = print_entries(m, hist_data);
5210         if (n_entries < 0)
5211                 n_entries = 0;
5212
5213         track_data_snapshot_print(m, hist_data);
5214
5215         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5216                    (u64)atomic64_read(&hist_data->map->hits),
5217                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5218 }
5219
5220 static int hist_show(struct seq_file *m, void *v)
5221 {
5222         struct event_trigger_data *data;
5223         struct trace_event_file *event_file;
5224         int n = 0, ret = 0;
5225
5226         mutex_lock(&event_mutex);
5227
5228         event_file = event_file_data(m->private);
5229         if (unlikely(!event_file)) {
5230                 ret = -ENODEV;
5231                 goto out_unlock;
5232         }
5233
5234         list_for_each_entry(data, &event_file->triggers, list) {
5235                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5236                         hist_trigger_show(m, data, n++);
5237         }
5238
5239  out_unlock:
5240         mutex_unlock(&event_mutex);
5241
5242         return ret;
5243 }
5244
5245 static int event_hist_open(struct inode *inode, struct file *file)
5246 {
5247         int ret;
5248
5249         ret = security_locked_down(LOCKDOWN_TRACEFS);
5250         if (ret)
5251                 return ret;
5252
5253         return single_open(file, hist_show, file);
5254 }
5255
5256 const struct file_operations event_hist_fops = {
5257         .open = event_hist_open,
5258         .read = seq_read,
5259         .llseek = seq_lseek,
5260         .release = single_release,
5261 };
5262
5263 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5264 static void hist_field_debug_show_flags(struct seq_file *m,
5265                                         unsigned long flags)
5266 {
5267         seq_puts(m, "      flags:\n");
5268
5269         if (flags & HIST_FIELD_FL_KEY)
5270                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5271         else if (flags & HIST_FIELD_FL_HITCOUNT)
5272                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5273         else if (flags & HIST_FIELD_FL_VAR)
5274                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5275         else if (flags & HIST_FIELD_FL_VAR_REF)
5276                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5277         else
5278                 seq_puts(m, "        VAL: normal u64 value\n");
5279
5280         if (flags & HIST_FIELD_FL_ALIAS)
5281                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5282         else if (flags & HIST_FIELD_FL_CONST)
5283                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5284 }
5285
5286 static int hist_field_debug_show(struct seq_file *m,
5287                                  struct hist_field *field, unsigned long flags)
5288 {
5289         if ((field->flags & flags) != flags) {
5290                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5291                 return -EINVAL;
5292         }
5293
5294         hist_field_debug_show_flags(m, field->flags);
5295         if (field->field)
5296                 seq_printf(m, "      ftrace_event_field name: %s\n",
5297                            field->field->name);
5298
5299         if (field->flags & HIST_FIELD_FL_VAR) {
5300                 seq_printf(m, "      var.name: %s\n", field->var.name);
5301                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5302                            field->var.idx);
5303         }
5304
5305         if (field->flags & HIST_FIELD_FL_CONST)
5306                 seq_printf(m, "      constant: %llu\n", field->constant);
5307
5308         if (field->flags & HIST_FIELD_FL_ALIAS)
5309                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5310                            field->var_ref_idx);
5311
5312         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5313                 seq_printf(m, "      name: %s\n", field->name);
5314                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5315                            field->var.idx);
5316                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5317                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5318                            field->var_ref_idx);
5319                 if (field->system)
5320                         seq_printf(m, "      system: %s\n", field->system);
5321                 if (field->event_name)
5322                         seq_printf(m, "      event_name: %s\n", field->event_name);
5323         }
5324
5325         seq_printf(m, "      type: %s\n", field->type);
5326         seq_printf(m, "      size: %u\n", field->size);
5327         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5328
5329         return 0;
5330 }
5331
5332 static int field_var_debug_show(struct seq_file *m,
5333                                 struct field_var *field_var, unsigned int i,
5334                                 bool save_vars)
5335 {
5336         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5337         struct hist_field *field;
5338         int ret = 0;
5339
5340         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5341
5342         field = field_var->var;
5343
5344         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5345
5346         hist_field_debug_show_flags(m, field->flags);
5347         seq_printf(m, "      var.name: %s\n", field->var.name);
5348         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5349                    field->var.idx);
5350
5351         field = field_var->val;
5352
5353         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5354         if (field->field)
5355                 seq_printf(m, "      ftrace_event_field name: %s\n",
5356                            field->field->name);
5357         else {
5358                 ret = -EINVAL;
5359                 goto out;
5360         }
5361
5362         seq_printf(m, "      type: %s\n", field->type);
5363         seq_printf(m, "      size: %u\n", field->size);
5364         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5365 out:
5366         return ret;
5367 }
5368
5369 static int hist_action_debug_show(struct seq_file *m,
5370                                   struct action_data *data, int i)
5371 {
5372         int ret = 0;
5373
5374         if (data->handler == HANDLER_ONMAX ||
5375             data->handler == HANDLER_ONCHANGE) {
5376                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5377                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5378                                             HIST_FIELD_FL_VAR_REF);
5379                 if (ret)
5380                         goto out;
5381
5382                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5383                 ret = hist_field_debug_show(m, data->track_data.track_var,
5384                                             HIST_FIELD_FL_VAR);
5385                 if (ret)
5386                         goto out;
5387         }
5388
5389         if (data->handler == HANDLER_ONMATCH) {
5390                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5391                            i, data->match_data.event_system);
5392                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5393                            i, data->match_data.event);
5394         }
5395 out:
5396         return ret;
5397 }
5398
5399 static int hist_actions_debug_show(struct seq_file *m,
5400                                    struct hist_trigger_data *hist_data)
5401 {
5402         int i, ret = 0;
5403
5404         if (hist_data->n_actions)
5405                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5406
5407         for (i = 0; i < hist_data->n_actions; i++) {
5408                 struct action_data *action = hist_data->actions[i];
5409
5410                 ret = hist_action_debug_show(m, action, i);
5411                 if (ret)
5412                         goto out;
5413         }
5414
5415         if (hist_data->n_save_vars)
5416                 seq_puts(m, "\n  save action variables (save() params):\n");
5417
5418         for (i = 0; i < hist_data->n_save_vars; i++) {
5419                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5420                 if (ret)
5421                         goto out;
5422         }
5423 out:
5424         return ret;
5425 }
5426
5427 static void hist_trigger_debug_show(struct seq_file *m,
5428                                     struct event_trigger_data *data, int n)
5429 {
5430         struct hist_trigger_data *hist_data;
5431         int i, ret;
5432
5433         if (n > 0)
5434                 seq_puts(m, "\n\n");
5435
5436         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5437         data->ops->print(m, data->ops, data);
5438         seq_puts(m, "#\n\n");
5439
5440         hist_data = data->private_data;
5441
5442         seq_printf(m, "hist_data: %p\n\n", hist_data);
5443         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5444         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5445         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5446
5447         seq_puts(m, "\n  val fields:\n\n");
5448
5449         seq_puts(m, "    hist_data->fields[0]:\n");
5450         ret = hist_field_debug_show(m, hist_data->fields[0],
5451                                     HIST_FIELD_FL_HITCOUNT);
5452         if (ret)
5453                 return;
5454
5455         for (i = 1; i < hist_data->n_vals; i++) {
5456                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5457                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5458                 if (ret)
5459                         return;
5460         }
5461
5462         seq_puts(m, "\n  key fields:\n");
5463
5464         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5465                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5466                 ret = hist_field_debug_show(m, hist_data->fields[i],
5467                                             HIST_FIELD_FL_KEY);
5468                 if (ret)
5469                         return;
5470         }
5471
5472         if (hist_data->n_var_refs)
5473                 seq_puts(m, "\n  variable reference fields:\n");
5474
5475         for (i = 0; i < hist_data->n_var_refs; i++) {
5476                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5477                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5478                                             HIST_FIELD_FL_VAR_REF);
5479                 if (ret)
5480                         return;
5481         }
5482
5483         if (hist_data->n_field_vars)
5484                 seq_puts(m, "\n  field variables:\n");
5485
5486         for (i = 0; i < hist_data->n_field_vars; i++) {
5487                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5488                 if (ret)
5489                         return;
5490         }
5491
5492         ret = hist_actions_debug_show(m, hist_data);
5493         if (ret)
5494                 return;
5495 }
5496
5497 static int hist_debug_show(struct seq_file *m, void *v)
5498 {
5499         struct event_trigger_data *data;
5500         struct trace_event_file *event_file;
5501         int n = 0, ret = 0;
5502
5503         mutex_lock(&event_mutex);
5504
5505         event_file = event_file_data(m->private);
5506         if (unlikely(!event_file)) {
5507                 ret = -ENODEV;
5508                 goto out_unlock;
5509         }
5510
5511         list_for_each_entry(data, &event_file->triggers, list) {
5512                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5513                         hist_trigger_debug_show(m, data, n++);
5514         }
5515
5516  out_unlock:
5517         mutex_unlock(&event_mutex);
5518
5519         return ret;
5520 }
5521
5522 static int event_hist_debug_open(struct inode *inode, struct file *file)
5523 {
5524         int ret;
5525
5526         ret = security_locked_down(LOCKDOWN_TRACEFS);
5527         if (ret)
5528                 return ret;
5529
5530         return single_open(file, hist_debug_show, file);
5531 }
5532
5533 const struct file_operations event_hist_debug_fops = {
5534         .open = event_hist_debug_open,
5535         .read = seq_read,
5536         .llseek = seq_lseek,
5537         .release = single_release,
5538 };
5539 #endif
5540
5541 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5542 {
5543         const char *field_name = hist_field_name(hist_field, 0);
5544
5545         if (hist_field->var.name)
5546                 seq_printf(m, "%s=", hist_field->var.name);
5547
5548         if (hist_field->flags & HIST_FIELD_FL_CPU)
5549                 seq_puts(m, "common_cpu");
5550         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5551                 seq_printf(m, "%llu", hist_field->constant);
5552         else if (field_name) {
5553                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5554                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5555                         seq_putc(m, '$');
5556                 seq_printf(m, "%s", field_name);
5557         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5558                 seq_puts(m, "common_timestamp");
5559
5560         if (hist_field->flags) {
5561                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5562                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5563                         const char *flags = get_hist_field_flags(hist_field);
5564
5565                         if (flags)
5566                                 seq_printf(m, ".%s", flags);
5567                 }
5568         }
5569         if (hist_field->buckets)
5570                 seq_printf(m, "=%ld", hist_field->buckets);
5571 }
5572
5573 static int event_hist_trigger_print(struct seq_file *m,
5574                                     struct event_trigger_ops *ops,
5575                                     struct event_trigger_data *data)
5576 {
5577         struct hist_trigger_data *hist_data = data->private_data;
5578         struct hist_field *field;
5579         bool have_var = false;
5580         unsigned int i;
5581
5582         seq_puts(m, "hist:");
5583
5584         if (data->name)
5585                 seq_printf(m, "%s:", data->name);
5586
5587         seq_puts(m, "keys=");
5588
5589         for_each_hist_key_field(i, hist_data) {
5590                 field = hist_data->fields[i];
5591
5592                 if (i > hist_data->n_vals)
5593                         seq_puts(m, ",");
5594
5595                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5596                         seq_puts(m, "stacktrace");
5597                 else
5598                         hist_field_print(m, field);
5599         }
5600
5601         seq_puts(m, ":vals=");
5602
5603         for_each_hist_val_field(i, hist_data) {
5604                 field = hist_data->fields[i];
5605                 if (field->flags & HIST_FIELD_FL_VAR) {
5606                         have_var = true;
5607                         continue;
5608                 }
5609
5610                 if (i == HITCOUNT_IDX)
5611                         seq_puts(m, "hitcount");
5612                 else {
5613                         seq_puts(m, ",");
5614                         hist_field_print(m, field);
5615                 }
5616         }
5617
5618         if (have_var) {
5619                 unsigned int n = 0;
5620
5621                 seq_puts(m, ":");
5622
5623                 for_each_hist_val_field(i, hist_data) {
5624                         field = hist_data->fields[i];
5625
5626                         if (field->flags & HIST_FIELD_FL_VAR) {
5627                                 if (n++)
5628                                         seq_puts(m, ",");
5629                                 hist_field_print(m, field);
5630                         }
5631                 }
5632         }
5633
5634         seq_puts(m, ":sort=");
5635
5636         for (i = 0; i < hist_data->n_sort_keys; i++) {
5637                 struct tracing_map_sort_key *sort_key;
5638                 unsigned int idx, first_key_idx;
5639
5640                 /* skip VAR vals */
5641                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5642
5643                 sort_key = &hist_data->sort_keys[i];
5644                 idx = sort_key->field_idx;
5645
5646                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5647                         return -EINVAL;
5648
5649                 if (i > 0)
5650                         seq_puts(m, ",");
5651
5652                 if (idx == HITCOUNT_IDX)
5653                         seq_puts(m, "hitcount");
5654                 else {
5655                         if (idx >= first_key_idx)
5656                                 idx += hist_data->n_vars;
5657                         hist_field_print(m, hist_data->fields[idx]);
5658                 }
5659
5660                 if (sort_key->descending)
5661                         seq_puts(m, ".descending");
5662         }
5663         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5664         if (hist_data->enable_timestamps)
5665                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5666
5667         print_actions_spec(m, hist_data);
5668
5669         if (data->filter_str)
5670                 seq_printf(m, " if %s", data->filter_str);
5671
5672         if (data->paused)
5673                 seq_puts(m, " [paused]");
5674         else
5675                 seq_puts(m, " [active]");
5676
5677         seq_putc(m, '\n');
5678
5679         return 0;
5680 }
5681
5682 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5683                                    struct event_trigger_data *data)
5684 {
5685         struct hist_trigger_data *hist_data = data->private_data;
5686
5687         if (!data->ref && hist_data->attrs->name)
5688                 save_named_trigger(hist_data->attrs->name, data);
5689
5690         data->ref++;
5691
5692         return 0;
5693 }
5694
5695 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5696 {
5697         struct trace_event_file *file;
5698         unsigned int i;
5699         char *cmd;
5700         int ret;
5701
5702         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5703                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5704                 cmd = hist_data->field_var_hists[i]->cmd;
5705                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5706                                               "!hist", "hist", cmd);
5707                 WARN_ON_ONCE(ret < 0);
5708         }
5709 }
5710
5711 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5712                                     struct event_trigger_data *data)
5713 {
5714         struct hist_trigger_data *hist_data = data->private_data;
5715
5716         if (WARN_ON_ONCE(data->ref <= 0))
5717                 return;
5718
5719         data->ref--;
5720         if (!data->ref) {
5721                 if (data->name)
5722                         del_named_trigger(data);
5723
5724                 trigger_data_free(data);
5725
5726                 remove_hist_vars(hist_data);
5727
5728                 unregister_field_var_hists(hist_data);
5729
5730                 destroy_hist_data(hist_data);
5731         }
5732 }
5733
5734 static struct event_trigger_ops event_hist_trigger_ops = {
5735         .func                   = event_hist_trigger,
5736         .print                  = event_hist_trigger_print,
5737         .init                   = event_hist_trigger_init,
5738         .free                   = event_hist_trigger_free,
5739 };
5740
5741 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5742                                          struct event_trigger_data *data)
5743 {
5744         data->ref++;
5745
5746         save_named_trigger(data->named_data->name, data);
5747
5748         event_hist_trigger_init(ops, data->named_data);
5749
5750         return 0;
5751 }
5752
5753 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5754                                           struct event_trigger_data *data)
5755 {
5756         if (WARN_ON_ONCE(data->ref <= 0))
5757                 return;
5758
5759         event_hist_trigger_free(ops, data->named_data);
5760
5761         data->ref--;
5762         if (!data->ref) {
5763                 del_named_trigger(data);
5764                 trigger_data_free(data);
5765         }
5766 }
5767
5768 static struct event_trigger_ops event_hist_trigger_named_ops = {
5769         .func                   = event_hist_trigger,
5770         .print                  = event_hist_trigger_print,
5771         .init                   = event_hist_trigger_named_init,
5772         .free                   = event_hist_trigger_named_free,
5773 };
5774
5775 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5776                                                             char *param)
5777 {
5778         return &event_hist_trigger_ops;
5779 }
5780
5781 static void hist_clear(struct event_trigger_data *data)
5782 {
5783         struct hist_trigger_data *hist_data = data->private_data;
5784
5785         if (data->name)
5786                 pause_named_trigger(data);
5787
5788         tracepoint_synchronize_unregister();
5789
5790         tracing_map_clear(hist_data->map);
5791
5792         if (data->name)
5793                 unpause_named_trigger(data);
5794 }
5795
5796 static bool compatible_field(struct ftrace_event_field *field,
5797                              struct ftrace_event_field *test_field)
5798 {
5799         if (field == test_field)
5800                 return true;
5801         if (field == NULL || test_field == NULL)
5802                 return false;
5803         if (strcmp(field->name, test_field->name) != 0)
5804                 return false;
5805         if (strcmp(field->type, test_field->type) != 0)
5806                 return false;
5807         if (field->size != test_field->size)
5808                 return false;
5809         if (field->is_signed != test_field->is_signed)
5810                 return false;
5811
5812         return true;
5813 }
5814
5815 static bool hist_trigger_match(struct event_trigger_data *data,
5816                                struct event_trigger_data *data_test,
5817                                struct event_trigger_data *named_data,
5818                                bool ignore_filter)
5819 {
5820         struct tracing_map_sort_key *sort_key, *sort_key_test;
5821         struct hist_trigger_data *hist_data, *hist_data_test;
5822         struct hist_field *key_field, *key_field_test;
5823         unsigned int i;
5824
5825         if (named_data && (named_data != data_test) &&
5826             (named_data != data_test->named_data))
5827                 return false;
5828
5829         if (!named_data && is_named_trigger(data_test))
5830                 return false;
5831
5832         hist_data = data->private_data;
5833         hist_data_test = data_test->private_data;
5834
5835         if (hist_data->n_vals != hist_data_test->n_vals ||
5836             hist_data->n_fields != hist_data_test->n_fields ||
5837             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5838                 return false;
5839
5840         if (!ignore_filter) {
5841                 if ((data->filter_str && !data_test->filter_str) ||
5842                    (!data->filter_str && data_test->filter_str))
5843                         return false;
5844         }
5845
5846         for_each_hist_field(i, hist_data) {
5847                 key_field = hist_data->fields[i];
5848                 key_field_test = hist_data_test->fields[i];
5849
5850                 if (key_field->flags != key_field_test->flags)
5851                         return false;
5852                 if (!compatible_field(key_field->field, key_field_test->field))
5853                         return false;
5854                 if (key_field->offset != key_field_test->offset)
5855                         return false;
5856                 if (key_field->size != key_field_test->size)
5857                         return false;
5858                 if (key_field->is_signed != key_field_test->is_signed)
5859                         return false;
5860                 if (!!key_field->var.name != !!key_field_test->var.name)
5861                         return false;
5862                 if (key_field->var.name &&
5863                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5864                         return false;
5865         }
5866
5867         for (i = 0; i < hist_data->n_sort_keys; i++) {
5868                 sort_key = &hist_data->sort_keys[i];
5869                 sort_key_test = &hist_data_test->sort_keys[i];
5870
5871                 if (sort_key->field_idx != sort_key_test->field_idx ||
5872                     sort_key->descending != sort_key_test->descending)
5873                         return false;
5874         }
5875
5876         if (!ignore_filter && data->filter_str &&
5877             (strcmp(data->filter_str, data_test->filter_str) != 0))
5878                 return false;
5879
5880         if (!actions_match(hist_data, hist_data_test))
5881                 return false;
5882
5883         return true;
5884 }
5885
5886 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5887                                  struct event_trigger_data *data,
5888                                  struct trace_event_file *file)
5889 {
5890         struct hist_trigger_data *hist_data = data->private_data;
5891         struct event_trigger_data *test, *named_data = NULL;
5892         struct trace_array *tr = file->tr;
5893         int ret = 0;
5894
5895         if (hist_data->attrs->name) {
5896                 named_data = find_named_trigger(hist_data->attrs->name);
5897                 if (named_data) {
5898                         if (!hist_trigger_match(data, named_data, named_data,
5899                                                 true)) {
5900                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5901                                 ret = -EINVAL;
5902                                 goto out;
5903                         }
5904                 }
5905         }
5906
5907         if (hist_data->attrs->name && !named_data)
5908                 goto new;
5909
5910         lockdep_assert_held(&event_mutex);
5911
5912         list_for_each_entry(test, &file->triggers, list) {
5913                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5914                         if (!hist_trigger_match(data, test, named_data, false))
5915                                 continue;
5916                         if (hist_data->attrs->pause)
5917                                 test->paused = true;
5918                         else if (hist_data->attrs->cont)
5919                                 test->paused = false;
5920                         else if (hist_data->attrs->clear)
5921                                 hist_clear(test);
5922                         else {
5923                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5924                                 ret = -EEXIST;
5925                         }
5926                         goto out;
5927                 }
5928         }
5929  new:
5930         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5931                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5932                 ret = -ENOENT;
5933                 goto out;
5934         }
5935
5936         if (hist_data->attrs->pause)
5937                 data->paused = true;
5938
5939         if (named_data) {
5940                 data->private_data = named_data->private_data;
5941                 set_named_trigger_data(data, named_data);
5942                 data->ops = &event_hist_trigger_named_ops;
5943         }
5944
5945         if (data->ops->init) {
5946                 ret = data->ops->init(data->ops, data);
5947                 if (ret < 0)
5948                         goto out;
5949         }
5950
5951         if (hist_data->enable_timestamps) {
5952                 char *clock = hist_data->attrs->clock;
5953
5954                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5955                 if (ret) {
5956                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5957                         goto out;
5958                 }
5959
5960                 tracing_set_filter_buffering(file->tr, true);
5961         }
5962
5963         if (named_data)
5964                 destroy_hist_data(hist_data);
5965
5966         ret++;
5967  out:
5968         return ret;
5969 }
5970
5971 static int hist_trigger_enable(struct event_trigger_data *data,
5972                                struct trace_event_file *file)
5973 {
5974         int ret = 0;
5975
5976         list_add_tail_rcu(&data->list, &file->triggers);
5977
5978         update_cond_flag(file);
5979
5980         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5981                 list_del_rcu(&data->list);
5982                 update_cond_flag(file);
5983                 ret--;
5984         }
5985
5986         return ret;
5987 }
5988
5989 static bool have_hist_trigger_match(struct event_trigger_data *data,
5990                                     struct trace_event_file *file)
5991 {
5992         struct hist_trigger_data *hist_data = data->private_data;
5993         struct event_trigger_data *test, *named_data = NULL;
5994         bool match = false;
5995
5996         lockdep_assert_held(&event_mutex);
5997
5998         if (hist_data->attrs->name)
5999                 named_data = find_named_trigger(hist_data->attrs->name);
6000
6001         list_for_each_entry(test, &file->triggers, list) {
6002                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6003                         if (hist_trigger_match(data, test, named_data, false)) {
6004                                 match = true;
6005                                 break;
6006                         }
6007                 }
6008         }
6009
6010         return match;
6011 }
6012
6013 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6014                                     struct trace_event_file *file)
6015 {
6016         struct hist_trigger_data *hist_data = data->private_data;
6017         struct event_trigger_data *test, *named_data = NULL;
6018
6019         lockdep_assert_held(&event_mutex);
6020
6021         if (hist_data->attrs->name)
6022                 named_data = find_named_trigger(hist_data->attrs->name);
6023
6024         list_for_each_entry(test, &file->triggers, list) {
6025                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6026                         if (!hist_trigger_match(data, test, named_data, false))
6027                                 continue;
6028                         hist_data = test->private_data;
6029                         if (check_var_refs(hist_data))
6030                                 return true;
6031                         break;
6032                 }
6033         }
6034
6035         return false;
6036 }
6037
6038 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6039                                     struct event_trigger_data *data,
6040                                     struct trace_event_file *file)
6041 {
6042         struct hist_trigger_data *hist_data = data->private_data;
6043         struct event_trigger_data *test, *named_data = NULL;
6044         bool unregistered = false;
6045
6046         lockdep_assert_held(&event_mutex);
6047
6048         if (hist_data->attrs->name)
6049                 named_data = find_named_trigger(hist_data->attrs->name);
6050
6051         list_for_each_entry(test, &file->triggers, list) {
6052                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6053                         if (!hist_trigger_match(data, test, named_data, false))
6054                                 continue;
6055                         unregistered = true;
6056                         list_del_rcu(&test->list);
6057                         trace_event_trigger_enable_disable(file, 0);
6058                         update_cond_flag(file);
6059                         break;
6060                 }
6061         }
6062
6063         if (unregistered && test->ops->free)
6064                 test->ops->free(test->ops, test);
6065
6066         if (hist_data->enable_timestamps) {
6067                 if (!hist_data->remove || unregistered)
6068                         tracing_set_filter_buffering(file->tr, false);
6069         }
6070 }
6071
6072 static bool hist_file_check_refs(struct trace_event_file *file)
6073 {
6074         struct hist_trigger_data *hist_data;
6075         struct event_trigger_data *test;
6076
6077         lockdep_assert_held(&event_mutex);
6078
6079         list_for_each_entry(test, &file->triggers, list) {
6080                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6081                         hist_data = test->private_data;
6082                         if (check_var_refs(hist_data))
6083                                 return true;
6084                 }
6085         }
6086
6087         return false;
6088 }
6089
6090 static void hist_unreg_all(struct trace_event_file *file)
6091 {
6092         struct event_trigger_data *test, *n;
6093         struct hist_trigger_data *hist_data;
6094         struct synth_event *se;
6095         const char *se_name;
6096
6097         lockdep_assert_held(&event_mutex);
6098
6099         if (hist_file_check_refs(file))
6100                 return;
6101
6102         list_for_each_entry_safe(test, n, &file->triggers, list) {
6103                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6104                         hist_data = test->private_data;
6105                         list_del_rcu(&test->list);
6106                         trace_event_trigger_enable_disable(file, 0);
6107
6108                         se_name = trace_event_name(file->event_call);
6109                         se = find_synth_event(se_name);
6110                         if (se)
6111                                 se->ref--;
6112
6113                         update_cond_flag(file);
6114                         if (hist_data->enable_timestamps)
6115                                 tracing_set_filter_buffering(file->tr, false);
6116                         if (test->ops->free)
6117                                 test->ops->free(test->ops, test);
6118                 }
6119         }
6120 }
6121
6122 static int event_hist_trigger_func(struct event_command *cmd_ops,
6123                                    struct trace_event_file *file,
6124                                    char *glob, char *cmd, char *param)
6125 {
6126         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6127         struct event_trigger_data *trigger_data;
6128         struct hist_trigger_attrs *attrs;
6129         struct event_trigger_ops *trigger_ops;
6130         struct hist_trigger_data *hist_data;
6131         struct synth_event *se;
6132         const char *se_name;
6133         bool remove = false;
6134         char *trigger, *p, *start;
6135         int ret = 0;
6136
6137         lockdep_assert_held(&event_mutex);
6138
6139         if (glob && strlen(glob)) {
6140                 hist_err_clear();
6141                 last_cmd_set(file, param);
6142         }
6143
6144         if (!param)
6145                 return -EINVAL;
6146
6147         if (glob[0] == '!')
6148                 remove = true;
6149
6150         /*
6151          * separate the trigger from the filter (k:v [if filter])
6152          * allowing for whitespace in the trigger
6153          */
6154         p = trigger = param;
6155         do {
6156                 p = strstr(p, "if");
6157                 if (!p)
6158                         break;
6159                 if (p == param)
6160                         return -EINVAL;
6161                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6162                         p++;
6163                         continue;
6164                 }
6165                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6166                         return -EINVAL;
6167                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6168                         p++;
6169                         continue;
6170                 }
6171                 break;
6172         } while (p);
6173
6174         if (!p)
6175                 param = NULL;
6176         else {
6177                 *(p - 1) = '\0';
6178                 param = strstrip(p);
6179                 trigger = strstrip(trigger);
6180         }
6181
6182         /*
6183          * To simplify arithmetic expression parsing, replace occurrences of
6184          * '.sym-offset' modifier with '.symXoffset'
6185          */
6186         start = strstr(trigger, ".sym-offset");
6187         while (start) {
6188                 *(start + 4) = 'X';
6189                 start = strstr(start + 11, ".sym-offset");
6190         }
6191
6192         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6193         if (IS_ERR(attrs))
6194                 return PTR_ERR(attrs);
6195
6196         if (attrs->map_bits)
6197                 hist_trigger_bits = attrs->map_bits;
6198
6199         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6200         if (IS_ERR(hist_data)) {
6201                 destroy_hist_trigger_attrs(attrs);
6202                 return PTR_ERR(hist_data);
6203         }
6204
6205         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6206
6207         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6208         if (!trigger_data) {
6209                 ret = -ENOMEM;
6210                 goto out_free;
6211         }
6212
6213         trigger_data->count = -1;
6214         trigger_data->ops = trigger_ops;
6215         trigger_data->cmd_ops = cmd_ops;
6216
6217         INIT_LIST_HEAD(&trigger_data->list);
6218         RCU_INIT_POINTER(trigger_data->filter, NULL);
6219
6220         trigger_data->private_data = hist_data;
6221
6222         /* if param is non-empty, it's supposed to be a filter */
6223         if (param && cmd_ops->set_filter) {
6224                 ret = cmd_ops->set_filter(param, trigger_data, file);
6225                 if (ret < 0)
6226                         goto out_free;
6227         }
6228
6229         if (remove) {
6230                 if (!have_hist_trigger_match(trigger_data, file))
6231                         goto out_free;
6232
6233                 if (hist_trigger_check_refs(trigger_data, file)) {
6234                         ret = -EBUSY;
6235                         goto out_free;
6236                 }
6237
6238                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6239                 se_name = trace_event_name(file->event_call);
6240                 se = find_synth_event(se_name);
6241                 if (se)
6242                         se->ref--;
6243                 ret = 0;
6244                 goto out_free;
6245         }
6246
6247         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6248         /*
6249          * The above returns on success the # of triggers registered,
6250          * but if it didn't register any it returns zero.  Consider no
6251          * triggers registered a failure too.
6252          */
6253         if (!ret) {
6254                 if (!(attrs->pause || attrs->cont || attrs->clear))
6255                         ret = -ENOENT;
6256                 goto out_free;
6257         } else if (ret < 0)
6258                 goto out_free;
6259
6260         if (get_named_trigger_data(trigger_data))
6261                 goto enable;
6262
6263         if (has_hist_vars(hist_data))
6264                 save_hist_vars(hist_data);
6265
6266         ret = create_actions(hist_data);
6267         if (ret)
6268                 goto out_unreg;
6269
6270         ret = tracing_map_init(hist_data->map);
6271         if (ret)
6272                 goto out_unreg;
6273 enable:
6274         ret = hist_trigger_enable(trigger_data, file);
6275         if (ret)
6276                 goto out_unreg;
6277
6278         se_name = trace_event_name(file->event_call);
6279         se = find_synth_event(se_name);
6280         if (se)
6281                 se->ref++;
6282         /* Just return zero, not the number of registered triggers */
6283         ret = 0;
6284  out:
6285         if (ret == 0)
6286                 hist_err_clear();
6287
6288         return ret;
6289  out_unreg:
6290         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6291  out_free:
6292         if (cmd_ops->set_filter)
6293                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6294
6295         remove_hist_vars(hist_data);
6296
6297         kfree(trigger_data);
6298
6299         destroy_hist_data(hist_data);
6300         goto out;
6301 }
6302
6303 static struct event_command trigger_hist_cmd = {
6304         .name                   = "hist",
6305         .trigger_type           = ETT_EVENT_HIST,
6306         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6307         .func                   = event_hist_trigger_func,
6308         .reg                    = hist_register_trigger,
6309         .unreg                  = hist_unregister_trigger,
6310         .unreg_all              = hist_unreg_all,
6311         .get_trigger_ops        = event_hist_get_trigger_ops,
6312         .set_filter             = set_trigger_filter,
6313 };
6314
6315 __init int register_trigger_hist_cmd(void)
6316 {
6317         int ret;
6318
6319         ret = register_event_command(&trigger_hist_cmd);
6320         WARN_ON(ret < 0);
6321
6322         return ret;
6323 }
6324
6325 static void
6326 hist_enable_trigger(struct event_trigger_data *data,
6327                     struct trace_buffer *buffer,  void *rec,
6328                     struct ring_buffer_event *event)
6329 {
6330         struct enable_trigger_data *enable_data = data->private_data;
6331         struct event_trigger_data *test;
6332
6333         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6334                                 lockdep_is_held(&event_mutex)) {
6335                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6336                         if (enable_data->enable)
6337                                 test->paused = false;
6338                         else
6339                                 test->paused = true;
6340                 }
6341         }
6342 }
6343
6344 static void
6345 hist_enable_count_trigger(struct event_trigger_data *data,
6346                           struct trace_buffer *buffer,  void *rec,
6347                           struct ring_buffer_event *event)
6348 {
6349         if (!data->count)
6350                 return;
6351
6352         if (data->count != -1)
6353                 (data->count)--;
6354
6355         hist_enable_trigger(data, buffer, rec, event);
6356 }
6357
6358 static struct event_trigger_ops hist_enable_trigger_ops = {
6359         .func                   = hist_enable_trigger,
6360         .print                  = event_enable_trigger_print,
6361         .init                   = event_trigger_init,
6362         .free                   = event_enable_trigger_free,
6363 };
6364
6365 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6366         .func                   = hist_enable_count_trigger,
6367         .print                  = event_enable_trigger_print,
6368         .init                   = event_trigger_init,
6369         .free                   = event_enable_trigger_free,
6370 };
6371
6372 static struct event_trigger_ops hist_disable_trigger_ops = {
6373         .func                   = hist_enable_trigger,
6374         .print                  = event_enable_trigger_print,
6375         .init                   = event_trigger_init,
6376         .free                   = event_enable_trigger_free,
6377 };
6378
6379 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6380         .func                   = hist_enable_count_trigger,
6381         .print                  = event_enable_trigger_print,
6382         .init                   = event_trigger_init,
6383         .free                   = event_enable_trigger_free,
6384 };
6385
6386 static struct event_trigger_ops *
6387 hist_enable_get_trigger_ops(char *cmd, char *param)
6388 {
6389         struct event_trigger_ops *ops;
6390         bool enable;
6391
6392         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6393
6394         if (enable)
6395                 ops = param ? &hist_enable_count_trigger_ops :
6396                         &hist_enable_trigger_ops;
6397         else
6398                 ops = param ? &hist_disable_count_trigger_ops :
6399                         &hist_disable_trigger_ops;
6400
6401         return ops;
6402 }
6403
6404 static void hist_enable_unreg_all(struct trace_event_file *file)
6405 {
6406         struct event_trigger_data *test, *n;
6407
6408         list_for_each_entry_safe(test, n, &file->triggers, list) {
6409                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6410                         list_del_rcu(&test->list);
6411                         update_cond_flag(file);
6412                         trace_event_trigger_enable_disable(file, 0);
6413                         if (test->ops->free)
6414                                 test->ops->free(test->ops, test);
6415                 }
6416         }
6417 }
6418
6419 static struct event_command trigger_hist_enable_cmd = {
6420         .name                   = ENABLE_HIST_STR,
6421         .trigger_type           = ETT_HIST_ENABLE,
6422         .func                   = event_enable_trigger_func,
6423         .reg                    = event_enable_register_trigger,
6424         .unreg                  = event_enable_unregister_trigger,
6425         .unreg_all              = hist_enable_unreg_all,
6426         .get_trigger_ops        = hist_enable_get_trigger_ops,
6427         .set_filter             = set_trigger_filter,
6428 };
6429
6430 static struct event_command trigger_hist_disable_cmd = {
6431         .name                   = DISABLE_HIST_STR,
6432         .trigger_type           = ETT_HIST_ENABLE,
6433         .func                   = event_enable_trigger_func,
6434         .reg                    = event_enable_register_trigger,
6435         .unreg                  = event_enable_unregister_trigger,
6436         .unreg_all              = hist_enable_unreg_all,
6437         .get_trigger_ops        = hist_enable_get_trigger_ops,
6438         .set_filter             = set_trigger_filter,
6439 };
6440
6441 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6442 {
6443         unregister_event_command(&trigger_hist_enable_cmd);
6444         unregister_event_command(&trigger_hist_disable_cmd);
6445 }
6446
6447 __init int register_trigger_hist_enable_disable_cmds(void)
6448 {
6449         int ret;
6450
6451         ret = register_event_command(&trigger_hist_enable_cmd);
6452         if (WARN_ON(ret < 0))
6453                 return ret;
6454         ret = register_event_command(&trigger_hist_disable_cmd);
6455         if (WARN_ON(ret < 0))
6456                 unregister_trigger_hist_enable_disable_cmds();
6457
6458         return ret;
6459 }