kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),
  73
  74 #undef C
  75 #define C(a, b)         HIST_ERR_##a
  76
  77 enum { ERRORS };
  78
  79 #undef C
  80 #define C(a, b)         b
  81
  82 static const char *err_text[] = { ERRORS };
  83
  84 struct hist_field;
  85
  86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  87                                 struct tracing_map_elt *elt,
  88                                 struct trace_buffer *buffer,
  89                                 struct ring_buffer_event *rbe,
  90                                 void *event);
  91
  92 #define HIST_FIELD_OPERANDS_MAX 2
  93 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  94 #define HIST_ACTIONS_MAX        8
  95 #define HIST_CONST_DIGITS_MAX   21
  96 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  97
  98 enum field_op_id {
  99         FIELD_OP_NONE,
 100         FIELD_OP_PLUS,
 101         FIELD_OP_MINUS,
 102         FIELD_OP_UNARY_MINUS,
 103         FIELD_OP_DIV,
 104         FIELD_OP_MULT,
 105 };
 106
 107 /*
 108  * A hist_var (histogram variable) contains variable information for
 109  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 110  * flag set.  A hist_var has a variable name e.g. ts0, and is
 111  * associated with a given histogram trigger, as specified by
 112  * hist_data.  The hist_var idx is the unique index assigned to the
 113  * variable by the hist trigger's tracing_map.  The idx is what is
 114  * used to set a variable's value and, by a variable reference, to
 115  * retrieve it.
 116  */
 117 struct hist_var {
 118         char                            *name;
 119         struct hist_trigger_data        *hist_data;
 120         unsigned int                    idx;
 121 };
 122
 123 struct hist_field {
 124         struct ftrace_event_field       *field;
 125         unsigned long                   flags;
 126         hist_field_fn_t                 fn;
 127         unsigned int                    ref;
 128         unsigned int                    size;
 129         unsigned int                    offset;
 130         unsigned int                    is_signed;
 131         unsigned long                   buckets;
 132         const char                      *type;
 133         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 134         struct hist_trigger_data        *hist_data;
 135
 136         /*
 137          * Variable fields contain variable-specific info in var.
 138          */
 139         struct hist_var                 var;
 140         enum field_op_id                operator;
 141         char                            *system;
 142         char                            *event_name;
 143
 144         /*
 145          * The name field is used for EXPR and VAR_REF fields.  VAR
 146          * fields contain the variable name in var.name.
 147          */
 148         char                            *name;
 149
 150         /*
 151          * When a histogram trigger is hit, if it has any references
 152          * to variables, the values of those variables are collected
 153          * into a var_ref_vals array by resolve_var_refs().  The
 154          * current value of each variable is read from the tracing_map
 155          * using the hist field's hist_var.idx and entered into the
 156          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 157          */
 158         unsigned int                    var_ref_idx;
 159         bool                            read_once;
 160
 161         unsigned int                    var_str_idx;
 162
 163         /* Numeric literals are represented as u64 */
 164         u64                             constant;
 165         /* Used to optimize division by constants */
 166         u64                             div_multiplier;
 167 };
 168
 169 static u64 hist_field_none(struct hist_field *field,
 170                            struct tracing_map_elt *elt,
 171                            struct trace_buffer *buffer,
 172                            struct ring_buffer_event *rbe,
 173                            void *event)
 174 {
 175         return 0;
 176 }
 177
 178 static u64 hist_field_const(struct hist_field *field,
 179                            struct tracing_map_elt *elt,
 180                            struct trace_buffer *buffer,
 181                            struct ring_buffer_event *rbe,
 182                            void *event)
 183 {
 184         return field->constant;
 185 }
 186
 187 static u64 hist_field_counter(struct hist_field *field,
 188                               struct tracing_map_elt *elt,
 189                               struct trace_buffer *buffer,
 190                               struct ring_buffer_event *rbe,
 191                               void *event)
 192 {
 193         return 1;
 194 }
 195
 196 static u64 hist_field_string(struct hist_field *hist_field,
 197                              struct tracing_map_elt *elt,
 198                              struct trace_buffer *buffer,
 199                              struct ring_buffer_event *rbe,
 200                              void *event)
 201 {
 202         char *addr = (char *)(event + hist_field->field->offset);
 203
 204         return (u64)(unsigned long)addr;
 205 }
 206
 207 static u64 hist_field_dynstring(struct hist_field *hist_field,
 208                                 struct tracing_map_elt *elt,
 209                                 struct trace_buffer *buffer,
 210                                 struct ring_buffer_event *rbe,
 211                                 void *event)
 212 {
 213         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 214         int str_loc = str_item & 0xffff;
 215         char *addr = (char *)(event + str_loc);
 216
 217         return (u64)(unsigned long)addr;
 218 }
 219
 220 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 221                                    struct tracing_map_elt *elt,
 222                                    struct trace_buffer *buffer,
 223                                    struct ring_buffer_event *rbe,
 224                                    void *event)
 225 {
 226         u32 *item = event + hist_field->field->offset;
 227         u32 str_item = *item;
 228         int str_loc = str_item & 0xffff;
 229         char *addr = (char *)&item[1] + str_loc;
 230
 231         return (u64)(unsigned long)addr;
 232 }
 233
 234 static u64 hist_field_pstring(struct hist_field *hist_field,
 235                               struct tracing_map_elt *elt,
 236                               struct trace_buffer *buffer,
 237                               struct ring_buffer_event *rbe,
 238                               void *event)
 239 {
 240         char **addr = (char **)(event + hist_field->field->offset);
 241
 242         return (u64)(unsigned long)*addr;
 243 }
 244
 245 static u64 hist_field_log2(struct hist_field *hist_field,
 246                            struct tracing_map_elt *elt,
 247                            struct trace_buffer *buffer,
 248                            struct ring_buffer_event *rbe,
 249                            void *event)
 250 {
 251         struct hist_field *operand = hist_field->operands[0];
 252
 253         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 254
 255         return (u64) ilog2(roundup_pow_of_two(val));
 256 }
 257
 258 static u64 hist_field_bucket(struct hist_field *hist_field,
 259                              struct tracing_map_elt *elt,
 260                              struct trace_buffer *buffer,
 261                              struct ring_buffer_event *rbe,
 262                              void *event)
 263 {
 264         struct hist_field *operand = hist_field->operands[0];
 265         unsigned long buckets = hist_field->buckets;
 266
 267         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 268
 269         if (WARN_ON_ONCE(!buckets))
 270                 return val;
 271
 272         if (val >= LONG_MAX)
 273                 val = div64_ul(val, buckets);
 274         else
 275                 val = (u64)((unsigned long)val / buckets);
 276         return val * buckets;
 277 }
 278
 279 static u64 hist_field_plus(struct hist_field *hist_field,
 280                            struct tracing_map_elt *elt,
 281                            struct trace_buffer *buffer,
 282                            struct ring_buffer_event *rbe,
 283                            void *event)
 284 {
 285         struct hist_field *operand1 = hist_field->operands[0];
 286         struct hist_field *operand2 = hist_field->operands[1];
 287
 288         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 289         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 290
 291         return val1 + val2;
 292 }
 293
 294 static u64 hist_field_minus(struct hist_field *hist_field,
 295                             struct tracing_map_elt *elt,
 296                             struct trace_buffer *buffer,
 297                             struct ring_buffer_event *rbe,
 298                             void *event)
 299 {
 300         struct hist_field *operand1 = hist_field->operands[0];
 301         struct hist_field *operand2 = hist_field->operands[1];
 302
 303         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 304         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 305
 306         return val1 - val2;
 307 }
 308
 309 static u64 hist_field_div(struct hist_field *hist_field,
 310                            struct tracing_map_elt *elt,
 311                            struct trace_buffer *buffer,
 312                            struct ring_buffer_event *rbe,
 313                            void *event)
 314 {
 315         struct hist_field *operand1 = hist_field->operands[0];
 316         struct hist_field *operand2 = hist_field->operands[1];
 317
 318         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 319         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 320
 321         /* Return -1 for the undefined case */
 322         if (!val2)
 323                 return -1;
 324
 325         /* Use shift if the divisor is a power of 2 */
 326         if (!(val2 & (val2 - 1)))
 327                 return val1 >> __ffs64(val2);
 328
 329         return div64_u64(val1, val2);
 330 }
 331
 332 static u64 div_by_power_of_two(struct hist_field *hist_field,
 333                                 struct tracing_map_elt *elt,
 334                                 struct trace_buffer *buffer,
 335                                 struct ring_buffer_event *rbe,
 336                                 void *event)
 337 {
 338         struct hist_field *operand1 = hist_field->operands[0];
 339         struct hist_field *operand2 = hist_field->operands[1];
 340
 341         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 342
 343         return val1 >> __ffs64(operand2->constant);
 344 }
 345
 346 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 347                                 struct tracing_map_elt *elt,
 348                                 struct trace_buffer *buffer,
 349                                 struct ring_buffer_event *rbe,
 350                                 void *event)
 351 {
 352         struct hist_field *operand1 = hist_field->operands[0];
 353         struct hist_field *operand2 = hist_field->operands[1];
 354
 355         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 356
 357         return div64_u64(val1, operand2->constant);
 358 }
 359
 360 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 361                                 struct tracing_map_elt *elt,
 362                                 struct trace_buffer *buffer,
 363                                 struct ring_buffer_event *rbe,
 364                                 void *event)
 365 {
 366         struct hist_field *operand1 = hist_field->operands[0];
 367         struct hist_field *operand2 = hist_field->operands[1];
 368
 369         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 370
 371         /*
 372          * If the divisor is a constant, do a multiplication and shift instead.
 373          *
 374          * Choose Z = some power of 2. If Y <= Z, then:
 375          *     X / Y = (X * (Z / Y)) / Z
 376          *
 377          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 378          *     X / Y = (X * mult) / Z
 379          *
 380          * The division by Z can be replaced by a shift since Z is a power of 2:
 381          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 382          *
 383          * As long, as X < Z the results will not be off by more than 1.
 384          */
 385         if (val1 < (1 << HIST_DIV_SHIFT)) {
 386                 u64 mult = operand2->div_multiplier;
 387
 388                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 389         }
 390
 391         return div64_u64(val1, operand2->constant);
 392 }
 393
 394 static u64 hist_field_mult(struct hist_field *hist_field,
 395                            struct tracing_map_elt *elt,
 396                            struct trace_buffer *buffer,
 397                            struct ring_buffer_event *rbe,
 398                            void *event)
 399 {
 400         struct hist_field *operand1 = hist_field->operands[0];
 401         struct hist_field *operand2 = hist_field->operands[1];
 402
 403         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 404         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 405
 406         return val1 * val2;
 407 }
 408
 409 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 410                                   struct tracing_map_elt *elt,
 411                                   struct trace_buffer *buffer,
 412                                   struct ring_buffer_event *rbe,
 413                                   void *event)
 414 {
 415         struct hist_field *operand = hist_field->operands[0];
 416
 417         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 418         u64 val = (u64)-sval;
 419
 420         return val;
 421 }
 422
 423 #define DEFINE_HIST_FIELD_FN(type)                                      \
 424         static u64 hist_field_##type(struct hist_field *hist_field,     \
 425                                      struct tracing_map_elt *elt,       \
 426                                      struct trace_buffer *buffer,       \
 427                                      struct ring_buffer_event *rbe,     \
 428                                      void *event)                       \
 429 {                                                                       \
 430         type *addr = (type *)(event + hist_field->field->offset);       \
 431                                                                         \
 432         return (u64)(unsigned long)*addr;                               \
 433 }
 434
 435 DEFINE_HIST_FIELD_FN(s64);
 436 DEFINE_HIST_FIELD_FN(u64);
 437 DEFINE_HIST_FIELD_FN(s32);
 438 DEFINE_HIST_FIELD_FN(u32);
 439 DEFINE_HIST_FIELD_FN(s16);
 440 DEFINE_HIST_FIELD_FN(u16);
 441 DEFINE_HIST_FIELD_FN(s8);
 442 DEFINE_HIST_FIELD_FN(u8);
 443
 444 #define for_each_hist_field(i, hist_data)       \
 445         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 446
 447 #define for_each_hist_val_field(i, hist_data)   \
 448         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 449
 450 #define for_each_hist_key_field(i, hist_data)   \
 451         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 452
 453 #define HIST_STACKTRACE_DEPTH   16
 454 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 455 #define HIST_STACKTRACE_SKIP    5
 456
 457 #define HITCOUNT_IDX            0
 458 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 459
 460 enum hist_field_flags {
 461         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 462         HIST_FIELD_FL_KEY               = 1 << 1,
 463         HIST_FIELD_FL_STRING            = 1 << 2,
 464         HIST_FIELD_FL_HEX               = 1 << 3,
 465         HIST_FIELD_FL_SYM               = 1 << 4,
 466         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 467         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 468         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 469         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 470         HIST_FIELD_FL_LOG2              = 1 << 9,
 471         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 472         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 473         HIST_FIELD_FL_VAR               = 1 << 12,
 474         HIST_FIELD_FL_EXPR              = 1 << 13,
 475         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 476         HIST_FIELD_FL_CPU               = 1 << 15,
 477         HIST_FIELD_FL_ALIAS             = 1 << 16,
 478         HIST_FIELD_FL_BUCKET            = 1 << 17,
 479         HIST_FIELD_FL_CONST             = 1 << 18,
 480 };
 481
 482 struct var_defs {
 483         unsigned int    n_vars;
 484         char            *name[TRACING_MAP_VARS_MAX];
 485         char            *expr[TRACING_MAP_VARS_MAX];
 486 };
 487
 488 struct hist_trigger_attrs {
 489         char            *keys_str;
 490         char            *vals_str;
 491         char            *sort_key_str;
 492         char            *name;
 493         char            *clock;
 494         bool            pause;
 495         bool            cont;
 496         bool            clear;
 497         bool            ts_in_usecs;
 498         unsigned int    map_bits;
 499
 500         char            *assignment_str[TRACING_MAP_VARS_MAX];
 501         unsigned int    n_assignments;
 502
 503         char            *action_str[HIST_ACTIONS_MAX];
 504         unsigned int    n_actions;
 505
 506         struct var_defs var_defs;
 507 };
 508
 509 struct field_var {
 510         struct hist_field       *var;
 511         struct hist_field       *val;
 512 };
 513
 514 struct field_var_hist {
 515         struct hist_trigger_data        *hist_data;
 516         char                            *cmd;
 517 };
 518
 519 struct hist_trigger_data {
 520         struct hist_field               *fields[HIST_FIELDS_MAX];
 521         unsigned int                    n_vals;
 522         unsigned int                    n_keys;
 523         unsigned int                    n_fields;
 524         unsigned int                    n_vars;
 525         unsigned int                    n_var_str;
 526         unsigned int                    key_size;
 527         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 528         unsigned int                    n_sort_keys;
 529         struct trace_event_file         *event_file;
 530         struct hist_trigger_attrs       *attrs;
 531         struct tracing_map              *map;
 532         bool                            enable_timestamps;
 533         bool                            remove;
 534         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 535         unsigned int                    n_var_refs;
 536
 537         struct action_data              *actions[HIST_ACTIONS_MAX];
 538         unsigned int                    n_actions;
 539
 540         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 541         unsigned int                    n_field_vars;
 542         unsigned int                    n_field_var_str;
 543         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 544         unsigned int                    n_field_var_hists;
 545
 546         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 547         unsigned int                    n_save_vars;
 548         unsigned int                    n_save_var_str;
 549 };
 550
 551 struct action_data;
 552
 553 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 554                              struct tracing_map_elt *elt,
 555                              struct trace_buffer *buffer, void *rec,
 556                              struct ring_buffer_event *rbe, void *key,
 557                              struct action_data *data, u64 *var_ref_vals);
 558
 559 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 560
 561 enum handler_id {
 562         HANDLER_ONMATCH = 1,
 563         HANDLER_ONMAX,
 564         HANDLER_ONCHANGE,
 565 };
 566
 567 enum action_id {
 568         ACTION_SAVE = 1,
 569         ACTION_TRACE,
 570         ACTION_SNAPSHOT,
 571 };
 572
 573 struct action_data {
 574         enum handler_id         handler;
 575         enum action_id          action;
 576         char                    *action_name;
 577         action_fn_t             fn;
 578
 579         unsigned int            n_params;
 580         char                    *params[SYNTH_FIELDS_MAX];
 581
 582         /*
 583          * When a histogram trigger is hit, the values of any
 584          * references to variables, including variables being passed
 585          * as parameters to synthetic events, are collected into a
 586          * var_ref_vals array.  This var_ref_idx array is an array of
 587          * indices into the var_ref_vals array, one for each synthetic
 588          * event param, and is passed to the synthetic event
 589          * invocation.
 590          */
 591         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 592         struct synth_event      *synth_event;
 593         bool                    use_trace_keyword;
 594         char                    *synth_event_name;
 595
 596         union {
 597                 struct {
 598                         char                    *event;
 599                         char                    *event_system;
 600                 } match_data;
 601
 602                 struct {
 603                         /*
 604                          * var_str contains the $-unstripped variable
 605                          * name referenced by var_ref, and used when
 606                          * printing the action.  Because var_ref
 607                          * creation is deferred to create_actions(),
 608                          * we need a per-action way to save it until
 609                          * then, thus var_str.
 610                          */
 611                         char                    *var_str;
 612
 613                         /*
 614                          * var_ref refers to the variable being
 615                          * tracked e.g onmax($var).
 616                          */
 617                         struct hist_field       *var_ref;
 618
 619                         /*
 620                          * track_var contains the 'invisible' tracking
 621                          * variable created to keep the current
 622                          * e.g. max value.
 623                          */
 624                         struct hist_field       *track_var;
 625
 626                         check_track_val_fn_t    check_val;
 627                         action_fn_t             save_data;
 628                 } track_data;
 629         };
 630 };
 631
 632 struct track_data {
 633         u64                             track_val;
 634         bool                            updated;
 635
 636         unsigned int                    key_len;
 637         void                            *key;
 638         struct tracing_map_elt          elt;
 639
 640         struct action_data              *action_data;
 641         struct hist_trigger_data        *hist_data;
 642 };
 643
 644 struct hist_elt_data {
 645         char *comm;
 646         u64 *var_ref_vals;
 647         char **field_var_str;
 648         int n_field_var_str;
 649 };
 650
 651 struct snapshot_context {
 652         struct tracing_map_elt  *elt;
 653         void                    *key;
 654 };
 655
 656 /*
 657  * Returns the specific division function to use if the divisor
 658  * is constant. This avoids extra branches when the trigger is hit.
 659  */
 660 static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
 661 {
 662         u64 div = divisor->constant;
 663
 664         if (!(div & (div - 1)))
 665                 return div_by_power_of_two;
 666
 667         /* If the divisor is too large, do a regular division */
 668         if (div > (1 << HIST_DIV_SHIFT))
 669                 return div_by_not_power_of_two;
 670
 671         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 672         return div_by_mult_and_shift;
 673 }
 674
 675 static void track_data_free(struct track_data *track_data)
 676 {
 677         struct hist_elt_data *elt_data;
 678
 679         if (!track_data)
 680                 return;
 681
 682         kfree(track_data->key);
 683
 684         elt_data = track_data->elt.private_data;
 685         if (elt_data) {
 686                 kfree(elt_data->comm);
 687                 kfree(elt_data);
 688         }
 689
 690         kfree(track_data);
 691 }
 692
 693 static struct track_data *track_data_alloc(unsigned int key_len,
 694                                            struct action_data *action_data,
 695                                            struct hist_trigger_data *hist_data)
 696 {
 697         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 698         struct hist_elt_data *elt_data;
 699
 700         if (!data)
 701                 return ERR_PTR(-ENOMEM);
 702
 703         data->key = kzalloc(key_len, GFP_KERNEL);
 704         if (!data->key) {
 705                 track_data_free(data);
 706                 return ERR_PTR(-ENOMEM);
 707         }
 708
 709         data->key_len = key_len;
 710         data->action_data = action_data;
 711         data->hist_data = hist_data;
 712
 713         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 714         if (!elt_data) {
 715                 track_data_free(data);
 716                 return ERR_PTR(-ENOMEM);
 717         }
 718
 719         data->elt.private_data = elt_data;
 720
 721         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 722         if (!elt_data->comm) {
 723                 track_data_free(data);
 724                 return ERR_PTR(-ENOMEM);
 725         }
 726
 727         return data;
 728 }
 729
 730 static char last_cmd[MAX_FILTER_STR_VAL];
 731 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 732
 733 static int errpos(char *str)
 734 {
 735         return err_pos(last_cmd, str);
 736 }
 737
 738 static void last_cmd_set(struct trace_event_file *file, char *str)
 739 {
 740         const char *system = NULL, *name = NULL;
 741         struct trace_event_call *call;
 742
 743         if (!str)
 744                 return;
 745
 746         strcpy(last_cmd, "hist:");
 747         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 748
 749         if (file) {
 750                 call = file->event_call;
 751                 system = call->class->system;
 752                 if (system) {
 753                         name = trace_event_name(call);
 754                         if (!name)
 755                                 system = NULL;
 756                 }
 757         }
 758
 759         if (system)
 760                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 761 }
 762
 763 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 764 {
 765         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 766                         err_type, err_pos);
 767 }
 768
 769 static void hist_err_clear(void)
 770 {
 771         last_cmd[0] = '\0';
 772         last_cmd_loc[0] = '\0';
 773 }
 774
 775 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 776                                     unsigned int *var_ref_idx);
 777
 778 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 779                                unsigned int *var_ref_idx)
 780 {
 781         struct tracepoint *tp = event->tp;
 782
 783         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 784                 struct tracepoint_func *probe_func_ptr;
 785                 synth_probe_func_t probe_func;
 786                 void *__data;
 787
 788                 if (!(cpu_online(raw_smp_processor_id())))
 789                         return;
 790
 791                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 792                 if (probe_func_ptr) {
 793                         do {
 794                                 probe_func = probe_func_ptr->func;
 795                                 __data = probe_func_ptr->data;
 796                                 probe_func(__data, var_ref_vals, var_ref_idx);
 797                         } while ((++probe_func_ptr)->func);
 798                 }
 799         }
 800 }
 801
 802 static void action_trace(struct hist_trigger_data *hist_data,
 803                          struct tracing_map_elt *elt,
 804                          struct trace_buffer *buffer, void *rec,
 805                          struct ring_buffer_event *rbe, void *key,
 806                          struct action_data *data, u64 *var_ref_vals)
 807 {
 808         struct synth_event *event = data->synth_event;
 809
 810         trace_synth(event, var_ref_vals, data->var_ref_idx);
 811 }
 812
 813 struct hist_var_data {
 814         struct list_head list;
 815         struct hist_trigger_data *hist_data;
 816 };
 817
 818 static u64 hist_field_timestamp(struct hist_field *hist_field,
 819                                 struct tracing_map_elt *elt,
 820                                 struct trace_buffer *buffer,
 821                                 struct ring_buffer_event *rbe,
 822                                 void *event)
 823 {
 824         struct hist_trigger_data *hist_data = hist_field->hist_data;
 825         struct trace_array *tr = hist_data->event_file->tr;
 826
 827         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 828
 829         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 830                 ts = ns2usecs(ts);
 831
 832         return ts;
 833 }
 834
 835 static u64 hist_field_cpu(struct hist_field *hist_field,
 836                           struct tracing_map_elt *elt,
 837                           struct trace_buffer *buffer,
 838                           struct ring_buffer_event *rbe,
 839                           void *event)
 840 {
 841         int cpu = smp_processor_id();
 842
 843         return cpu;
 844 }
 845
 846 /**
 847  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 848  * @hist_field: The VAR_REF field to check
 849  * @var_data: The hist trigger that owns the variable
 850  * @var_idx: The trigger variable identifier
 851  *
 852  * Check the given VAR_REF field to see whether or not it references
 853  * the given variable associated with the given trigger.
 854  *
 855  * Return: The VAR_REF field if it does reference the variable, NULL if not
 856  */
 857 static struct hist_field *
 858 check_field_for_var_ref(struct hist_field *hist_field,
 859                         struct hist_trigger_data *var_data,
 860                         unsigned int var_idx)
 861 {
 862         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 863
 864         if (hist_field && hist_field->var.idx == var_idx &&
 865             hist_field->var.hist_data == var_data)
 866                 return hist_field;
 867
 868         return NULL;
 869 }
 870
 871 /**
 872  * find_var_ref - Check if a trigger has a reference to a trigger variable
 873  * @hist_data: The hist trigger that might have a reference to the variable
 874  * @var_data: The hist trigger that owns the variable
 875  * @var_idx: The trigger variable identifier
 876  *
 877  * Check the list of var_refs[] on the first hist trigger to see
 878  * whether any of them are references to the variable on the second
 879  * trigger.
 880  *
 881  * Return: The VAR_REF field referencing the variable if so, NULL if not
 882  */
 883 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 884                                        struct hist_trigger_data *var_data,
 885                                        unsigned int var_idx)
 886 {
 887         struct hist_field *hist_field;
 888         unsigned int i;
 889
 890         for (i = 0; i < hist_data->n_var_refs; i++) {
 891                 hist_field = hist_data->var_refs[i];
 892                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 893                         return hist_field;
 894         }
 895
 896         return NULL;
 897 }
 898
 899 /**
 900  * find_any_var_ref - Check if there is a reference to a given trigger variable
 901  * @hist_data: The hist trigger
 902  * @var_idx: The trigger variable identifier
 903  *
 904  * Check to see whether the given variable is currently referenced by
 905  * any other trigger.
 906  *
 907  * The trigger the variable is defined on is explicitly excluded - the
 908  * assumption being that a self-reference doesn't prevent a trigger
 909  * from being removed.
 910  *
 911  * Return: The VAR_REF field referencing the variable if so, NULL if not
 912  */
 913 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 914                                            unsigned int var_idx)
 915 {
 916         struct trace_array *tr = hist_data->event_file->tr;
 917         struct hist_field *found = NULL;
 918         struct hist_var_data *var_data;
 919
 920         list_for_each_entry(var_data, &tr->hist_vars, list) {
 921                 if (var_data->hist_data == hist_data)
 922                         continue;
 923                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 924                 if (found)
 925                         break;
 926         }
 927
 928         return found;
 929 }
 930
 931 /**
 932  * check_var_refs - Check if there is a reference to any of trigger's variables
 933  * @hist_data: The hist trigger
 934  *
 935  * A trigger can define one or more variables.  If any one of them is
 936  * currently referenced by any other trigger, this function will
 937  * determine that.
 938
 939  * Typically used to determine whether or not a trigger can be removed
 940  * - if there are any references to a trigger's variables, it cannot.
 941  *
 942  * Return: True if there is a reference to any of trigger's variables
 943  */
 944 static bool check_var_refs(struct hist_trigger_data *hist_data)
 945 {
 946         struct hist_field *field;
 947         bool found = false;
 948         int i;
 949
 950         for_each_hist_field(i, hist_data) {
 951                 field = hist_data->fields[i];
 952                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 953                         if (find_any_var_ref(hist_data, field->var.idx)) {
 954                                 found = true;
 955                                 break;
 956                         }
 957                 }
 958         }
 959
 960         return found;
 961 }
 962
 963 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 964 {
 965         struct trace_array *tr = hist_data->event_file->tr;
 966         struct hist_var_data *var_data, *found = NULL;
 967
 968         list_for_each_entry(var_data, &tr->hist_vars, list) {
 969                 if (var_data->hist_data == hist_data) {
 970                         found = var_data;
 971                         break;
 972                 }
 973         }
 974
 975         return found;
 976 }
 977
 978 static bool field_has_hist_vars(struct hist_field *hist_field,
 979                                 unsigned int level)
 980 {
 981         int i;
 982
 983         if (level > 3)
 984                 return false;
 985
 986         if (!hist_field)
 987                 return false;
 988
 989         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 990             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 991                 return true;
 992
 993         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 994                 struct hist_field *operand;
 995
 996                 operand = hist_field->operands[i];
 997                 if (field_has_hist_vars(operand, level + 1))
 998                         return true;
 999         }
1000
1001         return false;
1002 }
1003
1004 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1005 {
1006         struct hist_field *hist_field;
1007         int i;
1008
1009         for_each_hist_field(i, hist_data) {
1010                 hist_field = hist_data->fields[i];
1011                 if (field_has_hist_vars(hist_field, 0))
1012                         return true;
1013         }
1014
1015         return false;
1016 }
1017
1018 static int save_hist_vars(struct hist_trigger_data *hist_data)
1019 {
1020         struct trace_array *tr = hist_data->event_file->tr;
1021         struct hist_var_data *var_data;
1022
1023         var_data = find_hist_vars(hist_data);
1024         if (var_data)
1025                 return 0;
1026
1027         if (tracing_check_open_get_tr(tr))
1028                 return -ENODEV;
1029
1030         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1031         if (!var_data) {
1032                 trace_array_put(tr);
1033                 return -ENOMEM;
1034         }
1035
1036         var_data->hist_data = hist_data;
1037         list_add(&var_data->list, &tr->hist_vars);
1038
1039         return 0;
1040 }
1041
1042 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1043 {
1044         struct trace_array *tr = hist_data->event_file->tr;
1045         struct hist_var_data *var_data;
1046
1047         var_data = find_hist_vars(hist_data);
1048         if (!var_data)
1049                 return;
1050
1051         if (WARN_ON(check_var_refs(hist_data)))
1052                 return;
1053
1054         list_del(&var_data->list);
1055
1056         kfree(var_data);
1057
1058         trace_array_put(tr);
1059 }
1060
1061 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1062                                          const char *var_name)
1063 {
1064         struct hist_field *hist_field, *found = NULL;
1065         int i;
1066
1067         for_each_hist_field(i, hist_data) {
1068                 hist_field = hist_data->fields[i];
1069                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1070                     strcmp(hist_field->var.name, var_name) == 0) {
1071                         found = hist_field;
1072                         break;
1073                 }
1074         }
1075
1076         return found;
1077 }
1078
1079 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1080                                    struct trace_event_file *file,
1081                                    const char *var_name)
1082 {
1083         struct hist_trigger_data *test_data;
1084         struct event_trigger_data *test;
1085         struct hist_field *hist_field;
1086
1087         lockdep_assert_held(&event_mutex);
1088
1089         hist_field = find_var_field(hist_data, var_name);
1090         if (hist_field)
1091                 return hist_field;
1092
1093         list_for_each_entry(test, &file->triggers, list) {
1094                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1095                         test_data = test->private_data;
1096                         hist_field = find_var_field(test_data, var_name);
1097                         if (hist_field)
1098                                 return hist_field;
1099                 }
1100         }
1101
1102         return NULL;
1103 }
1104
1105 static struct trace_event_file *find_var_file(struct trace_array *tr,
1106                                               char *system,
1107                                               char *event_name,
1108                                               char *var_name)
1109 {
1110         struct hist_trigger_data *var_hist_data;
1111         struct hist_var_data *var_data;
1112         struct trace_event_file *file, *found = NULL;
1113
1114         if (system)
1115                 return find_event_file(tr, system, event_name);
1116
1117         list_for_each_entry(var_data, &tr->hist_vars, list) {
1118                 var_hist_data = var_data->hist_data;
1119                 file = var_hist_data->event_file;
1120                 if (file == found)
1121                         continue;
1122
1123                 if (find_var_field(var_hist_data, var_name)) {
1124                         if (found) {
1125                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1126                                 return NULL;
1127                         }
1128
1129                         found = file;
1130                 }
1131         }
1132
1133         return found;
1134 }
1135
1136 static struct hist_field *find_file_var(struct trace_event_file *file,
1137                                         const char *var_name)
1138 {
1139         struct hist_trigger_data *test_data;
1140         struct event_trigger_data *test;
1141         struct hist_field *hist_field;
1142
1143         lockdep_assert_held(&event_mutex);
1144
1145         list_for_each_entry(test, &file->triggers, list) {
1146                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1147                         test_data = test->private_data;
1148                         hist_field = find_var_field(test_data, var_name);
1149                         if (hist_field)
1150                                 return hist_field;
1151                 }
1152         }
1153
1154         return NULL;
1155 }
1156
1157 static struct hist_field *
1158 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1159 {
1160         struct trace_array *tr = hist_data->event_file->tr;
1161         struct hist_field *hist_field, *found = NULL;
1162         struct trace_event_file *file;
1163         unsigned int i;
1164
1165         for (i = 0; i < hist_data->n_actions; i++) {
1166                 struct action_data *data = hist_data->actions[i];
1167
1168                 if (data->handler == HANDLER_ONMATCH) {
1169                         char *system = data->match_data.event_system;
1170                         char *event_name = data->match_data.event;
1171
1172                         file = find_var_file(tr, system, event_name, var_name);
1173                         if (!file)
1174                                 continue;
1175                         hist_field = find_file_var(file, var_name);
1176                         if (hist_field) {
1177                                 if (found) {
1178                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1179                                                  errpos(var_name));
1180                                         return ERR_PTR(-EINVAL);
1181                                 }
1182
1183                                 found = hist_field;
1184                         }
1185                 }
1186         }
1187         return found;
1188 }
1189
1190 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1191                                          char *system,
1192                                          char *event_name,
1193                                          char *var_name)
1194 {
1195         struct trace_array *tr = hist_data->event_file->tr;
1196         struct hist_field *hist_field = NULL;
1197         struct trace_event_file *file;
1198
1199         if (!system || !event_name) {
1200                 hist_field = find_match_var(hist_data, var_name);
1201                 if (IS_ERR(hist_field))
1202                         return NULL;
1203                 if (hist_field)
1204                         return hist_field;
1205         }
1206
1207         file = find_var_file(tr, system, event_name, var_name);
1208         if (!file)
1209                 return NULL;
1210
1211         hist_field = find_file_var(file, var_name);
1212
1213         return hist_field;
1214 }
1215
1216 static u64 hist_field_var_ref(struct hist_field *hist_field,
1217                               struct tracing_map_elt *elt,
1218                               struct trace_buffer *buffer,
1219                               struct ring_buffer_event *rbe,
1220                               void *event)
1221 {
1222         struct hist_elt_data *elt_data;
1223         u64 var_val = 0;
1224
1225         if (WARN_ON_ONCE(!elt))
1226                 return var_val;
1227
1228         elt_data = elt->private_data;
1229         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1230
1231         return var_val;
1232 }
1233
1234 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1235                              u64 *var_ref_vals, bool self)
1236 {
1237         struct hist_trigger_data *var_data;
1238         struct tracing_map_elt *var_elt;
1239         struct hist_field *hist_field;
1240         unsigned int i, var_idx;
1241         bool resolved = true;
1242         u64 var_val = 0;
1243
1244         for (i = 0; i < hist_data->n_var_refs; i++) {
1245                 hist_field = hist_data->var_refs[i];
1246                 var_idx = hist_field->var.idx;
1247                 var_data = hist_field->var.hist_data;
1248
1249                 if (var_data == NULL) {
1250                         resolved = false;
1251                         break;
1252                 }
1253
1254                 if ((self && var_data != hist_data) ||
1255                     (!self && var_data == hist_data))
1256                         continue;
1257
1258                 var_elt = tracing_map_lookup(var_data->map, key);
1259                 if (!var_elt) {
1260                         resolved = false;
1261                         break;
1262                 }
1263
1264                 if (!tracing_map_var_set(var_elt, var_idx)) {
1265                         resolved = false;
1266                         break;
1267                 }
1268
1269                 if (self || !hist_field->read_once)
1270                         var_val = tracing_map_read_var(var_elt, var_idx);
1271                 else
1272                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1273
1274                 var_ref_vals[i] = var_val;
1275         }
1276
1277         return resolved;
1278 }
1279
1280 static const char *hist_field_name(struct hist_field *field,
1281                                    unsigned int level)
1282 {
1283         const char *field_name = "";
1284
1285         if (level > 1)
1286                 return field_name;
1287
1288         if (field->field)
1289                 field_name = field->field->name;
1290         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1291                  field->flags & HIST_FIELD_FL_ALIAS ||
1292                  field->flags & HIST_FIELD_FL_BUCKET)
1293                 field_name = hist_field_name(field->operands[0], ++level);
1294         else if (field->flags & HIST_FIELD_FL_CPU)
1295                 field_name = "common_cpu";
1296         else if (field->flags & HIST_FIELD_FL_EXPR ||
1297                  field->flags & HIST_FIELD_FL_VAR_REF) {
1298                 if (field->system) {
1299                         static char full_name[MAX_FILTER_STR_VAL];
1300
1301                         strcat(full_name, field->system);
1302                         strcat(full_name, ".");
1303                         strcat(full_name, field->event_name);
1304                         strcat(full_name, ".");
1305                         strcat(full_name, field->name);
1306                         field_name = full_name;
1307                 } else
1308                         field_name = field->name;
1309         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1310                 field_name = "common_timestamp";
1311
1312         if (field_name == NULL)
1313                 field_name = "";
1314
1315         return field_name;
1316 }
1317
1318 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1319 {
1320         hist_field_fn_t fn = NULL;
1321
1322         switch (field_size) {
1323         case 8:
1324                 if (field_is_signed)
1325                         fn = hist_field_s64;
1326                 else
1327                         fn = hist_field_u64;
1328                 break;
1329         case 4:
1330                 if (field_is_signed)
1331                         fn = hist_field_s32;
1332                 else
1333                         fn = hist_field_u32;
1334                 break;
1335         case 2:
1336                 if (field_is_signed)
1337                         fn = hist_field_s16;
1338                 else
1339                         fn = hist_field_u16;
1340                 break;
1341         case 1:
1342                 if (field_is_signed)
1343                         fn = hist_field_s8;
1344                 else
1345                         fn = hist_field_u8;
1346                 break;
1347         }
1348
1349         return fn;
1350 }
1351
1352 static int parse_map_size(char *str)
1353 {
1354         unsigned long size, map_bits;
1355         int ret;
1356
1357         ret = kstrtoul(str, 0, &size);
1358         if (ret)
1359                 goto out;
1360
1361         map_bits = ilog2(roundup_pow_of_two(size));
1362         if (map_bits < TRACING_MAP_BITS_MIN ||
1363             map_bits > TRACING_MAP_BITS_MAX)
1364                 ret = -EINVAL;
1365         else
1366                 ret = map_bits;
1367  out:
1368         return ret;
1369 }
1370
1371 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1372 {
1373         unsigned int i;
1374
1375         if (!attrs)
1376                 return;
1377
1378         for (i = 0; i < attrs->n_assignments; i++)
1379                 kfree(attrs->assignment_str[i]);
1380
1381         for (i = 0; i < attrs->n_actions; i++)
1382                 kfree(attrs->action_str[i]);
1383
1384         kfree(attrs->name);
1385         kfree(attrs->sort_key_str);
1386         kfree(attrs->keys_str);
1387         kfree(attrs->vals_str);
1388         kfree(attrs->clock);
1389         kfree(attrs);
1390 }
1391
1392 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1393 {
1394         int ret = -EINVAL;
1395
1396         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1397                 return ret;
1398
1399         if ((str_has_prefix(str, "onmatch(")) ||
1400             (str_has_prefix(str, "onmax(")) ||
1401             (str_has_prefix(str, "onchange("))) {
1402                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1403                 if (!attrs->action_str[attrs->n_actions]) {
1404                         ret = -ENOMEM;
1405                         return ret;
1406                 }
1407                 attrs->n_actions++;
1408                 ret = 0;
1409         }
1410         return ret;
1411 }
1412
1413 static int parse_assignment(struct trace_array *tr,
1414                             char *str, struct hist_trigger_attrs *attrs)
1415 {
1416         int len, ret = 0;
1417
1418         if ((len = str_has_prefix(str, "key=")) ||
1419             (len = str_has_prefix(str, "keys="))) {
1420                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1421                 if (!attrs->keys_str) {
1422                         ret = -ENOMEM;
1423                         goto out;
1424                 }
1425         } else if ((len = str_has_prefix(str, "val=")) ||
1426                    (len = str_has_prefix(str, "vals=")) ||
1427                    (len = str_has_prefix(str, "values="))) {
1428                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1429                 if (!attrs->vals_str) {
1430                         ret = -ENOMEM;
1431                         goto out;
1432                 }
1433         } else if ((len = str_has_prefix(str, "sort="))) {
1434                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1435                 if (!attrs->sort_key_str) {
1436                         ret = -ENOMEM;
1437                         goto out;
1438                 }
1439         } else if (str_has_prefix(str, "name=")) {
1440                 attrs->name = kstrdup(str, GFP_KERNEL);
1441                 if (!attrs->name) {
1442                         ret = -ENOMEM;
1443                         goto out;
1444                 }
1445         } else if ((len = str_has_prefix(str, "clock="))) {
1446                 str += len;
1447
1448                 str = strstrip(str);
1449                 attrs->clock = kstrdup(str, GFP_KERNEL);
1450                 if (!attrs->clock) {
1451                         ret = -ENOMEM;
1452                         goto out;
1453                 }
1454         } else if ((len = str_has_prefix(str, "size="))) {
1455                 int map_bits = parse_map_size(str + len);
1456
1457                 if (map_bits < 0) {
1458                         ret = map_bits;
1459                         goto out;
1460                 }
1461                 attrs->map_bits = map_bits;
1462         } else {
1463                 char *assignment;
1464
1465                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1466                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1467                         ret = -EINVAL;
1468                         goto out;
1469                 }
1470
1471                 assignment = kstrdup(str, GFP_KERNEL);
1472                 if (!assignment) {
1473                         ret = -ENOMEM;
1474                         goto out;
1475                 }
1476
1477                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1478         }
1479  out:
1480         return ret;
1481 }
1482
1483 static struct hist_trigger_attrs *
1484 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1485 {
1486         struct hist_trigger_attrs *attrs;
1487         int ret = 0;
1488
1489         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1490         if (!attrs)
1491                 return ERR_PTR(-ENOMEM);
1492
1493         while (trigger_str) {
1494                 char *str = strsep(&trigger_str, ":");
1495                 char *rhs;
1496
1497                 rhs = strchr(str, '=');
1498                 if (rhs) {
1499                         if (!strlen(++rhs)) {
1500                                 ret = -EINVAL;
1501                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1502                                 goto free;
1503                         }
1504                         ret = parse_assignment(tr, str, attrs);
1505                         if (ret)
1506                                 goto free;
1507                 } else if (strcmp(str, "pause") == 0)
1508                         attrs->pause = true;
1509                 else if ((strcmp(str, "cont") == 0) ||
1510                          (strcmp(str, "continue") == 0))
1511                         attrs->cont = true;
1512                 else if (strcmp(str, "clear") == 0)
1513                         attrs->clear = true;
1514                 else {
1515                         ret = parse_action(str, attrs);
1516                         if (ret)
1517                                 goto free;
1518                 }
1519         }
1520
1521         if (!attrs->keys_str) {
1522                 ret = -EINVAL;
1523                 goto free;
1524         }
1525
1526         if (!attrs->clock) {
1527                 attrs->clock = kstrdup("global", GFP_KERNEL);
1528                 if (!attrs->clock) {
1529                         ret = -ENOMEM;
1530                         goto free;
1531                 }
1532         }
1533
1534         return attrs;
1535  free:
1536         destroy_hist_trigger_attrs(attrs);
1537
1538         return ERR_PTR(ret);
1539 }
1540
1541 static inline void save_comm(char *comm, struct task_struct *task)
1542 {
1543         if (!task->pid) {
1544                 strcpy(comm, "<idle>");
1545                 return;
1546         }
1547
1548         if (WARN_ON_ONCE(task->pid < 0)) {
1549                 strcpy(comm, "<XXX>");
1550                 return;
1551         }
1552
1553         strncpy(comm, task->comm, TASK_COMM_LEN);
1554 }
1555
1556 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1557 {
1558         unsigned int i;
1559
1560         for (i = 0; i < elt_data->n_field_var_str; i++)
1561                 kfree(elt_data->field_var_str[i]);
1562
1563         kfree(elt_data->field_var_str);
1564
1565         kfree(elt_data->comm);
1566         kfree(elt_data);
1567 }
1568
1569 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1570 {
1571         struct hist_elt_data *elt_data = elt->private_data;
1572
1573         hist_elt_data_free(elt_data);
1574 }
1575
1576 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1577 {
1578         struct hist_trigger_data *hist_data = elt->map->private_data;
1579         unsigned int size = TASK_COMM_LEN;
1580         struct hist_elt_data *elt_data;
1581         struct hist_field *hist_field;
1582         unsigned int i, n_str;
1583
1584         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1585         if (!elt_data)
1586                 return -ENOMEM;
1587
1588         for_each_hist_field(i, hist_data) {
1589                 hist_field = hist_data->fields[i];
1590
1591                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1592                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1593                         if (!elt_data->comm) {
1594                                 kfree(elt_data);
1595                                 return -ENOMEM;
1596                         }
1597                         break;
1598                 }
1599         }
1600
1601         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1602                 hist_data->n_var_str;
1603         if (n_str > SYNTH_FIELDS_MAX) {
1604                 hist_elt_data_free(elt_data);
1605                 return -EINVAL;
1606         }
1607
1608         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1609
1610         size = STR_VAR_LEN_MAX;
1611
1612         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1613         if (!elt_data->field_var_str) {
1614                 hist_elt_data_free(elt_data);
1615                 return -EINVAL;
1616         }
1617         elt_data->n_field_var_str = n_str;
1618
1619         for (i = 0; i < n_str; i++) {
1620                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1621                 if (!elt_data->field_var_str[i]) {
1622                         hist_elt_data_free(elt_data);
1623                         return -ENOMEM;
1624                 }
1625         }
1626
1627         elt->private_data = elt_data;
1628
1629         return 0;
1630 }
1631
1632 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1633 {
1634         struct hist_elt_data *elt_data = elt->private_data;
1635
1636         if (elt_data->comm)
1637                 save_comm(elt_data->comm, current);
1638 }
1639
1640 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1641         .elt_alloc      = hist_trigger_elt_data_alloc,
1642         .elt_free       = hist_trigger_elt_data_free,
1643         .elt_init       = hist_trigger_elt_data_init,
1644 };
1645
1646 static const char *get_hist_field_flags(struct hist_field *hist_field)
1647 {
1648         const char *flags_str = NULL;
1649
1650         if (hist_field->flags & HIST_FIELD_FL_HEX)
1651                 flags_str = "hex";
1652         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1653                 flags_str = "sym";
1654         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1655                 flags_str = "sym-offset";
1656         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1657                 flags_str = "execname";
1658         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1659                 flags_str = "syscall";
1660         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1661                 flags_str = "log2";
1662         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1663                 flags_str = "buckets";
1664         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1665                 flags_str = "usecs";
1666
1667         return flags_str;
1668 }
1669
1670 static void expr_field_str(struct hist_field *field, char *expr)
1671 {
1672         if (field->flags & HIST_FIELD_FL_VAR_REF)
1673                 strcat(expr, "$");
1674         else if (field->flags & HIST_FIELD_FL_CONST) {
1675                 char str[HIST_CONST_DIGITS_MAX];
1676
1677                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1678                 strcat(expr, str);
1679         }
1680
1681         strcat(expr, hist_field_name(field, 0));
1682
1683         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1684                 const char *flags_str = get_hist_field_flags(field);
1685
1686                 if (flags_str) {
1687                         strcat(expr, ".");
1688                         strcat(expr, flags_str);
1689                 }
1690         }
1691 }
1692
1693 static char *expr_str(struct hist_field *field, unsigned int level)
1694 {
1695         char *expr;
1696
1697         if (level > 1)
1698                 return NULL;
1699
1700         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1701         if (!expr)
1702                 return NULL;
1703
1704         if (!field->operands[0]) {
1705                 expr_field_str(field, expr);
1706                 return expr;
1707         }
1708
1709         if (field->operator == FIELD_OP_UNARY_MINUS) {
1710                 char *subexpr;
1711
1712                 strcat(expr, "-(");
1713                 subexpr = expr_str(field->operands[0], ++level);
1714                 if (!subexpr) {
1715                         kfree(expr);
1716                         return NULL;
1717                 }
1718                 strcat(expr, subexpr);
1719                 strcat(expr, ")");
1720
1721                 kfree(subexpr);
1722
1723                 return expr;
1724         }
1725
1726         expr_field_str(field->operands[0], expr);
1727
1728         switch (field->operator) {
1729         case FIELD_OP_MINUS:
1730                 strcat(expr, "-");
1731                 break;
1732         case FIELD_OP_PLUS:
1733                 strcat(expr, "+");
1734                 break;
1735         case FIELD_OP_DIV:
1736                 strcat(expr, "/");
1737                 break;
1738         case FIELD_OP_MULT:
1739                 strcat(expr, "*");
1740                 break;
1741         default:
1742                 kfree(expr);
1743                 return NULL;
1744         }
1745
1746         expr_field_str(field->operands[1], expr);
1747
1748         return expr;
1749 }
1750
1751 /*
1752  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1753  * of the expression tree to be evaluated.
1754  */
1755 static int contains_operator(char *str, char **sep)
1756 {
1757         enum field_op_id field_op = FIELD_OP_NONE;
1758         char *minus_op, *plus_op, *div_op, *mult_op;
1759
1760
1761         /*
1762          * Report the last occurrence of the operators first, so that the
1763          * expression is evaluated left to right. This is important since
1764          * subtraction and division are not associative.
1765          *
1766          *      e.g
1767          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1768          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1769          */
1770
1771         /*
1772          * First, find lower precedence addition and subtraction
1773          * since the expression will be evaluated recursively.
1774          */
1775         minus_op = strrchr(str, '-');
1776         if (minus_op) {
1777                 /*
1778                  * Unary minus is not supported in sub-expressions. If
1779                  * present, it is always the next root operator.
1780                  */
1781                 if (minus_op == str) {
1782                         field_op = FIELD_OP_UNARY_MINUS;
1783                         goto out;
1784                 }
1785
1786                 field_op = FIELD_OP_MINUS;
1787         }
1788
1789         plus_op = strrchr(str, '+');
1790         if (plus_op || minus_op) {
1791                 /*
1792                  * For operators of the same precedence use to rightmost as the
1793                  * root, so that the expression is evaluated left to right.
1794                  */
1795                 if (plus_op > minus_op)
1796                         field_op = FIELD_OP_PLUS;
1797                 goto out;
1798         }
1799
1800         /*
1801          * Multiplication and division have higher precedence than addition and
1802          * subtraction.
1803          */
1804         div_op = strrchr(str, '/');
1805         if (div_op)
1806                 field_op = FIELD_OP_DIV;
1807
1808         mult_op = strrchr(str, '*');
1809         /*
1810          * For operators of the same precedence use to rightmost as the
1811          * root, so that the expression is evaluated left to right.
1812          */
1813         if (mult_op > div_op)
1814                 field_op = FIELD_OP_MULT;
1815
1816 out:
1817         if (sep) {
1818                 switch (field_op) {
1819                 case FIELD_OP_UNARY_MINUS:
1820                 case FIELD_OP_MINUS:
1821                         *sep = minus_op;
1822                         break;
1823                 case FIELD_OP_PLUS:
1824                         *sep = plus_op;
1825                         break;
1826                 case FIELD_OP_DIV:
1827                         *sep = div_op;
1828                         break;
1829                 case FIELD_OP_MULT:
1830                         *sep = mult_op;
1831                         break;
1832                 case FIELD_OP_NONE:
1833                 default:
1834                         *sep = NULL;
1835                         break;
1836                 }
1837         }
1838
1839         return field_op;
1840 }
1841
1842 static void get_hist_field(struct hist_field *hist_field)
1843 {
1844         hist_field->ref++;
1845 }
1846
1847 static void __destroy_hist_field(struct hist_field *hist_field)
1848 {
1849         if (--hist_field->ref > 1)
1850                 return;
1851
1852         kfree(hist_field->var.name);
1853         kfree(hist_field->name);
1854
1855         /* Can likely be a const */
1856         kfree_const(hist_field->type);
1857
1858         kfree(hist_field->system);
1859         kfree(hist_field->event_name);
1860
1861         kfree(hist_field);
1862 }
1863
1864 static void destroy_hist_field(struct hist_field *hist_field,
1865                                unsigned int level)
1866 {
1867         unsigned int i;
1868
1869         if (level > 3)
1870                 return;
1871
1872         if (!hist_field)
1873                 return;
1874
1875         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1876                 return; /* var refs will be destroyed separately */
1877
1878         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1879                 destroy_hist_field(hist_field->operands[i], level + 1);
1880
1881         __destroy_hist_field(hist_field);
1882 }
1883
1884 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1885                                             struct ftrace_event_field *field,
1886                                             unsigned long flags,
1887                                             char *var_name)
1888 {
1889         struct hist_field *hist_field;
1890
1891         if (field && is_function_field(field))
1892                 return NULL;
1893
1894         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1895         if (!hist_field)
1896                 return NULL;
1897
1898         hist_field->ref = 1;
1899
1900         hist_field->hist_data = hist_data;
1901
1902         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1903                 goto out; /* caller will populate */
1904
1905         if (flags & HIST_FIELD_FL_VAR_REF) {
1906                 hist_field->fn = hist_field_var_ref;
1907                 goto out;
1908         }
1909
1910         if (flags & HIST_FIELD_FL_HITCOUNT) {
1911                 hist_field->fn = hist_field_counter;
1912                 hist_field->size = sizeof(u64);
1913                 hist_field->type = "u64";
1914                 goto out;
1915         }
1916
1917         if (flags & HIST_FIELD_FL_CONST) {
1918                 hist_field->fn = hist_field_const;
1919                 hist_field->size = sizeof(u64);
1920                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1921                 if (!hist_field->type)
1922                         goto free;
1923                 goto out;
1924         }
1925
1926         if (flags & HIST_FIELD_FL_STACKTRACE) {
1927                 hist_field->fn = hist_field_none;
1928                 goto out;
1929         }
1930
1931         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1932                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1933                 hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1934                         hist_field_bucket;
1935                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1936                 hist_field->size = hist_field->operands[0]->size;
1937                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1938                 if (!hist_field->type)
1939                         goto free;
1940                 goto out;
1941         }
1942
1943         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1944                 hist_field->fn = hist_field_timestamp;
1945                 hist_field->size = sizeof(u64);
1946                 hist_field->type = "u64";
1947                 goto out;
1948         }
1949
1950         if (flags & HIST_FIELD_FL_CPU) {
1951                 hist_field->fn = hist_field_cpu;
1952                 hist_field->size = sizeof(int);
1953                 hist_field->type = "unsigned int";
1954                 goto out;
1955         }
1956
1957         if (WARN_ON_ONCE(!field))
1958                 goto out;
1959
1960         /* Pointers to strings are just pointers and dangerous to dereference */
1961         if (is_string_field(field) &&
1962             (field->filter_type != FILTER_PTR_STRING)) {
1963                 flags |= HIST_FIELD_FL_STRING;
1964
1965                 hist_field->size = MAX_FILTER_STR_VAL;
1966                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1967                 if (!hist_field->type)
1968                         goto free;
1969
1970                 if (field->filter_type == FILTER_STATIC_STRING) {
1971                         hist_field->fn = hist_field_string;
1972                         hist_field->size = field->size;
1973                 } else if (field->filter_type == FILTER_DYN_STRING) {
1974                         hist_field->fn = hist_field_dynstring;
1975                 } else if (field->filter_type == FILTER_RDYN_STRING)
1976                         hist_field->fn = hist_field_reldynstring;
1977                 else
1978                         hist_field->fn = hist_field_pstring;
1979         } else {
1980                 hist_field->size = field->size;
1981                 hist_field->is_signed = field->is_signed;
1982                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1983                 if (!hist_field->type)
1984                         goto free;
1985
1986                 hist_field->fn = select_value_fn(field->size,
1987                                                  field->is_signed);
1988                 if (!hist_field->fn) {
1989                         destroy_hist_field(hist_field, 0);
1990                         return NULL;
1991                 }
1992         }
1993  out:
1994         hist_field->field = field;
1995         hist_field->flags = flags;
1996
1997         if (var_name) {
1998                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1999                 if (!hist_field->var.name)
2000                         goto free;
2001         }
2002
2003         return hist_field;
2004  free:
2005         destroy_hist_field(hist_field, 0);
2006         return NULL;
2007 }
2008
2009 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2010 {
2011         unsigned int i;
2012
2013         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2014                 if (hist_data->fields[i]) {
2015                         destroy_hist_field(hist_data->fields[i], 0);
2016                         hist_data->fields[i] = NULL;
2017                 }
2018         }
2019
2020         for (i = 0; i < hist_data->n_var_refs; i++) {
2021                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2022                 __destroy_hist_field(hist_data->var_refs[i]);
2023                 hist_data->var_refs[i] = NULL;
2024         }
2025 }
2026
2027 static int init_var_ref(struct hist_field *ref_field,
2028                         struct hist_field *var_field,
2029                         char *system, char *event_name)
2030 {
2031         int err = 0;
2032
2033         ref_field->var.idx = var_field->var.idx;
2034         ref_field->var.hist_data = var_field->hist_data;
2035         ref_field->size = var_field->size;
2036         ref_field->is_signed = var_field->is_signed;
2037         ref_field->flags |= var_field->flags &
2038                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2039
2040         if (system) {
2041                 ref_field->system = kstrdup(system, GFP_KERNEL);
2042                 if (!ref_field->system)
2043                         return -ENOMEM;
2044         }
2045
2046         if (event_name) {
2047                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2048                 if (!ref_field->event_name) {
2049                         err = -ENOMEM;
2050                         goto free;
2051                 }
2052         }
2053
2054         if (var_field->var.name) {
2055                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2056                 if (!ref_field->name) {
2057                         err = -ENOMEM;
2058                         goto free;
2059                 }
2060         } else if (var_field->name) {
2061                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2062                 if (!ref_field->name) {
2063                         err = -ENOMEM;
2064                         goto free;
2065                 }
2066         }
2067
2068         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2069         if (!ref_field->type) {
2070                 err = -ENOMEM;
2071                 goto free;
2072         }
2073  out:
2074         return err;
2075  free:
2076         kfree(ref_field->system);
2077         kfree(ref_field->event_name);
2078         kfree(ref_field->name);
2079
2080         goto out;
2081 }
2082
2083 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2084                             struct hist_field *var_field)
2085 {
2086         struct hist_field *ref_field;
2087         int i;
2088
2089         for (i = 0; i < hist_data->n_var_refs; i++) {
2090                 ref_field = hist_data->var_refs[i];
2091                 if (ref_field->var.idx == var_field->var.idx &&
2092                     ref_field->var.hist_data == var_field->hist_data)
2093                         return i;
2094         }
2095
2096         return -ENOENT;
2097 }
2098
2099 /**
2100  * create_var_ref - Create a variable reference and attach it to trigger
2101  * @hist_data: The trigger that will be referencing the variable
2102  * @var_field: The VAR field to create a reference to
2103  * @system: The optional system string
2104  * @event_name: The optional event_name string
2105  *
2106  * Given a variable hist_field, create a VAR_REF hist_field that
2107  * represents a reference to it.
2108  *
2109  * This function also adds the reference to the trigger that
2110  * now references the variable.
2111  *
2112  * Return: The VAR_REF field if successful, NULL if not
2113  */
2114 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2115                                          struct hist_field *var_field,
2116                                          char *system, char *event_name)
2117 {
2118         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2119         struct hist_field *ref_field;
2120         int i;
2121
2122         /* Check if the variable already exists */
2123         for (i = 0; i < hist_data->n_var_refs; i++) {
2124                 ref_field = hist_data->var_refs[i];
2125                 if (ref_field->var.idx == var_field->var.idx &&
2126                     ref_field->var.hist_data == var_field->hist_data) {
2127                         get_hist_field(ref_field);
2128                         return ref_field;
2129                 }
2130         }
2131
2132         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2133         if (ref_field) {
2134                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2135                         destroy_hist_field(ref_field, 0);
2136                         return NULL;
2137                 }
2138
2139                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2140                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2141         }
2142
2143         return ref_field;
2144 }
2145
2146 static bool is_var_ref(char *var_name)
2147 {
2148         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2149                 return false;
2150
2151         return true;
2152 }
2153
2154 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2155                                  char *var_name)
2156 {
2157         char *name, *field;
2158         unsigned int i;
2159
2160         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2161                 name = hist_data->attrs->var_defs.name[i];
2162
2163                 if (strcmp(var_name, name) == 0) {
2164                         field = hist_data->attrs->var_defs.expr[i];
2165                         if (contains_operator(field, NULL) || is_var_ref(field))
2166                                 continue;
2167                         return field;
2168                 }
2169         }
2170
2171         return NULL;
2172 }
2173
2174 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2175                                  char *system, char *event_name,
2176                                  char *var_name)
2177 {
2178         struct trace_event_call *call;
2179
2180         if (system && event_name) {
2181                 call = hist_data->event_file->event_call;
2182
2183                 if (strcmp(system, call->class->system) != 0)
2184                         return NULL;
2185
2186                 if (strcmp(event_name, trace_event_name(call)) != 0)
2187                         return NULL;
2188         }
2189
2190         if (!!system != !!event_name)
2191                 return NULL;
2192
2193         if (!is_var_ref(var_name))
2194                 return NULL;
2195
2196         var_name++;
2197
2198         return field_name_from_var(hist_data, var_name);
2199 }
2200
2201 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2202                                         char *system, char *event_name,
2203                                         char *var_name)
2204 {
2205         struct hist_field *var_field = NULL, *ref_field = NULL;
2206         struct trace_array *tr = hist_data->event_file->tr;
2207
2208         if (!is_var_ref(var_name))
2209                 return NULL;
2210
2211         var_name++;
2212
2213         var_field = find_event_var(hist_data, system, event_name, var_name);
2214         if (var_field)
2215                 ref_field = create_var_ref(hist_data, var_field,
2216                                            system, event_name);
2217
2218         if (!ref_field)
2219                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2220
2221         return ref_field;
2222 }
2223
2224 static struct ftrace_event_field *
2225 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2226             char *field_str, unsigned long *flags, unsigned long *buckets)
2227 {
2228         struct ftrace_event_field *field = NULL;
2229         char *field_name, *modifier, *str;
2230         struct trace_array *tr = file->tr;
2231
2232         modifier = str = kstrdup(field_str, GFP_KERNEL);
2233         if (!modifier)
2234                 return ERR_PTR(-ENOMEM);
2235
2236         field_name = strsep(&modifier, ".");
2237         if (modifier) {
2238                 if (strcmp(modifier, "hex") == 0)
2239                         *flags |= HIST_FIELD_FL_HEX;
2240                 else if (strcmp(modifier, "sym") == 0)
2241                         *flags |= HIST_FIELD_FL_SYM;
2242                 /*
2243                  * 'sym-offset' occurrences in the trigger string are modified
2244                  * to 'symXoffset' to simplify arithmetic expression parsing.
2245                  */
2246                 else if (strcmp(modifier, "symXoffset") == 0)
2247                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2248                 else if ((strcmp(modifier, "execname") == 0) &&
2249                          (strcmp(field_name, "common_pid") == 0))
2250                         *flags |= HIST_FIELD_FL_EXECNAME;
2251                 else if (strcmp(modifier, "syscall") == 0)
2252                         *flags |= HIST_FIELD_FL_SYSCALL;
2253                 else if (strcmp(modifier, "log2") == 0)
2254                         *flags |= HIST_FIELD_FL_LOG2;
2255                 else if (strcmp(modifier, "usecs") == 0)
2256                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2257                 else if (strncmp(modifier, "bucket", 6) == 0) {
2258                         int ret;
2259
2260                         modifier += 6;
2261
2262                         if (*modifier == 's')
2263                                 modifier++;
2264                         if (*modifier != '=')
2265                                 goto error;
2266                         modifier++;
2267                         ret = kstrtoul(modifier, 0, buckets);
2268                         if (ret || !(*buckets))
2269                                 goto error;
2270                         *flags |= HIST_FIELD_FL_BUCKET;
2271                 } else {
2272  error:
2273                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2274                         field = ERR_PTR(-EINVAL);
2275                         goto out;
2276                 }
2277         }
2278
2279         if (strcmp(field_name, "common_timestamp") == 0) {
2280                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2281                 hist_data->enable_timestamps = true;
2282                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2283                         hist_data->attrs->ts_in_usecs = true;
2284         } else if (strcmp(field_name, "common_cpu") == 0)
2285                 *flags |= HIST_FIELD_FL_CPU;
2286         else {
2287                 field = trace_find_event_field(file->event_call, field_name);
2288                 if (!field || !field->size) {
2289                         /*
2290                          * For backward compatibility, if field_name
2291                          * was "cpu", then we treat this the same as
2292                          * common_cpu.
2293                          */
2294                         if (strcmp(field_name, "cpu") == 0) {
2295                                 *flags |= HIST_FIELD_FL_CPU;
2296                         } else {
2297                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2298                                          errpos(field_name));
2299                                 field = ERR_PTR(-EINVAL);
2300                                 goto out;
2301                         }
2302                 }
2303         }
2304  out:
2305         kfree(str);
2306
2307         return field;
2308 }
2309
2310 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2311                                        struct hist_field *var_ref,
2312                                        char *var_name)
2313 {
2314         struct hist_field *alias = NULL;
2315         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2316
2317         alias = create_hist_field(hist_data, NULL, flags, var_name);
2318         if (!alias)
2319                 return NULL;
2320
2321         alias->fn = var_ref->fn;
2322         alias->operands[0] = var_ref;
2323
2324         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2325                 destroy_hist_field(alias, 0);
2326                 return NULL;
2327         }
2328
2329         alias->var_ref_idx = var_ref->var_ref_idx;
2330
2331         return alias;
2332 }
2333
2334 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2335                                       char *str, char *var_name,
2336                                       unsigned long *flags)
2337 {
2338         struct trace_array *tr = hist_data->event_file->tr;
2339         struct hist_field *field = NULL;
2340         u64 constant;
2341
2342         if (kstrtoull(str, 0, &constant)) {
2343                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2344                 return NULL;
2345         }
2346
2347         *flags |= HIST_FIELD_FL_CONST;
2348         field = create_hist_field(hist_data, NULL, *flags, var_name);
2349         if (!field)
2350                 return NULL;
2351
2352         field->constant = constant;
2353
2354         return field;
2355 }
2356
2357 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2358                                      struct trace_event_file *file, char *str,
2359                                      unsigned long *flags, char *var_name)
2360 {
2361         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2362         struct ftrace_event_field *field = NULL;
2363         struct hist_field *hist_field = NULL;
2364         unsigned long buckets = 0;
2365         int ret = 0;
2366
2367         if (isdigit(str[0])) {
2368                 hist_field = parse_const(hist_data, str, var_name, flags);
2369                 if (!hist_field) {
2370                         ret = -EINVAL;
2371                         goto out;
2372                 }
2373                 return hist_field;
2374         }
2375
2376         s = strchr(str, '.');
2377         if (s) {
2378                 s = strchr(++s, '.');
2379                 if (s) {
2380                         ref_system = strsep(&str, ".");
2381                         if (!str) {
2382                                 ret = -EINVAL;
2383                                 goto out;
2384                         }
2385                         ref_event = strsep(&str, ".");
2386                         if (!str) {
2387                                 ret = -EINVAL;
2388                                 goto out;
2389                         }
2390                         ref_var = str;
2391                 }
2392         }
2393
2394         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2395         if (!s) {
2396                 hist_field = parse_var_ref(hist_data, ref_system,
2397                                            ref_event, ref_var);
2398                 if (hist_field) {
2399                         if (var_name) {
2400                                 hist_field = create_alias(hist_data, hist_field, var_name);
2401                                 if (!hist_field) {
2402                                         ret = -ENOMEM;
2403                                         goto out;
2404                                 }
2405                         }
2406                         return hist_field;
2407                 }
2408         } else
2409                 str = s;
2410
2411         field = parse_field(hist_data, file, str, flags, &buckets);
2412         if (IS_ERR(field)) {
2413                 ret = PTR_ERR(field);
2414                 goto out;
2415         }
2416
2417         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2418         if (!hist_field) {
2419                 ret = -ENOMEM;
2420                 goto out;
2421         }
2422         hist_field->buckets = buckets;
2423
2424         return hist_field;
2425  out:
2426         return ERR_PTR(ret);
2427 }
2428
2429 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2430                                      struct trace_event_file *file,
2431                                      char *str, unsigned long flags,
2432                                      char *var_name, unsigned int *n_subexprs);
2433
2434 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2435                                       struct trace_event_file *file,
2436                                       char *str, unsigned long flags,
2437                                       char *var_name, unsigned int *n_subexprs)
2438 {
2439         struct hist_field *operand1, *expr = NULL;
2440         unsigned long operand_flags;
2441         int ret = 0;
2442         char *s;
2443
2444         /* Unary minus operator, increment n_subexprs */
2445         ++*n_subexprs;
2446
2447         /* we support only -(xxx) i.e. explicit parens required */
2448
2449         if (*n_subexprs > 3) {
2450                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2451                 ret = -EINVAL;
2452                 goto free;
2453         }
2454
2455         str++; /* skip leading '-' */
2456
2457         s = strchr(str, '(');
2458         if (s)
2459                 str++;
2460         else {
2461                 ret = -EINVAL;
2462                 goto free;
2463         }
2464
2465         s = strrchr(str, ')');
2466         if (s) {
2467                  /* unary minus not supported in sub-expressions */
2468                 if (*(s+1) != '\0') {
2469                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2470                                  errpos(str));
2471                         ret = -EINVAL;
2472                         goto free;
2473                 }
2474                 *s = '\0';
2475         }
2476         else {
2477                 ret = -EINVAL; /* no closing ')' */
2478                 goto free;
2479         }
2480
2481         flags |= HIST_FIELD_FL_EXPR;
2482         expr = create_hist_field(hist_data, NULL, flags, var_name);
2483         if (!expr) {
2484                 ret = -ENOMEM;
2485                 goto free;
2486         }
2487
2488         operand_flags = 0;
2489         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2490         if (IS_ERR(operand1)) {
2491                 ret = PTR_ERR(operand1);
2492                 goto free;
2493         }
2494         if (operand1->flags & HIST_FIELD_FL_STRING) {
2495                 /* String type can not be the operand of unary operator. */
2496                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2497                 destroy_hist_field(operand1, 0);
2498                 ret = -EINVAL;
2499                 goto free;
2500         }
2501
2502         expr->flags |= operand1->flags &
2503                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2504         expr->fn = hist_field_unary_minus;
2505         expr->operands[0] = operand1;
2506         expr->size = operand1->size;
2507         expr->is_signed = operand1->is_signed;
2508         expr->operator = FIELD_OP_UNARY_MINUS;
2509         expr->name = expr_str(expr, 0);
2510         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2511         if (!expr->type) {
2512                 ret = -ENOMEM;
2513                 goto free;
2514         }
2515
2516         return expr;
2517  free:
2518         destroy_hist_field(expr, 0);
2519         return ERR_PTR(ret);
2520 }
2521
2522 /*
2523  * If the operands are var refs, return pointers the
2524  * variable(s) referenced in var1 and var2, else NULL.
2525  */
2526 static int check_expr_operands(struct trace_array *tr,
2527                                struct hist_field *operand1,
2528                                struct hist_field *operand2,
2529                                struct hist_field **var1,
2530                                struct hist_field **var2)
2531 {
2532         unsigned long operand1_flags = operand1->flags;
2533         unsigned long operand2_flags = operand2->flags;
2534
2535         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2536             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2537                 struct hist_field *var;
2538
2539                 var = find_var_field(operand1->var.hist_data, operand1->name);
2540                 if (!var)
2541                         return -EINVAL;
2542                 operand1_flags = var->flags;
2543                 *var1 = var;
2544         }
2545
2546         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2547             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2548                 struct hist_field *var;
2549
2550                 var = find_var_field(operand2->var.hist_data, operand2->name);
2551                 if (!var)
2552                         return -EINVAL;
2553                 operand2_flags = var->flags;
2554                 *var2 = var;
2555         }
2556
2557         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2558             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2559                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2560                 return -EINVAL;
2561         }
2562
2563         return 0;
2564 }
2565
2566 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2567                                      struct trace_event_file *file,
2568                                      char *str, unsigned long flags,
2569                                      char *var_name, unsigned int *n_subexprs)
2570 {
2571         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2572         struct hist_field *var1 = NULL, *var2 = NULL;
2573         unsigned long operand_flags, operand2_flags;
2574         int field_op, ret = -EINVAL;
2575         char *sep, *operand1_str;
2576         hist_field_fn_t op_fn;
2577         bool combine_consts;
2578
2579         if (*n_subexprs > 3) {
2580                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2581                 return ERR_PTR(-EINVAL);
2582         }
2583
2584         field_op = contains_operator(str, &sep);
2585
2586         if (field_op == FIELD_OP_NONE)
2587                 return parse_atom(hist_data, file, str, &flags, var_name);
2588
2589         if (field_op == FIELD_OP_UNARY_MINUS)
2590                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2591
2592         /* Binary operator found, increment n_subexprs */
2593         ++*n_subexprs;
2594
2595         /* Split the expression string at the root operator */
2596         if (!sep)
2597                 return ERR_PTR(-EINVAL);
2598
2599         *sep = '\0';
2600         operand1_str = str;
2601         str = sep+1;
2602
2603         /* Binary operator requires both operands */
2604         if (*operand1_str == '\0' || *str == '\0')
2605                 return ERR_PTR(-EINVAL);
2606
2607         operand_flags = 0;
2608
2609         /* LHS of string is an expression e.g. a+b in a+b+c */
2610         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2611         if (IS_ERR(operand1))
2612                 return ERR_CAST(operand1);
2613
2614         if (operand1->flags & HIST_FIELD_FL_STRING) {
2615                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2616                 ret = -EINVAL;
2617                 goto free_op1;
2618         }
2619
2620         /* RHS of string is another expression e.g. c in a+b+c */
2621         operand_flags = 0;
2622         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2623         if (IS_ERR(operand2)) {
2624                 ret = PTR_ERR(operand2);
2625                 goto free_op1;
2626         }
2627         if (operand2->flags & HIST_FIELD_FL_STRING) {
2628                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2629                 ret = -EINVAL;
2630                 goto free_operands;
2631         }
2632
2633         switch (field_op) {
2634         case FIELD_OP_MINUS:
2635                 op_fn = hist_field_minus;
2636                 break;
2637         case FIELD_OP_PLUS:
2638                 op_fn = hist_field_plus;
2639                 break;
2640         case FIELD_OP_DIV:
2641                 op_fn = hist_field_div;
2642                 break;
2643         case FIELD_OP_MULT:
2644                 op_fn = hist_field_mult;
2645                 break;
2646         default:
2647                 ret = -EINVAL;
2648                 goto free_operands;
2649         }
2650
2651         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2652         if (ret)
2653                 goto free_operands;
2654
2655         operand_flags = var1 ? var1->flags : operand1->flags;
2656         operand2_flags = var2 ? var2->flags : operand2->flags;
2657
2658         /*
2659          * If both operands are constant, the expression can be
2660          * collapsed to a single constant.
2661          */
2662         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2663
2664         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2665
2666         flags |= operand1->flags &
2667                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2668
2669         expr = create_hist_field(hist_data, NULL, flags, var_name);
2670         if (!expr) {
2671                 ret = -ENOMEM;
2672                 goto free_operands;
2673         }
2674
2675         operand1->read_once = true;
2676         operand2->read_once = true;
2677
2678         /* The operands are now owned and free'd by 'expr' */
2679         expr->operands[0] = operand1;
2680         expr->operands[1] = operand2;
2681
2682         if (field_op == FIELD_OP_DIV &&
2683                         operand2_flags & HIST_FIELD_FL_CONST) {
2684                 u64 divisor = var2 ? var2->constant : operand2->constant;
2685
2686                 if (!divisor) {
2687                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2688                         ret = -EDOM;
2689                         goto free_expr;
2690                 }
2691
2692                 /*
2693                  * Copy the divisor here so we don't have to look it up
2694                  * later if this is a var ref
2695                  */
2696                 operand2->constant = divisor;
2697                 op_fn = hist_field_get_div_fn(operand2);
2698         }
2699
2700         if (combine_consts) {
2701                 if (var1)
2702                         expr->operands[0] = var1;
2703                 if (var2)
2704                         expr->operands[1] = var2;
2705
2706                 expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2707
2708                 expr->operands[0] = NULL;
2709                 expr->operands[1] = NULL;
2710
2711                 /*
2712                  * var refs won't be destroyed immediately
2713                  * See: destroy_hist_field()
2714                  */
2715                 destroy_hist_field(operand2, 0);
2716                 destroy_hist_field(operand1, 0);
2717
2718                 expr->name = expr_str(expr, 0);
2719         } else {
2720                 expr->fn = op_fn;
2721
2722                 /* The operand sizes should be the same, so just pick one */
2723                 expr->size = operand1->size;
2724                 expr->is_signed = operand1->is_signed;
2725
2726                 expr->operator = field_op;
2727                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2728                 if (!expr->type) {
2729                         ret = -ENOMEM;
2730                         goto free_expr;
2731                 }
2732
2733                 expr->name = expr_str(expr, 0);
2734         }
2735
2736         return expr;
2737
2738 free_operands:
2739         destroy_hist_field(operand2, 0);
2740 free_op1:
2741         destroy_hist_field(operand1, 0);
2742         return ERR_PTR(ret);
2743
2744 free_expr:
2745         destroy_hist_field(expr, 0);
2746         return ERR_PTR(ret);
2747 }
2748
2749 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2750                                  struct trace_event_file *file)
2751 {
2752         struct event_trigger_data *test;
2753
2754         lockdep_assert_held(&event_mutex);
2755
2756         list_for_each_entry(test, &file->triggers, list) {
2757                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2758                         if (test->private_data == hist_data)
2759                                 return test->filter_str;
2760                 }
2761         }
2762
2763         return NULL;
2764 }
2765
2766 static struct event_command trigger_hist_cmd;
2767 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2768                                     struct trace_event_file *file,
2769                                     char *glob, char *cmd, char *param);
2770
2771 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2772                             struct hist_trigger_data *hist_data,
2773                             unsigned int n_keys)
2774 {
2775         struct hist_field *target_hist_field, *hist_field;
2776         unsigned int n, i, j;
2777
2778         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2779                 return false;
2780
2781         i = hist_data->n_vals;
2782         j = target_hist_data->n_vals;
2783
2784         for (n = 0; n < n_keys; n++) {
2785                 hist_field = hist_data->fields[i + n];
2786                 target_hist_field = target_hist_data->fields[j + n];
2787
2788                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2789                         return false;
2790                 if (hist_field->size != target_hist_field->size)
2791                         return false;
2792                 if (hist_field->is_signed != target_hist_field->is_signed)
2793                         return false;
2794         }
2795
2796         return true;
2797 }
2798
2799 static struct hist_trigger_data *
2800 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2801                      struct trace_event_file *file)
2802 {
2803         struct hist_trigger_data *hist_data;
2804         struct event_trigger_data *test;
2805         unsigned int n_keys;
2806
2807         lockdep_assert_held(&event_mutex);
2808
2809         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2810
2811         list_for_each_entry(test, &file->triggers, list) {
2812                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2813                         hist_data = test->private_data;
2814
2815                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2816                                 return hist_data;
2817                 }
2818         }
2819
2820         return NULL;
2821 }
2822
2823 static struct trace_event_file *event_file(struct trace_array *tr,
2824                                            char *system, char *event_name)
2825 {
2826         struct trace_event_file *file;
2827
2828         file = __find_event_file(tr, system, event_name);
2829         if (!file)
2830                 return ERR_PTR(-EINVAL);
2831
2832         return file;
2833 }
2834
2835 static struct hist_field *
2836 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2837                          char *system, char *event_name, char *field_name)
2838 {
2839         struct hist_field *event_var;
2840         char *synthetic_name;
2841
2842         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2843         if (!synthetic_name)
2844                 return ERR_PTR(-ENOMEM);
2845
2846         strcpy(synthetic_name, "synthetic_");
2847         strcat(synthetic_name, field_name);
2848
2849         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2850
2851         kfree(synthetic_name);
2852
2853         return event_var;
2854 }
2855
2856 /**
2857  * create_field_var_hist - Automatically create a histogram and var for a field
2858  * @target_hist_data: The target hist trigger
2859  * @subsys_name: Optional subsystem name
2860  * @event_name: Optional event name
2861  * @field_name: The name of the field (and the resulting variable)
2862  *
2863  * Hist trigger actions fetch data from variables, not directly from
2864  * events.  However, for convenience, users are allowed to directly
2865  * specify an event field in an action, which will be automatically
2866  * converted into a variable on their behalf.
2867  *
2868  * If a user specifies a field on an event that isn't the event the
2869  * histogram currently being defined (the target event histogram), the
2870  * only way that can be accomplished is if a new hist trigger is
2871  * created and the field variable defined on that.
2872  *
2873  * This function creates a new histogram compatible with the target
2874  * event (meaning a histogram with the same key as the target
2875  * histogram), and creates a variable for the specified field, but
2876  * with 'synthetic_' prepended to the variable name in order to avoid
2877  * collision with normal field variables.
2878  *
2879  * Return: The variable created for the field.
2880  */
2881 static struct hist_field *
2882 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2883                       char *subsys_name, char *event_name, char *field_name)
2884 {
2885         struct trace_array *tr = target_hist_data->event_file->tr;
2886         struct hist_trigger_data *hist_data;
2887         unsigned int i, n, first = true;
2888         struct field_var_hist *var_hist;
2889         struct trace_event_file *file;
2890         struct hist_field *key_field;
2891         struct hist_field *event_var;
2892         char *saved_filter;
2893         char *cmd;
2894         int ret;
2895
2896         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2897                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2898                 return ERR_PTR(-EINVAL);
2899         }
2900
2901         file = event_file(tr, subsys_name, event_name);
2902
2903         if (IS_ERR(file)) {
2904                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2905                 ret = PTR_ERR(file);
2906                 return ERR_PTR(ret);
2907         }
2908
2909         /*
2910          * Look for a histogram compatible with target.  We'll use the
2911          * found histogram specification to create a new matching
2912          * histogram with our variable on it.  target_hist_data is not
2913          * yet a registered histogram so we can't use that.
2914          */
2915         hist_data = find_compatible_hist(target_hist_data, file);
2916         if (!hist_data) {
2917                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2918                 return ERR_PTR(-EINVAL);
2919         }
2920
2921         /* See if a synthetic field variable has already been created */
2922         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2923                                              event_name, field_name);
2924         if (!IS_ERR_OR_NULL(event_var))
2925                 return event_var;
2926
2927         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2928         if (!var_hist)
2929                 return ERR_PTR(-ENOMEM);
2930
2931         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2932         if (!cmd) {
2933                 kfree(var_hist);
2934                 return ERR_PTR(-ENOMEM);
2935         }
2936
2937         /* Use the same keys as the compatible histogram */
2938         strcat(cmd, "keys=");
2939
2940         for_each_hist_key_field(i, hist_data) {
2941                 key_field = hist_data->fields[i];
2942                 if (!first)
2943                         strcat(cmd, ",");
2944                 strcat(cmd, key_field->field->name);
2945                 first = false;
2946         }
2947
2948         /* Create the synthetic field variable specification */
2949         strcat(cmd, ":synthetic_");
2950         strcat(cmd, field_name);
2951         strcat(cmd, "=");
2952         strcat(cmd, field_name);
2953
2954         /* Use the same filter as the compatible histogram */
2955         saved_filter = find_trigger_filter(hist_data, file);
2956         if (saved_filter) {
2957                 strcat(cmd, " if ");
2958                 strcat(cmd, saved_filter);
2959         }
2960
2961         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2962         if (!var_hist->cmd) {
2963                 kfree(cmd);
2964                 kfree(var_hist);
2965                 return ERR_PTR(-ENOMEM);
2966         }
2967
2968         /* Save the compatible histogram information */
2969         var_hist->hist_data = hist_data;
2970
2971         /* Create the new histogram with our variable */
2972         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
2973                                        "", "hist", cmd);
2974         if (ret) {
2975                 kfree(cmd);
2976                 kfree(var_hist->cmd);
2977                 kfree(var_hist);
2978                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2979                 return ERR_PTR(ret);
2980         }
2981
2982         kfree(cmd);
2983
2984         /* If we can't find the variable, something went wrong */
2985         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2986                                              event_name, field_name);
2987         if (IS_ERR_OR_NULL(event_var)) {
2988                 kfree(var_hist->cmd);
2989                 kfree(var_hist);
2990                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2991                 return ERR_PTR(-EINVAL);
2992         }
2993
2994         n = target_hist_data->n_field_var_hists;
2995         target_hist_data->field_var_hists[n] = var_hist;
2996         target_hist_data->n_field_var_hists++;
2997
2998         return event_var;
2999 }
3000
3001 static struct hist_field *
3002 find_target_event_var(struct hist_trigger_data *hist_data,
3003                       char *subsys_name, char *event_name, char *var_name)
3004 {
3005         struct trace_event_file *file = hist_data->event_file;
3006         struct hist_field *hist_field = NULL;
3007
3008         if (subsys_name) {
3009                 struct trace_event_call *call;
3010
3011                 if (!event_name)
3012                         return NULL;
3013
3014                 call = file->event_call;
3015
3016                 if (strcmp(subsys_name, call->class->system) != 0)
3017                         return NULL;
3018
3019                 if (strcmp(event_name, trace_event_name(call)) != 0)
3020                         return NULL;
3021         }
3022
3023         hist_field = find_var_field(hist_data, var_name);
3024
3025         return hist_field;
3026 }
3027
3028 static inline void __update_field_vars(struct tracing_map_elt *elt,
3029                                        struct trace_buffer *buffer,
3030                                        struct ring_buffer_event *rbe,
3031                                        void *rec,
3032                                        struct field_var **field_vars,
3033                                        unsigned int n_field_vars,
3034                                        unsigned int field_var_str_start)
3035 {
3036         struct hist_elt_data *elt_data = elt->private_data;
3037         unsigned int i, j, var_idx;
3038         u64 var_val;
3039
3040         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3041                 struct field_var *field_var = field_vars[i];
3042                 struct hist_field *var = field_var->var;
3043                 struct hist_field *val = field_var->val;
3044
3045                 var_val = val->fn(val, elt, buffer, rbe, rec);
3046                 var_idx = var->var.idx;
3047
3048                 if (val->flags & HIST_FIELD_FL_STRING) {
3049                         char *str = elt_data->field_var_str[j++];
3050                         char *val_str = (char *)(uintptr_t)var_val;
3051                         unsigned int size;
3052
3053                         size = min(val->size, STR_VAR_LEN_MAX);
3054                         strscpy(str, val_str, size);
3055                         var_val = (u64)(uintptr_t)str;
3056                 }
3057                 tracing_map_set_var(elt, var_idx, var_val);
3058         }
3059 }
3060
3061 static void update_field_vars(struct hist_trigger_data *hist_data,
3062                               struct tracing_map_elt *elt,
3063                               struct trace_buffer *buffer,
3064                               struct ring_buffer_event *rbe,
3065                               void *rec)
3066 {
3067         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3068                             hist_data->n_field_vars, 0);
3069 }
3070
3071 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3072                                  struct tracing_map_elt *elt,
3073                                  struct trace_buffer *buffer,  void *rec,
3074                                  struct ring_buffer_event *rbe, void *key,
3075                                  struct action_data *data, u64 *var_ref_vals)
3076 {
3077         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3078                             hist_data->n_save_vars, hist_data->n_field_var_str);
3079 }
3080
3081 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3082                                      struct trace_event_file *file,
3083                                      char *name, int size, const char *type)
3084 {
3085         struct hist_field *var;
3086         int idx;
3087
3088         if (find_var(hist_data, file, name) && !hist_data->remove) {
3089                 var = ERR_PTR(-EINVAL);
3090                 goto out;
3091         }
3092
3093         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3094         if (!var) {
3095                 var = ERR_PTR(-ENOMEM);
3096                 goto out;
3097         }
3098
3099         idx = tracing_map_add_var(hist_data->map);
3100         if (idx < 0) {
3101                 kfree(var);
3102                 var = ERR_PTR(-EINVAL);
3103                 goto out;
3104         }
3105
3106         var->ref = 1;
3107         var->flags = HIST_FIELD_FL_VAR;
3108         var->var.idx = idx;
3109         var->var.hist_data = var->hist_data = hist_data;
3110         var->size = size;
3111         var->var.name = kstrdup(name, GFP_KERNEL);
3112         var->type = kstrdup_const(type, GFP_KERNEL);
3113         if (!var->var.name || !var->type) {
3114                 kfree_const(var->type);
3115                 kfree(var->var.name);
3116                 kfree(var);
3117                 var = ERR_PTR(-ENOMEM);
3118         }
3119  out:
3120         return var;
3121 }
3122
3123 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3124                                           struct trace_event_file *file,
3125                                           char *field_name)
3126 {
3127         struct hist_field *val = NULL, *var = NULL;
3128         unsigned long flags = HIST_FIELD_FL_VAR;
3129         struct trace_array *tr = file->tr;
3130         struct field_var *field_var;
3131         int ret = 0;
3132
3133         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3134                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3135                 ret = -EINVAL;
3136                 goto err;
3137         }
3138
3139         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3140         if (IS_ERR(val)) {
3141                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3142                 ret = PTR_ERR(val);
3143                 goto err;
3144         }
3145
3146         var = create_var(hist_data, file, field_name, val->size, val->type);
3147         if (IS_ERR(var)) {
3148                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3149                 kfree(val);
3150                 ret = PTR_ERR(var);
3151                 goto err;
3152         }
3153
3154         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3155         if (!field_var) {
3156                 kfree(val);
3157                 kfree(var);
3158                 ret =  -ENOMEM;
3159                 goto err;
3160         }
3161
3162         field_var->var = var;
3163         field_var->val = val;
3164  out:
3165         return field_var;
3166  err:
3167         field_var = ERR_PTR(ret);
3168         goto out;
3169 }
3170
3171 /**
3172  * create_target_field_var - Automatically create a variable for a field
3173  * @target_hist_data: The target hist trigger
3174  * @subsys_name: Optional subsystem name
3175  * @event_name: Optional event name
3176  * @var_name: The name of the field (and the resulting variable)
3177  *
3178  * Hist trigger actions fetch data from variables, not directly from
3179  * events.  However, for convenience, users are allowed to directly
3180  * specify an event field in an action, which will be automatically
3181  * converted into a variable on their behalf.
3182
3183  * This function creates a field variable with the name var_name on
3184  * the hist trigger currently being defined on the target event.  If
3185  * subsys_name and event_name are specified, this function simply
3186  * verifies that they do in fact match the target event subsystem and
3187  * event name.
3188  *
3189  * Return: The variable created for the field.
3190  */
3191 static struct field_var *
3192 create_target_field_var(struct hist_trigger_data *target_hist_data,
3193                         char *subsys_name, char *event_name, char *var_name)
3194 {
3195         struct trace_event_file *file = target_hist_data->event_file;
3196
3197         if (subsys_name) {
3198                 struct trace_event_call *call;
3199
3200                 if (!event_name)
3201                         return NULL;
3202
3203                 call = file->event_call;
3204
3205                 if (strcmp(subsys_name, call->class->system) != 0)
3206                         return NULL;
3207
3208                 if (strcmp(event_name, trace_event_name(call)) != 0)
3209                         return NULL;
3210         }
3211
3212         return create_field_var(target_hist_data, file, var_name);
3213 }
3214
3215 static bool check_track_val_max(u64 track_val, u64 var_val)
3216 {
3217         if (var_val <= track_val)
3218                 return false;
3219
3220         return true;
3221 }
3222
3223 static bool check_track_val_changed(u64 track_val, u64 var_val)
3224 {
3225         if (var_val == track_val)
3226                 return false;
3227
3228         return true;
3229 }
3230
3231 static u64 get_track_val(struct hist_trigger_data *hist_data,
3232                          struct tracing_map_elt *elt,
3233                          struct action_data *data)
3234 {
3235         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3236         u64 track_val;
3237
3238         track_val = tracing_map_read_var(elt, track_var_idx);
3239
3240         return track_val;
3241 }
3242
3243 static void save_track_val(struct hist_trigger_data *hist_data,
3244                            struct tracing_map_elt *elt,
3245                            struct action_data *data, u64 var_val)
3246 {
3247         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3248
3249         tracing_map_set_var(elt, track_var_idx, var_val);
3250 }
3251
3252 static void save_track_data(struct hist_trigger_data *hist_data,
3253                             struct tracing_map_elt *elt,
3254                             struct trace_buffer *buffer, void *rec,
3255                             struct ring_buffer_event *rbe, void *key,
3256                             struct action_data *data, u64 *var_ref_vals)
3257 {
3258         if (data->track_data.save_data)
3259                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3260                                            key, data, var_ref_vals);
3261 }
3262
3263 static bool check_track_val(struct tracing_map_elt *elt,
3264                             struct action_data *data,
3265                             u64 var_val)
3266 {
3267         struct hist_trigger_data *hist_data;
3268         u64 track_val;
3269
3270         hist_data = data->track_data.track_var->hist_data;
3271         track_val = get_track_val(hist_data, elt, data);
3272
3273         return data->track_data.check_val(track_val, var_val);
3274 }
3275
3276 #ifdef CONFIG_TRACER_SNAPSHOT
3277 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3278 {
3279         /* called with tr->max_lock held */
3280         struct track_data *track_data = tr->cond_snapshot->cond_data;
3281         struct hist_elt_data *elt_data, *track_elt_data;
3282         struct snapshot_context *context = cond_data;
3283         struct action_data *action;
3284         u64 track_val;
3285
3286         if (!track_data)
3287                 return false;
3288
3289         action = track_data->action_data;
3290
3291         track_val = get_track_val(track_data->hist_data, context->elt,
3292                                   track_data->action_data);
3293
3294         if (!action->track_data.check_val(track_data->track_val, track_val))
3295                 return false;
3296
3297         track_data->track_val = track_val;
3298         memcpy(track_data->key, context->key, track_data->key_len);
3299
3300         elt_data = context->elt->private_data;
3301         track_elt_data = track_data->elt.private_data;
3302         if (elt_data->comm)
3303                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3304
3305         track_data->updated = true;
3306
3307         return true;
3308 }
3309
3310 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3311                                      struct tracing_map_elt *elt,
3312                                      struct trace_buffer *buffer, void *rec,
3313                                      struct ring_buffer_event *rbe, void *key,
3314                                      struct action_data *data,
3315                                      u64 *var_ref_vals)
3316 {
3317         struct trace_event_file *file = hist_data->event_file;
3318         struct snapshot_context context;
3319
3320         context.elt = elt;
3321         context.key = key;
3322
3323         tracing_snapshot_cond(file->tr, &context);
3324 }
3325
3326 static void hist_trigger_print_key(struct seq_file *m,
3327                                    struct hist_trigger_data *hist_data,
3328                                    void *key,
3329                                    struct tracing_map_elt *elt);
3330
3331 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3332 {
3333         unsigned int i;
3334
3335         if (!hist_data->n_actions)
3336                 return NULL;
3337
3338         for (i = 0; i < hist_data->n_actions; i++) {
3339                 struct action_data *data = hist_data->actions[i];
3340
3341                 if (data->action == ACTION_SNAPSHOT)
3342                         return data;
3343         }
3344
3345         return NULL;
3346 }
3347
3348 static void track_data_snapshot_print(struct seq_file *m,
3349                                       struct hist_trigger_data *hist_data)
3350 {
3351         struct trace_event_file *file = hist_data->event_file;
3352         struct track_data *track_data;
3353         struct action_data *action;
3354
3355         track_data = tracing_cond_snapshot_data(file->tr);
3356         if (!track_data)
3357                 return;
3358
3359         if (!track_data->updated)
3360                 return;
3361
3362         action = snapshot_action(hist_data);
3363         if (!action)
3364                 return;
3365
3366         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3367         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3368                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3369                    action->track_data.var_str, track_data->track_val);
3370
3371         seq_puts(m, "\ttriggered by event with key: ");
3372         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3373         seq_putc(m, '\n');
3374 }
3375 #else
3376 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3377 {
3378         return false;
3379 }
3380 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3381                                      struct tracing_map_elt *elt,
3382                                      struct trace_buffer *buffer, void *rec,
3383                                      struct ring_buffer_event *rbe, void *key,
3384                                      struct action_data *data,
3385                                      u64 *var_ref_vals) {}
3386 static void track_data_snapshot_print(struct seq_file *m,
3387                                       struct hist_trigger_data *hist_data) {}
3388 #endif /* CONFIG_TRACER_SNAPSHOT */
3389
3390 static void track_data_print(struct seq_file *m,
3391                              struct hist_trigger_data *hist_data,
3392                              struct tracing_map_elt *elt,
3393                              struct action_data *data)
3394 {
3395         u64 track_val = get_track_val(hist_data, elt, data);
3396         unsigned int i, save_var_idx;
3397
3398         if (data->handler == HANDLER_ONMAX)
3399                 seq_printf(m, "\n\tmax: %10llu", track_val);
3400         else if (data->handler == HANDLER_ONCHANGE)
3401                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3402
3403         if (data->action == ACTION_SNAPSHOT)
3404                 return;
3405
3406         for (i = 0; i < hist_data->n_save_vars; i++) {
3407                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3408                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3409                 u64 val;
3410
3411                 save_var_idx = save_var->var.idx;
3412
3413                 val = tracing_map_read_var(elt, save_var_idx);
3414
3415                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3416                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3417                                    (char *)(uintptr_t)(val));
3418                 } else
3419                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3420         }
3421 }
3422
3423 static void ontrack_action(struct hist_trigger_data *hist_data,
3424                            struct tracing_map_elt *elt,
3425                            struct trace_buffer *buffer, void *rec,
3426                            struct ring_buffer_event *rbe, void *key,
3427                            struct action_data *data, u64 *var_ref_vals)
3428 {
3429         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3430
3431         if (check_track_val(elt, data, var_val)) {
3432                 save_track_val(hist_data, elt, data, var_val);
3433                 save_track_data(hist_data, elt, buffer, rec, rbe,
3434                                 key, data, var_ref_vals);
3435         }
3436 }
3437
3438 static void action_data_destroy(struct action_data *data)
3439 {
3440         unsigned int i;
3441
3442         lockdep_assert_held(&event_mutex);
3443
3444         kfree(data->action_name);
3445
3446         for (i = 0; i < data->n_params; i++)
3447                 kfree(data->params[i]);
3448
3449         if (data->synth_event)
3450                 data->synth_event->ref--;
3451
3452         kfree(data->synth_event_name);
3453
3454         kfree(data);
3455 }
3456
3457 static void track_data_destroy(struct hist_trigger_data *hist_data,
3458                                struct action_data *data)
3459 {
3460         struct trace_event_file *file = hist_data->event_file;
3461
3462         destroy_hist_field(data->track_data.track_var, 0);
3463
3464         if (data->action == ACTION_SNAPSHOT) {
3465                 struct track_data *track_data;
3466
3467                 track_data = tracing_cond_snapshot_data(file->tr);
3468                 if (track_data && track_data->hist_data == hist_data) {
3469                         tracing_snapshot_cond_disable(file->tr);
3470                         track_data_free(track_data);
3471                 }
3472         }
3473
3474         kfree(data->track_data.var_str);
3475
3476         action_data_destroy(data);
3477 }
3478
3479 static int action_create(struct hist_trigger_data *hist_data,
3480                          struct action_data *data);
3481
3482 static int track_data_create(struct hist_trigger_data *hist_data,
3483                              struct action_data *data)
3484 {
3485         struct hist_field *var_field, *ref_field, *track_var = NULL;
3486         struct trace_event_file *file = hist_data->event_file;
3487         struct trace_array *tr = file->tr;
3488         char *track_data_var_str;
3489         int ret = 0;
3490
3491         track_data_var_str = data->track_data.var_str;
3492         if (track_data_var_str[0] != '$') {
3493                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3494                 return -EINVAL;
3495         }
3496         track_data_var_str++;
3497
3498         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3499         if (!var_field) {
3500                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3501                 return -EINVAL;
3502         }
3503
3504         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3505         if (!ref_field)
3506                 return -ENOMEM;
3507
3508         data->track_data.var_ref = ref_field;
3509
3510         if (data->handler == HANDLER_ONMAX)
3511                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3512         if (IS_ERR(track_var)) {
3513                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3514                 ret = PTR_ERR(track_var);
3515                 goto out;
3516         }
3517
3518         if (data->handler == HANDLER_ONCHANGE)
3519                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3520         if (IS_ERR(track_var)) {
3521                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3522                 ret = PTR_ERR(track_var);
3523                 goto out;
3524         }
3525         data->track_data.track_var = track_var;
3526
3527         ret = action_create(hist_data, data);
3528  out:
3529         return ret;
3530 }
3531
3532 static int parse_action_params(struct trace_array *tr, char *params,
3533                                struct action_data *data)
3534 {
3535         char *param, *saved_param;
3536         bool first_param = true;
3537         int ret = 0;
3538
3539         while (params) {
3540                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3541                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3542                         goto out;
3543                 }
3544
3545                 param = strsep(&params, ",");
3546                 if (!param) {
3547                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3548                         ret = -EINVAL;
3549                         goto out;
3550                 }
3551
3552                 param = strstrip(param);
3553                 if (strlen(param) < 2) {
3554                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3555                         ret = -EINVAL;
3556                         goto out;
3557                 }
3558
3559                 saved_param = kstrdup(param, GFP_KERNEL);
3560                 if (!saved_param) {
3561                         ret = -ENOMEM;
3562                         goto out;
3563                 }
3564
3565                 if (first_param && data->use_trace_keyword) {
3566                         data->synth_event_name = saved_param;
3567                         first_param = false;
3568                         continue;
3569                 }
3570                 first_param = false;
3571
3572                 data->params[data->n_params++] = saved_param;
3573         }
3574  out:
3575         return ret;
3576 }
3577
3578 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3579                         enum handler_id handler)
3580 {
3581         char *action_name;
3582         int ret = 0;
3583
3584         strsep(&str, ".");
3585         if (!str) {
3586                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3587                 ret = -EINVAL;
3588                 goto out;
3589         }
3590
3591         action_name = strsep(&str, "(");
3592         if (!action_name || !str) {
3593                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3594                 ret = -EINVAL;
3595                 goto out;
3596         }
3597
3598         if (str_has_prefix(action_name, "save")) {
3599                 char *params = strsep(&str, ")");
3600
3601                 if (!params) {
3602                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3603                         ret = -EINVAL;
3604                         goto out;
3605                 }
3606
3607                 ret = parse_action_params(tr, params, data);
3608                 if (ret)
3609                         goto out;
3610
3611                 if (handler == HANDLER_ONMAX)
3612                         data->track_data.check_val = check_track_val_max;
3613                 else if (handler == HANDLER_ONCHANGE)
3614                         data->track_data.check_val = check_track_val_changed;
3615                 else {
3616                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3617                         ret = -EINVAL;
3618                         goto out;
3619                 }
3620
3621                 data->track_data.save_data = save_track_data_vars;
3622                 data->fn = ontrack_action;
3623                 data->action = ACTION_SAVE;
3624         } else if (str_has_prefix(action_name, "snapshot")) {
3625                 char *params = strsep(&str, ")");
3626
3627                 if (!str) {
3628                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3629                         ret = -EINVAL;
3630                         goto out;
3631                 }
3632
3633                 if (handler == HANDLER_ONMAX)
3634                         data->track_data.check_val = check_track_val_max;
3635                 else if (handler == HANDLER_ONCHANGE)
3636                         data->track_data.check_val = check_track_val_changed;
3637                 else {
3638                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3639                         ret = -EINVAL;
3640                         goto out;
3641                 }
3642
3643                 data->track_data.save_data = save_track_data_snapshot;
3644                 data->fn = ontrack_action;
3645                 data->action = ACTION_SNAPSHOT;
3646         } else {
3647                 char *params = strsep(&str, ")");
3648
3649                 if (str_has_prefix(action_name, "trace"))
3650                         data->use_trace_keyword = true;
3651
3652                 if (params) {
3653                         ret = parse_action_params(tr, params, data);
3654                         if (ret)
3655                                 goto out;
3656                 }
3657
3658                 if (handler == HANDLER_ONMAX)
3659                         data->track_data.check_val = check_track_val_max;
3660                 else if (handler == HANDLER_ONCHANGE)
3661                         data->track_data.check_val = check_track_val_changed;
3662
3663                 if (handler != HANDLER_ONMATCH) {
3664                         data->track_data.save_data = action_trace;
3665                         data->fn = ontrack_action;
3666                 } else
3667                         data->fn = action_trace;
3668
3669                 data->action = ACTION_TRACE;
3670         }
3671
3672         data->action_name = kstrdup(action_name, GFP_KERNEL);
3673         if (!data->action_name) {
3674                 ret = -ENOMEM;
3675                 goto out;
3676         }
3677
3678         data->handler = handler;
3679  out:
3680         return ret;
3681 }
3682
3683 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3684                                             char *str, enum handler_id handler)
3685 {
3686         struct action_data *data;
3687         int ret = -EINVAL;
3688         char *var_str;
3689
3690         data = kzalloc(sizeof(*data), GFP_KERNEL);
3691         if (!data)
3692                 return ERR_PTR(-ENOMEM);
3693
3694         var_str = strsep(&str, ")");
3695         if (!var_str || !str) {
3696                 ret = -EINVAL;
3697                 goto free;
3698         }
3699
3700         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3701         if (!data->track_data.var_str) {
3702                 ret = -ENOMEM;
3703                 goto free;
3704         }
3705
3706         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3707         if (ret)
3708                 goto free;
3709  out:
3710         return data;
3711  free:
3712         track_data_destroy(hist_data, data);
3713         data = ERR_PTR(ret);
3714         goto out;
3715 }
3716
3717 static void onmatch_destroy(struct action_data *data)
3718 {
3719         kfree(data->match_data.event);
3720         kfree(data->match_data.event_system);
3721
3722         action_data_destroy(data);
3723 }
3724
3725 static void destroy_field_var(struct field_var *field_var)
3726 {
3727         if (!field_var)
3728                 return;
3729
3730         destroy_hist_field(field_var->var, 0);
3731         destroy_hist_field(field_var->val, 0);
3732
3733         kfree(field_var);
3734 }
3735
3736 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3737 {
3738         unsigned int i;
3739
3740         for (i = 0; i < hist_data->n_field_vars; i++)
3741                 destroy_field_var(hist_data->field_vars[i]);
3742
3743         for (i = 0; i < hist_data->n_save_vars; i++)
3744                 destroy_field_var(hist_data->save_vars[i]);
3745 }
3746
3747 static void save_field_var(struct hist_trigger_data *hist_data,
3748                            struct field_var *field_var)
3749 {
3750         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3751
3752         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3753                 hist_data->n_field_var_str++;
3754 }
3755
3756
3757 static int check_synth_field(struct synth_event *event,
3758                              struct hist_field *hist_field,
3759                              unsigned int field_pos)
3760 {
3761         struct synth_field *field;
3762
3763         if (field_pos >= event->n_fields)
3764                 return -EINVAL;
3765
3766         field = event->fields[field_pos];
3767
3768         /*
3769          * A dynamic string synth field can accept static or
3770          * dynamic. A static string synth field can only accept a
3771          * same-sized static string, which is checked for later.
3772          */
3773         if (strstr(hist_field->type, "char[") && field->is_string
3774             && field->is_dynamic)
3775                 return 0;
3776
3777         if (strcmp(field->type, hist_field->type) != 0) {
3778                 if (field->size != hist_field->size ||
3779                     (!field->is_string && field->is_signed != hist_field->is_signed))
3780                         return -EINVAL;
3781         }
3782
3783         return 0;
3784 }
3785
3786 static struct hist_field *
3787 trace_action_find_var(struct hist_trigger_data *hist_data,
3788                       struct action_data *data,
3789                       char *system, char *event, char *var)
3790 {
3791         struct trace_array *tr = hist_data->event_file->tr;
3792         struct hist_field *hist_field;
3793
3794         var++; /* skip '$' */
3795
3796         hist_field = find_target_event_var(hist_data, system, event, var);
3797         if (!hist_field) {
3798                 if (!system && data->handler == HANDLER_ONMATCH) {
3799                         system = data->match_data.event_system;
3800                         event = data->match_data.event;
3801                 }
3802
3803                 hist_field = find_event_var(hist_data, system, event, var);
3804         }
3805
3806         if (!hist_field)
3807                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3808
3809         return hist_field;
3810 }
3811
3812 static struct hist_field *
3813 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3814                               struct action_data *data, char *system,
3815                               char *event, char *var)
3816 {
3817         struct hist_field *hist_field = NULL;
3818         struct field_var *field_var;
3819
3820         /*
3821          * First try to create a field var on the target event (the
3822          * currently being defined).  This will create a variable for
3823          * unqualified fields on the target event, or if qualified,
3824          * target fields that have qualified names matching the target.
3825          */
3826         field_var = create_target_field_var(hist_data, system, event, var);
3827
3828         if (field_var && !IS_ERR(field_var)) {
3829                 save_field_var(hist_data, field_var);
3830                 hist_field = field_var->var;
3831         } else {
3832                 field_var = NULL;
3833                 /*
3834                  * If no explicit system.event is specified, default to
3835                  * looking for fields on the onmatch(system.event.xxx)
3836                  * event.
3837                  */
3838                 if (!system && data->handler == HANDLER_ONMATCH) {
3839                         system = data->match_data.event_system;
3840                         event = data->match_data.event;
3841                 }
3842
3843                 if (!event)
3844                         goto free;
3845                 /*
3846                  * At this point, we're looking at a field on another
3847                  * event.  Because we can't modify a hist trigger on
3848                  * another event to add a variable for a field, we need
3849                  * to create a new trigger on that event and create the
3850                  * variable at the same time.
3851                  */
3852                 hist_field = create_field_var_hist(hist_data, system, event, var);
3853                 if (IS_ERR(hist_field))
3854                         goto free;
3855         }
3856  out:
3857         return hist_field;
3858  free:
3859         destroy_field_var(field_var);
3860         hist_field = NULL;
3861         goto out;
3862 }
3863
3864 static int trace_action_create(struct hist_trigger_data *hist_data,
3865                                struct action_data *data)
3866 {
3867         struct trace_array *tr = hist_data->event_file->tr;
3868         char *event_name, *param, *system = NULL;
3869         struct hist_field *hist_field, *var_ref;
3870         unsigned int i;
3871         unsigned int field_pos = 0;
3872         struct synth_event *event;
3873         char *synth_event_name;
3874         int var_ref_idx, ret = 0;
3875
3876         lockdep_assert_held(&event_mutex);
3877
3878         if (data->use_trace_keyword)
3879                 synth_event_name = data->synth_event_name;
3880         else
3881                 synth_event_name = data->action_name;
3882
3883         event = find_synth_event(synth_event_name);
3884         if (!event) {
3885                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3886                 return -EINVAL;
3887         }
3888
3889         event->ref++;
3890
3891         for (i = 0; i < data->n_params; i++) {
3892                 char *p;
3893
3894                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3895                 if (!param) {
3896                         ret = -ENOMEM;
3897                         goto err;
3898                 }
3899
3900                 system = strsep(&param, ".");
3901                 if (!param) {
3902                         param = (char *)system;
3903                         system = event_name = NULL;
3904                 } else {
3905                         event_name = strsep(&param, ".");
3906                         if (!param) {
3907                                 kfree(p);
3908                                 ret = -EINVAL;
3909                                 goto err;
3910                         }
3911                 }
3912
3913                 if (param[0] == '$')
3914                         hist_field = trace_action_find_var(hist_data, data,
3915                                                            system, event_name,
3916                                                            param);
3917                 else
3918                         hist_field = trace_action_create_field_var(hist_data,
3919                                                                    data,
3920                                                                    system,
3921                                                                    event_name,
3922                                                                    param);
3923
3924                 if (!hist_field) {
3925                         kfree(p);
3926                         ret = -EINVAL;
3927                         goto err;
3928                 }
3929
3930                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3931                         var_ref = create_var_ref(hist_data, hist_field,
3932                                                  system, event_name);
3933                         if (!var_ref) {
3934                                 kfree(p);
3935                                 ret = -ENOMEM;
3936                                 goto err;
3937                         }
3938
3939                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3940                         if (WARN_ON(var_ref_idx < 0)) {
3941                                 kfree(p);
3942                                 ret = var_ref_idx;
3943                                 goto err;
3944                         }
3945
3946                         data->var_ref_idx[i] = var_ref_idx;
3947
3948                         field_pos++;
3949                         kfree(p);
3950                         continue;
3951                 }
3952
3953                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3954                 kfree(p);
3955                 ret = -EINVAL;
3956                 goto err;
3957         }
3958
3959         if (field_pos != event->n_fields) {
3960                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3961                 ret = -EINVAL;
3962                 goto err;
3963         }
3964
3965         data->synth_event = event;
3966  out:
3967         return ret;
3968  err:
3969         event->ref--;
3970
3971         goto out;
3972 }
3973
3974 static int action_create(struct hist_trigger_data *hist_data,
3975                          struct action_data *data)
3976 {
3977         struct trace_event_file *file = hist_data->event_file;
3978         struct trace_array *tr = file->tr;
3979         struct track_data *track_data;
3980         struct field_var *field_var;
3981         unsigned int i;
3982         char *param;
3983         int ret = 0;
3984
3985         if (data->action == ACTION_TRACE)
3986                 return trace_action_create(hist_data, data);
3987
3988         if (data->action == ACTION_SNAPSHOT) {
3989                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3990                 if (IS_ERR(track_data)) {
3991                         ret = PTR_ERR(track_data);
3992                         goto out;
3993                 }
3994
3995                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3996                                                    cond_snapshot_update);
3997                 if (ret)
3998                         track_data_free(track_data);
3999
4000                 goto out;
4001         }
4002
4003         if (data->action == ACTION_SAVE) {
4004                 if (hist_data->n_save_vars) {
4005                         ret = -EEXIST;
4006                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4007                         goto out;
4008                 }
4009
4010                 for (i = 0; i < data->n_params; i++) {
4011                         param = kstrdup(data->params[i], GFP_KERNEL);
4012                         if (!param) {
4013                                 ret = -ENOMEM;
4014                                 goto out;
4015                         }
4016
4017                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4018                         if (IS_ERR(field_var)) {
4019                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4020                                          errpos(param));
4021                                 ret = PTR_ERR(field_var);
4022                                 kfree(param);
4023                                 goto out;
4024                         }
4025
4026                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4027                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4028                                 hist_data->n_save_var_str++;
4029                         kfree(param);
4030                 }
4031         }
4032  out:
4033         return ret;
4034 }
4035
4036 static int onmatch_create(struct hist_trigger_data *hist_data,
4037                           struct action_data *data)
4038 {
4039         return action_create(hist_data, data);
4040 }
4041
4042 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4043 {
4044         char *match_event, *match_event_system;
4045         struct action_data *data;
4046         int ret = -EINVAL;
4047
4048         data = kzalloc(sizeof(*data), GFP_KERNEL);
4049         if (!data)
4050                 return ERR_PTR(-ENOMEM);
4051
4052         match_event = strsep(&str, ")");
4053         if (!match_event || !str) {
4054                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4055                 goto free;
4056         }
4057
4058         match_event_system = strsep(&match_event, ".");
4059         if (!match_event) {
4060                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4061                 goto free;
4062         }
4063
4064         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4065                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4066                 goto free;
4067         }
4068
4069         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4070         if (!data->match_data.event) {
4071                 ret = -ENOMEM;
4072                 goto free;
4073         }
4074
4075         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4076         if (!data->match_data.event_system) {
4077                 ret = -ENOMEM;
4078                 goto free;
4079         }
4080
4081         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4082         if (ret)
4083                 goto free;
4084  out:
4085         return data;
4086  free:
4087         onmatch_destroy(data);
4088         data = ERR_PTR(ret);
4089         goto out;
4090 }
4091
4092 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4093 {
4094         hist_data->fields[HITCOUNT_IDX] =
4095                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4096         if (!hist_data->fields[HITCOUNT_IDX])
4097                 return -ENOMEM;
4098
4099         hist_data->n_vals++;
4100         hist_data->n_fields++;
4101
4102         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4103                 return -EINVAL;
4104
4105         return 0;
4106 }
4107
4108 static int __create_val_field(struct hist_trigger_data *hist_data,
4109                               unsigned int val_idx,
4110                               struct trace_event_file *file,
4111                               char *var_name, char *field_str,
4112                               unsigned long flags)
4113 {
4114         struct hist_field *hist_field;
4115         int ret = 0, n_subexprs = 0;
4116
4117         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4118         if (IS_ERR(hist_field)) {
4119                 ret = PTR_ERR(hist_field);
4120                 goto out;
4121         }
4122
4123         hist_data->fields[val_idx] = hist_field;
4124
4125         ++hist_data->n_vals;
4126         ++hist_data->n_fields;
4127
4128         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4129                 ret = -EINVAL;
4130  out:
4131         return ret;
4132 }
4133
4134 static int create_val_field(struct hist_trigger_data *hist_data,
4135                             unsigned int val_idx,
4136                             struct trace_event_file *file,
4137                             char *field_str)
4138 {
4139         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4140                 return -EINVAL;
4141
4142         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4143 }
4144
4145 static const char *no_comm = "(no comm)";
4146
4147 static u64 hist_field_execname(struct hist_field *hist_field,
4148                                struct tracing_map_elt *elt,
4149                                struct trace_buffer *buffer,
4150                                struct ring_buffer_event *rbe,
4151                                void *event)
4152 {
4153         struct hist_elt_data *elt_data;
4154
4155         if (WARN_ON_ONCE(!elt))
4156                 return (u64)(unsigned long)no_comm;
4157
4158         elt_data = elt->private_data;
4159
4160         if (WARN_ON_ONCE(!elt_data->comm))
4161                 return (u64)(unsigned long)no_comm;
4162
4163         return (u64)(unsigned long)(elt_data->comm);
4164 }
4165
4166 /* Convert a var that points to common_pid.execname to a string */
4167 static void update_var_execname(struct hist_field *hist_field)
4168 {
4169         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4170                 HIST_FIELD_FL_EXECNAME;
4171         hist_field->size = MAX_FILTER_STR_VAL;
4172         hist_field->is_signed = 0;
4173
4174         kfree_const(hist_field->type);
4175         hist_field->type = "char[]";
4176
4177         hist_field->fn = hist_field_execname;
4178 }
4179
4180 static int create_var_field(struct hist_trigger_data *hist_data,
4181                             unsigned int val_idx,
4182                             struct trace_event_file *file,
4183                             char *var_name, char *expr_str)
4184 {
4185         struct trace_array *tr = hist_data->event_file->tr;
4186         unsigned long flags = 0;
4187         int ret;
4188
4189         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4190                 return -EINVAL;
4191
4192         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4193                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4194                 return -EINVAL;
4195         }
4196
4197         flags |= HIST_FIELD_FL_VAR;
4198         hist_data->n_vars++;
4199         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4200                 return -EINVAL;
4201
4202         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4203
4204         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4205                 update_var_execname(hist_data->fields[val_idx]);
4206
4207         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4208                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4209
4210         return ret;
4211 }
4212
4213 static int create_val_fields(struct hist_trigger_data *hist_data,
4214                              struct trace_event_file *file)
4215 {
4216         char *fields_str, *field_str;
4217         unsigned int i, j = 1;
4218         int ret;
4219
4220         ret = create_hitcount_val(hist_data);
4221         if (ret)
4222                 goto out;
4223
4224         fields_str = hist_data->attrs->vals_str;
4225         if (!fields_str)
4226                 goto out;
4227
4228         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4229                      j < TRACING_MAP_VALS_MAX; i++) {
4230                 field_str = strsep(&fields_str, ",");
4231                 if (!field_str)
4232                         break;
4233
4234                 if (strcmp(field_str, "hitcount") == 0)
4235                         continue;
4236
4237                 ret = create_val_field(hist_data, j++, file, field_str);
4238                 if (ret)
4239                         goto out;
4240         }
4241
4242         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4243                 ret = -EINVAL;
4244  out:
4245         return ret;
4246 }
4247
4248 static int create_key_field(struct hist_trigger_data *hist_data,
4249                             unsigned int key_idx,
4250                             unsigned int key_offset,
4251                             struct trace_event_file *file,
4252                             char *field_str)
4253 {
4254         struct trace_array *tr = hist_data->event_file->tr;
4255         struct hist_field *hist_field = NULL;
4256         unsigned long flags = 0;
4257         unsigned int key_size;
4258         int ret = 0, n_subexprs = 0;
4259
4260         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4261                 return -EINVAL;
4262
4263         flags |= HIST_FIELD_FL_KEY;
4264
4265         if (strcmp(field_str, "stacktrace") == 0) {
4266                 flags |= HIST_FIELD_FL_STACKTRACE;
4267                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4268                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4269         } else {
4270                 hist_field = parse_expr(hist_data, file, field_str, flags,
4271                                         NULL, &n_subexprs);
4272                 if (IS_ERR(hist_field)) {
4273                         ret = PTR_ERR(hist_field);
4274                         goto out;
4275                 }
4276
4277                 if (field_has_hist_vars(hist_field, 0)) {
4278                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4279                         destroy_hist_field(hist_field, 0);
4280                         ret = -EINVAL;
4281                         goto out;
4282                 }
4283
4284                 key_size = hist_field->size;
4285         }
4286
4287         hist_data->fields[key_idx] = hist_field;
4288
4289         key_size = ALIGN(key_size, sizeof(u64));
4290         hist_data->fields[key_idx]->size = key_size;
4291         hist_data->fields[key_idx]->offset = key_offset;
4292
4293         hist_data->key_size += key_size;
4294
4295         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4296                 ret = -EINVAL;
4297                 goto out;
4298         }
4299
4300         hist_data->n_keys++;
4301         hist_data->n_fields++;
4302
4303         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4304                 return -EINVAL;
4305
4306         ret = key_size;
4307  out:
4308         return ret;
4309 }
4310
4311 static int create_key_fields(struct hist_trigger_data *hist_data,
4312                              struct trace_event_file *file)
4313 {
4314         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4315         char *fields_str, *field_str;
4316         int ret = -EINVAL;
4317
4318         fields_str = hist_data->attrs->keys_str;
4319         if (!fields_str)
4320                 goto out;
4321
4322         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4323                 field_str = strsep(&fields_str, ",");
4324                 if (!field_str)
4325                         break;
4326                 ret = create_key_field(hist_data, i, key_offset,
4327                                        file, field_str);
4328                 if (ret < 0)
4329                         goto out;
4330                 key_offset += ret;
4331         }
4332         if (fields_str) {
4333                 ret = -EINVAL;
4334                 goto out;
4335         }
4336         ret = 0;
4337  out:
4338         return ret;
4339 }
4340
4341 static int create_var_fields(struct hist_trigger_data *hist_data,
4342                              struct trace_event_file *file)
4343 {
4344         unsigned int i, j = hist_data->n_vals;
4345         int ret = 0;
4346
4347         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4348
4349         for (i = 0; i < n_vars; i++) {
4350                 char *var_name = hist_data->attrs->var_defs.name[i];
4351                 char *expr = hist_data->attrs->var_defs.expr[i];
4352
4353                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4354                 if (ret)
4355                         goto out;
4356         }
4357  out:
4358         return ret;
4359 }
4360
4361 static void free_var_defs(struct hist_trigger_data *hist_data)
4362 {
4363         unsigned int i;
4364
4365         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4366                 kfree(hist_data->attrs->var_defs.name[i]);
4367                 kfree(hist_data->attrs->var_defs.expr[i]);
4368         }
4369
4370         hist_data->attrs->var_defs.n_vars = 0;
4371 }
4372
4373 static int parse_var_defs(struct hist_trigger_data *hist_data)
4374 {
4375         struct trace_array *tr = hist_data->event_file->tr;
4376         char *s, *str, *var_name, *field_str;
4377         unsigned int i, j, n_vars = 0;
4378         int ret = 0;
4379
4380         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4381                 str = hist_data->attrs->assignment_str[i];
4382                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4383                         field_str = strsep(&str, ",");
4384                         if (!field_str)
4385                                 break;
4386
4387                         var_name = strsep(&field_str, "=");
4388                         if (!var_name || !field_str) {
4389                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4390                                          errpos(var_name));
4391                                 ret = -EINVAL;
4392                                 goto free;
4393                         }
4394
4395                         if (n_vars == TRACING_MAP_VARS_MAX) {
4396                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4397                                 ret = -EINVAL;
4398                                 goto free;
4399                         }
4400
4401                         s = kstrdup(var_name, GFP_KERNEL);
4402                         if (!s) {
4403                                 ret = -ENOMEM;
4404                                 goto free;
4405                         }
4406                         hist_data->attrs->var_defs.name[n_vars] = s;
4407
4408                         s = kstrdup(field_str, GFP_KERNEL);
4409                         if (!s) {
4410                                 ret = -ENOMEM;
4411                                 goto free;
4412                         }
4413                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4414
4415                         hist_data->attrs->var_defs.n_vars = n_vars;
4416                 }
4417         }
4418
4419         return ret;
4420  free:
4421         free_var_defs(hist_data);
4422
4423         return ret;
4424 }
4425
4426 static int create_hist_fields(struct hist_trigger_data *hist_data,
4427                               struct trace_event_file *file)
4428 {
4429         int ret;
4430
4431         ret = parse_var_defs(hist_data);
4432         if (ret)
4433                 goto out;
4434
4435         ret = create_val_fields(hist_data, file);
4436         if (ret)
4437                 goto out;
4438
4439         ret = create_var_fields(hist_data, file);
4440         if (ret)
4441                 goto out;
4442
4443         ret = create_key_fields(hist_data, file);
4444         if (ret)
4445                 goto out;
4446  out:
4447         free_var_defs(hist_data);
4448
4449         return ret;
4450 }
4451
4452 static int is_descending(struct trace_array *tr, const char *str)
4453 {
4454         if (!str)
4455                 return 0;
4456
4457         if (strcmp(str, "descending") == 0)
4458                 return 1;
4459
4460         if (strcmp(str, "ascending") == 0)
4461                 return 0;
4462
4463         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4464
4465         return -EINVAL;
4466 }
4467
4468 static int create_sort_keys(struct hist_trigger_data *hist_data)
4469 {
4470         struct trace_array *tr = hist_data->event_file->tr;
4471         char *fields_str = hist_data->attrs->sort_key_str;
4472         struct tracing_map_sort_key *sort_key;
4473         int descending, ret = 0;
4474         unsigned int i, j, k;
4475
4476         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4477
4478         if (!fields_str)
4479                 goto out;
4480
4481         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4482                 struct hist_field *hist_field;
4483                 char *field_str, *field_name;
4484                 const char *test_name;
4485
4486                 sort_key = &hist_data->sort_keys[i];
4487
4488                 field_str = strsep(&fields_str, ",");
4489                 if (!field_str)
4490                         break;
4491
4492                 if (!*field_str) {
4493                         ret = -EINVAL;
4494                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4495                         break;
4496                 }
4497
4498                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4499                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4500                         ret = -EINVAL;
4501                         break;
4502                 }
4503
4504                 field_name = strsep(&field_str, ".");
4505                 if (!field_name || !*field_name) {
4506                         ret = -EINVAL;
4507                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4508                         break;
4509                 }
4510
4511                 if (strcmp(field_name, "hitcount") == 0) {
4512                         descending = is_descending(tr, field_str);
4513                         if (descending < 0) {
4514                                 ret = descending;
4515                                 break;
4516                         }
4517                         sort_key->descending = descending;
4518                         continue;
4519                 }
4520
4521                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4522                         unsigned int idx;
4523
4524                         hist_field = hist_data->fields[j];
4525                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4526                                 continue;
4527
4528                         idx = k++;
4529
4530                         test_name = hist_field_name(hist_field, 0);
4531
4532                         if (strcmp(field_name, test_name) == 0) {
4533                                 sort_key->field_idx = idx;
4534                                 descending = is_descending(tr, field_str);
4535                                 if (descending < 0) {
4536                                         ret = descending;
4537                                         goto out;
4538                                 }
4539                                 sort_key->descending = descending;
4540                                 break;
4541                         }
4542                 }
4543                 if (j == hist_data->n_fields) {
4544                         ret = -EINVAL;
4545                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4546                         break;
4547                 }
4548         }
4549
4550         hist_data->n_sort_keys = i;
4551  out:
4552         return ret;
4553 }
4554
4555 static void destroy_actions(struct hist_trigger_data *hist_data)
4556 {
4557         unsigned int i;
4558
4559         for (i = 0; i < hist_data->n_actions; i++) {
4560                 struct action_data *data = hist_data->actions[i];
4561
4562                 if (data->handler == HANDLER_ONMATCH)
4563                         onmatch_destroy(data);
4564                 else if (data->handler == HANDLER_ONMAX ||
4565                          data->handler == HANDLER_ONCHANGE)
4566                         track_data_destroy(hist_data, data);
4567                 else
4568                         kfree(data);
4569         }
4570 }
4571
4572 static int parse_actions(struct hist_trigger_data *hist_data)
4573 {
4574         struct trace_array *tr = hist_data->event_file->tr;
4575         struct action_data *data;
4576         unsigned int i;
4577         int ret = 0;
4578         char *str;
4579         int len;
4580
4581         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4582                 str = hist_data->attrs->action_str[i];
4583
4584                 if ((len = str_has_prefix(str, "onmatch("))) {
4585                         char *action_str = str + len;
4586
4587                         data = onmatch_parse(tr, action_str);
4588                         if (IS_ERR(data)) {
4589                                 ret = PTR_ERR(data);
4590                                 break;
4591                         }
4592                 } else if ((len = str_has_prefix(str, "onmax("))) {
4593                         char *action_str = str + len;
4594
4595                         data = track_data_parse(hist_data, action_str,
4596                                                 HANDLER_ONMAX);
4597                         if (IS_ERR(data)) {
4598                                 ret = PTR_ERR(data);
4599                                 break;
4600                         }
4601                 } else if ((len = str_has_prefix(str, "onchange("))) {
4602                         char *action_str = str + len;
4603
4604                         data = track_data_parse(hist_data, action_str,
4605                                                 HANDLER_ONCHANGE);
4606                         if (IS_ERR(data)) {
4607                                 ret = PTR_ERR(data);
4608                                 break;
4609                         }
4610                 } else {
4611                         ret = -EINVAL;
4612                         break;
4613                 }
4614
4615                 hist_data->actions[hist_data->n_actions++] = data;
4616         }
4617
4618         return ret;
4619 }
4620
4621 static int create_actions(struct hist_trigger_data *hist_data)
4622 {
4623         struct action_data *data;
4624         unsigned int i;
4625         int ret = 0;
4626
4627         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4628                 data = hist_data->actions[i];
4629
4630                 if (data->handler == HANDLER_ONMATCH) {
4631                         ret = onmatch_create(hist_data, data);
4632                         if (ret)
4633                                 break;
4634                 } else if (data->handler == HANDLER_ONMAX ||
4635                            data->handler == HANDLER_ONCHANGE) {
4636                         ret = track_data_create(hist_data, data);
4637                         if (ret)
4638                                 break;
4639                 } else {
4640                         ret = -EINVAL;
4641                         break;
4642                 }
4643         }
4644
4645         return ret;
4646 }
4647
4648 static void print_actions(struct seq_file *m,
4649                           struct hist_trigger_data *hist_data,
4650                           struct tracing_map_elt *elt)
4651 {
4652         unsigned int i;
4653
4654         for (i = 0; i < hist_data->n_actions; i++) {
4655                 struct action_data *data = hist_data->actions[i];
4656
4657                 if (data->action == ACTION_SNAPSHOT)
4658                         continue;
4659
4660                 if (data->handler == HANDLER_ONMAX ||
4661                     data->handler == HANDLER_ONCHANGE)
4662                         track_data_print(m, hist_data, elt, data);
4663         }
4664 }
4665
4666 static void print_action_spec(struct seq_file *m,
4667                               struct hist_trigger_data *hist_data,
4668                               struct action_data *data)
4669 {
4670         unsigned int i;
4671
4672         if (data->action == ACTION_SAVE) {
4673                 for (i = 0; i < hist_data->n_save_vars; i++) {
4674                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4675                         if (i < hist_data->n_save_vars - 1)
4676                                 seq_puts(m, ",");
4677                 }
4678         } else if (data->action == ACTION_TRACE) {
4679                 if (data->use_trace_keyword)
4680                         seq_printf(m, "%s", data->synth_event_name);
4681                 for (i = 0; i < data->n_params; i++) {
4682                         if (i || data->use_trace_keyword)
4683                                 seq_puts(m, ",");
4684                         seq_printf(m, "%s", data->params[i]);
4685                 }
4686         }
4687 }
4688
4689 static void print_track_data_spec(struct seq_file *m,
4690                                   struct hist_trigger_data *hist_data,
4691                                   struct action_data *data)
4692 {
4693         if (data->handler == HANDLER_ONMAX)
4694                 seq_puts(m, ":onmax(");
4695         else if (data->handler == HANDLER_ONCHANGE)
4696                 seq_puts(m, ":onchange(");
4697         seq_printf(m, "%s", data->track_data.var_str);
4698         seq_printf(m, ").%s(", data->action_name);
4699
4700         print_action_spec(m, hist_data, data);
4701
4702         seq_puts(m, ")");
4703 }
4704
4705 static void print_onmatch_spec(struct seq_file *m,
4706                                struct hist_trigger_data *hist_data,
4707                                struct action_data *data)
4708 {
4709         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4710                    data->match_data.event);
4711
4712         seq_printf(m, "%s(", data->action_name);
4713
4714         print_action_spec(m, hist_data, data);
4715
4716         seq_puts(m, ")");
4717 }
4718
4719 static bool actions_match(struct hist_trigger_data *hist_data,
4720                           struct hist_trigger_data *hist_data_test)
4721 {
4722         unsigned int i, j;
4723
4724         if (hist_data->n_actions != hist_data_test->n_actions)
4725                 return false;
4726
4727         for (i = 0; i < hist_data->n_actions; i++) {
4728                 struct action_data *data = hist_data->actions[i];
4729                 struct action_data *data_test = hist_data_test->actions[i];
4730                 char *action_name, *action_name_test;
4731
4732                 if (data->handler != data_test->handler)
4733                         return false;
4734                 if (data->action != data_test->action)
4735                         return false;
4736
4737                 if (data->n_params != data_test->n_params)
4738                         return false;
4739
4740                 for (j = 0; j < data->n_params; j++) {
4741                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4742                                 return false;
4743                 }
4744
4745                 if (data->use_trace_keyword)
4746                         action_name = data->synth_event_name;
4747                 else
4748                         action_name = data->action_name;
4749
4750                 if (data_test->use_trace_keyword)
4751                         action_name_test = data_test->synth_event_name;
4752                 else
4753                         action_name_test = data_test->action_name;
4754
4755                 if (strcmp(action_name, action_name_test) != 0)
4756                         return false;
4757
4758                 if (data->handler == HANDLER_ONMATCH) {
4759                         if (strcmp(data->match_data.event_system,
4760                                    data_test->match_data.event_system) != 0)
4761                                 return false;
4762                         if (strcmp(data->match_data.event,
4763                                    data_test->match_data.event) != 0)
4764                                 return false;
4765                 } else if (data->handler == HANDLER_ONMAX ||
4766                            data->handler == HANDLER_ONCHANGE) {
4767                         if (strcmp(data->track_data.var_str,
4768                                    data_test->track_data.var_str) != 0)
4769                                 return false;
4770                 }
4771         }
4772
4773         return true;
4774 }
4775
4776
4777 static void print_actions_spec(struct seq_file *m,
4778                                struct hist_trigger_data *hist_data)
4779 {
4780         unsigned int i;
4781
4782         for (i = 0; i < hist_data->n_actions; i++) {
4783                 struct action_data *data = hist_data->actions[i];
4784
4785                 if (data->handler == HANDLER_ONMATCH)
4786                         print_onmatch_spec(m, hist_data, data);
4787                 else if (data->handler == HANDLER_ONMAX ||
4788                          data->handler == HANDLER_ONCHANGE)
4789                         print_track_data_spec(m, hist_data, data);
4790         }
4791 }
4792
4793 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4794 {
4795         unsigned int i;
4796
4797         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4798                 kfree(hist_data->field_var_hists[i]->cmd);
4799                 kfree(hist_data->field_var_hists[i]);
4800         }
4801 }
4802
4803 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4804 {
4805         if (!hist_data)
4806                 return;
4807
4808         destroy_hist_trigger_attrs(hist_data->attrs);
4809         destroy_hist_fields(hist_data);
4810         tracing_map_destroy(hist_data->map);
4811
4812         destroy_actions(hist_data);
4813         destroy_field_vars(hist_data);
4814         destroy_field_var_hists(hist_data);
4815
4816         kfree(hist_data);
4817 }
4818
4819 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4820 {
4821         struct tracing_map *map = hist_data->map;
4822         struct ftrace_event_field *field;
4823         struct hist_field *hist_field;
4824         int i, idx = 0;
4825
4826         for_each_hist_field(i, hist_data) {
4827                 hist_field = hist_data->fields[i];
4828                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4829                         tracing_map_cmp_fn_t cmp_fn;
4830
4831                         field = hist_field->field;
4832
4833                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4834                                 cmp_fn = tracing_map_cmp_none;
4835                         else if (!field)
4836                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4837                                                              hist_field->is_signed);
4838                         else if (is_string_field(field))
4839                                 cmp_fn = tracing_map_cmp_string;
4840                         else
4841                                 cmp_fn = tracing_map_cmp_num(field->size,
4842                                                              field->is_signed);
4843                         idx = tracing_map_add_key_field(map,
4844                                                         hist_field->offset,
4845                                                         cmp_fn);
4846                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4847                         idx = tracing_map_add_sum_field(map);
4848
4849                 if (idx < 0)
4850                         return idx;
4851
4852                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4853                         idx = tracing_map_add_var(map);
4854                         if (idx < 0)
4855                                 return idx;
4856                         hist_field->var.idx = idx;
4857                         hist_field->var.hist_data = hist_data;
4858                 }
4859         }
4860
4861         return 0;
4862 }
4863
4864 static struct hist_trigger_data *
4865 create_hist_data(unsigned int map_bits,
4866                  struct hist_trigger_attrs *attrs,
4867                  struct trace_event_file *file,
4868                  bool remove)
4869 {
4870         const struct tracing_map_ops *map_ops = NULL;
4871         struct hist_trigger_data *hist_data;
4872         int ret = 0;
4873
4874         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4875         if (!hist_data)
4876                 return ERR_PTR(-ENOMEM);
4877
4878         hist_data->attrs = attrs;
4879         hist_data->remove = remove;
4880         hist_data->event_file = file;
4881
4882         ret = parse_actions(hist_data);
4883         if (ret)
4884                 goto free;
4885
4886         ret = create_hist_fields(hist_data, file);
4887         if (ret)
4888                 goto free;
4889
4890         ret = create_sort_keys(hist_data);
4891         if (ret)
4892                 goto free;
4893
4894         map_ops = &hist_trigger_elt_data_ops;
4895
4896         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4897                                             map_ops, hist_data);
4898         if (IS_ERR(hist_data->map)) {
4899                 ret = PTR_ERR(hist_data->map);
4900                 hist_data->map = NULL;
4901                 goto free;
4902         }
4903
4904         ret = create_tracing_map_fields(hist_data);
4905         if (ret)
4906                 goto free;
4907  out:
4908         return hist_data;
4909  free:
4910         hist_data->attrs = NULL;
4911
4912         destroy_hist_data(hist_data);
4913
4914         hist_data = ERR_PTR(ret);
4915
4916         goto out;
4917 }
4918
4919 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4920                                     struct tracing_map_elt *elt,
4921                                     struct trace_buffer *buffer, void *rec,
4922                                     struct ring_buffer_event *rbe,
4923                                     u64 *var_ref_vals)
4924 {
4925         struct hist_elt_data *elt_data;
4926         struct hist_field *hist_field;
4927         unsigned int i, var_idx;
4928         u64 hist_val;
4929
4930         elt_data = elt->private_data;
4931         elt_data->var_ref_vals = var_ref_vals;
4932
4933         for_each_hist_val_field(i, hist_data) {
4934                 hist_field = hist_data->fields[i];
4935                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4936                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4937                         var_idx = hist_field->var.idx;
4938
4939                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4940                                 unsigned int str_start, var_str_idx, idx;
4941                                 char *str, *val_str;
4942                                 unsigned int size;
4943
4944                                 str_start = hist_data->n_field_var_str +
4945                                         hist_data->n_save_var_str;
4946                                 var_str_idx = hist_field->var_str_idx;
4947                                 idx = str_start + var_str_idx;
4948
4949                                 str = elt_data->field_var_str[idx];
4950                                 val_str = (char *)(uintptr_t)hist_val;
4951
4952                                 size = min(hist_field->size, STR_VAR_LEN_MAX);
4953                                 strscpy(str, val_str, size);
4954
4955                                 hist_val = (u64)(uintptr_t)str;
4956                         }
4957                         tracing_map_set_var(elt, var_idx, hist_val);
4958                         continue;
4959                 }
4960                 tracing_map_update_sum(elt, i, hist_val);
4961         }
4962
4963         for_each_hist_key_field(i, hist_data) {
4964                 hist_field = hist_data->fields[i];
4965                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4966                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4967                         var_idx = hist_field->var.idx;
4968                         tracing_map_set_var(elt, var_idx, hist_val);
4969                 }
4970         }
4971
4972         update_field_vars(hist_data, elt, buffer, rbe, rec);
4973 }
4974
4975 static inline void add_to_key(char *compound_key, void *key,
4976                               struct hist_field *key_field, void *rec)
4977 {
4978         size_t size = key_field->size;
4979
4980         if (key_field->flags & HIST_FIELD_FL_STRING) {
4981                 struct ftrace_event_field *field;
4982
4983                 field = key_field->field;
4984                 if (field->filter_type == FILTER_DYN_STRING ||
4985                     field->filter_type == FILTER_RDYN_STRING)
4986                         size = *(u32 *)(rec + field->offset) >> 16;
4987                 else if (field->filter_type == FILTER_STATIC_STRING)
4988                         size = field->size;
4989
4990                 /* ensure NULL-termination */
4991                 if (size > key_field->size - 1)
4992                         size = key_field->size - 1;
4993
4994                 strncpy(compound_key + key_field->offset, (char *)key, size);
4995         } else
4996                 memcpy(compound_key + key_field->offset, key, size);
4997 }
4998
4999 static void
5000 hist_trigger_actions(struct hist_trigger_data *hist_data,
5001                      struct tracing_map_elt *elt,
5002                      struct trace_buffer *buffer, void *rec,
5003                      struct ring_buffer_event *rbe, void *key,
5004                      u64 *var_ref_vals)
5005 {
5006         struct action_data *data;
5007         unsigned int i;
5008
5009         for (i = 0; i < hist_data->n_actions; i++) {
5010                 data = hist_data->actions[i];
5011                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5012         }
5013 }
5014
5015 static void event_hist_trigger(struct event_trigger_data *data,
5016                                struct trace_buffer *buffer, void *rec,
5017                                struct ring_buffer_event *rbe)
5018 {
5019         struct hist_trigger_data *hist_data = data->private_data;
5020         bool use_compound_key = (hist_data->n_keys > 1);
5021         unsigned long entries[HIST_STACKTRACE_DEPTH];
5022         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5023         char compound_key[HIST_KEY_SIZE_MAX];
5024         struct tracing_map_elt *elt = NULL;
5025         struct hist_field *key_field;
5026         u64 field_contents;
5027         void *key = NULL;
5028         unsigned int i;
5029
5030         memset(compound_key, 0, hist_data->key_size);
5031
5032         for_each_hist_key_field(i, hist_data) {
5033                 key_field = hist_data->fields[i];
5034
5035                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5036                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5037                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5038                                          HIST_STACKTRACE_SKIP);
5039                         key = entries;
5040                 } else {
5041                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5042                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5043                                 key = (void *)(unsigned long)field_contents;
5044                                 use_compound_key = true;
5045                         } else
5046                                 key = (void *)&field_contents;
5047                 }
5048
5049                 if (use_compound_key)
5050                         add_to_key(compound_key, key, key_field, rec);
5051         }
5052
5053         if (use_compound_key)
5054                 key = compound_key;
5055
5056         if (hist_data->n_var_refs &&
5057             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5058                 return;
5059
5060         elt = tracing_map_insert(hist_data->map, key);
5061         if (!elt)
5062                 return;
5063
5064         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5065
5066         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5067                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5068 }
5069
5070 static void hist_trigger_stacktrace_print(struct seq_file *m,
5071                                           unsigned long *stacktrace_entries,
5072                                           unsigned int max_entries)
5073 {
5074         unsigned int spaces = 8;
5075         unsigned int i;
5076
5077         for (i = 0; i < max_entries; i++) {
5078                 if (!stacktrace_entries[i])
5079                         return;
5080
5081                 seq_printf(m, "%*c", 1 + spaces, ' ');
5082                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5083         }
5084 }
5085
5086 static void hist_trigger_print_key(struct seq_file *m,
5087                                    struct hist_trigger_data *hist_data,
5088                                    void *key,
5089                                    struct tracing_map_elt *elt)
5090 {
5091         struct hist_field *key_field;
5092         bool multiline = false;
5093         const char *field_name;
5094         unsigned int i;
5095         u64 uval;
5096
5097         seq_puts(m, "{ ");
5098
5099         for_each_hist_key_field(i, hist_data) {
5100                 key_field = hist_data->fields[i];
5101
5102                 if (i > hist_data->n_vals)
5103                         seq_puts(m, ", ");
5104
5105                 field_name = hist_field_name(key_field, 0);
5106
5107                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5108                         uval = *(u64 *)(key + key_field->offset);
5109                         seq_printf(m, "%s: %llx", field_name, uval);
5110                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5111                         uval = *(u64 *)(key + key_field->offset);
5112                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5113                                    uval, (void *)(uintptr_t)uval);
5114                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5115                         uval = *(u64 *)(key + key_field->offset);
5116                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5117                                    uval, (void *)(uintptr_t)uval);
5118                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5119                         struct hist_elt_data *elt_data = elt->private_data;
5120                         char *comm;
5121
5122                         if (WARN_ON_ONCE(!elt_data))
5123                                 return;
5124
5125                         comm = elt_data->comm;
5126
5127                         uval = *(u64 *)(key + key_field->offset);
5128                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5129                                    comm, uval);
5130                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5131                         const char *syscall_name;
5132
5133                         uval = *(u64 *)(key + key_field->offset);
5134                         syscall_name = get_syscall_name(uval);
5135                         if (!syscall_name)
5136                                 syscall_name = "unknown_syscall";
5137
5138                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5139                                    syscall_name, uval);
5140                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5141                         seq_puts(m, "stacktrace:\n");
5142                         hist_trigger_stacktrace_print(m,
5143                                                       key + key_field->offset,
5144                                                       HIST_STACKTRACE_DEPTH);
5145                         multiline = true;
5146                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5147                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5148                                    *(u64 *)(key + key_field->offset));
5149                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5150                         unsigned long buckets = key_field->buckets;
5151                         uval = *(u64 *)(key + key_field->offset);
5152                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5153                                    uval, uval + buckets -1);
5154                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5155                         seq_printf(m, "%s: %-50s", field_name,
5156                                    (char *)(key + key_field->offset));
5157                 } else {
5158                         uval = *(u64 *)(key + key_field->offset);
5159                         seq_printf(m, "%s: %10llu", field_name, uval);
5160                 }
5161         }
5162
5163         if (!multiline)
5164                 seq_puts(m, " ");
5165
5166         seq_puts(m, "}");
5167 }
5168
5169 static void hist_trigger_entry_print(struct seq_file *m,
5170                                      struct hist_trigger_data *hist_data,
5171                                      void *key,
5172                                      struct tracing_map_elt *elt)
5173 {
5174         const char *field_name;
5175         unsigned int i;
5176
5177         hist_trigger_print_key(m, hist_data, key, elt);
5178
5179         seq_printf(m, " hitcount: %10llu",
5180                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5181
5182         for (i = 1; i < hist_data->n_vals; i++) {
5183                 field_name = hist_field_name(hist_data->fields[i], 0);
5184
5185                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5186                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5187                         continue;
5188
5189                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5190                         seq_printf(m, "  %s: %10llx", field_name,
5191                                    tracing_map_read_sum(elt, i));
5192                 } else {
5193                         seq_printf(m, "  %s: %10llu", field_name,
5194                                    tracing_map_read_sum(elt, i));
5195                 }
5196         }
5197
5198         print_actions(m, hist_data, elt);
5199
5200         seq_puts(m, "\n");
5201 }
5202
5203 static int print_entries(struct seq_file *m,
5204                          struct hist_trigger_data *hist_data)
5205 {
5206         struct tracing_map_sort_entry **sort_entries = NULL;
5207         struct tracing_map *map = hist_data->map;
5208         int i, n_entries;
5209
5210         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5211                                              hist_data->n_sort_keys,
5212                                              &sort_entries);
5213         if (n_entries < 0)
5214                 return n_entries;
5215
5216         for (i = 0; i < n_entries; i++)
5217                 hist_trigger_entry_print(m, hist_data,
5218                                          sort_entries[i]->key,
5219                                          sort_entries[i]->elt);
5220
5221         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5222
5223         return n_entries;
5224 }
5225
5226 static void hist_trigger_show(struct seq_file *m,
5227                               struct event_trigger_data *data, int n)
5228 {
5229         struct hist_trigger_data *hist_data;
5230         int n_entries;
5231
5232         if (n > 0)
5233                 seq_puts(m, "\n\n");
5234
5235         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5236         data->ops->print(m, data->ops, data);
5237         seq_puts(m, "#\n\n");
5238
5239         hist_data = data->private_data;
5240         n_entries = print_entries(m, hist_data);
5241         if (n_entries < 0)
5242                 n_entries = 0;
5243
5244         track_data_snapshot_print(m, hist_data);
5245
5246         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5247                    (u64)atomic64_read(&hist_data->map->hits),
5248                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5249 }
5250
5251 static int hist_show(struct seq_file *m, void *v)
5252 {
5253         struct event_trigger_data *data;
5254         struct trace_event_file *event_file;
5255         int n = 0, ret = 0;
5256
5257         mutex_lock(&event_mutex);
5258
5259         event_file = event_file_data(m->private);
5260         if (unlikely(!event_file)) {
5261                 ret = -ENODEV;
5262                 goto out_unlock;
5263         }
5264
5265         list_for_each_entry(data, &event_file->triggers, list) {
5266                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5267                         hist_trigger_show(m, data, n++);
5268         }
5269
5270  out_unlock:
5271         mutex_unlock(&event_mutex);
5272
5273         return ret;
5274 }
5275
5276 static int event_hist_open(struct inode *inode, struct file *file)
5277 {
5278         int ret;
5279
5280         ret = security_locked_down(LOCKDOWN_TRACEFS);
5281         if (ret)
5282                 return ret;
5283
5284         return single_open(file, hist_show, file);
5285 }
5286
5287 const struct file_operations event_hist_fops = {
5288         .open = event_hist_open,
5289         .read = seq_read,
5290         .llseek = seq_lseek,
5291         .release = single_release,
5292 };
5293
5294 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5295 static void hist_field_debug_show_flags(struct seq_file *m,
5296                                         unsigned long flags)
5297 {
5298         seq_puts(m, "      flags:\n");
5299
5300         if (flags & HIST_FIELD_FL_KEY)
5301                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5302         else if (flags & HIST_FIELD_FL_HITCOUNT)
5303                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5304         else if (flags & HIST_FIELD_FL_VAR)
5305                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5306         else if (flags & HIST_FIELD_FL_VAR_REF)
5307                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5308         else
5309                 seq_puts(m, "        VAL: normal u64 value\n");
5310
5311         if (flags & HIST_FIELD_FL_ALIAS)
5312                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5313         else if (flags & HIST_FIELD_FL_CONST)
5314                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5315 }
5316
5317 static int hist_field_debug_show(struct seq_file *m,
5318                                  struct hist_field *field, unsigned long flags)
5319 {
5320         if ((field->flags & flags) != flags) {
5321                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5322                 return -EINVAL;
5323         }
5324
5325         hist_field_debug_show_flags(m, field->flags);
5326         if (field->field)
5327                 seq_printf(m, "      ftrace_event_field name: %s\n",
5328                            field->field->name);
5329
5330         if (field->flags & HIST_FIELD_FL_VAR) {
5331                 seq_printf(m, "      var.name: %s\n", field->var.name);
5332                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5333                            field->var.idx);
5334         }
5335
5336         if (field->flags & HIST_FIELD_FL_CONST)
5337                 seq_printf(m, "      constant: %llu\n", field->constant);
5338
5339         if (field->flags & HIST_FIELD_FL_ALIAS)
5340                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5341                            field->var_ref_idx);
5342
5343         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5344                 seq_printf(m, "      name: %s\n", field->name);
5345                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5346                            field->var.idx);
5347                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5348                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5349                            field->var_ref_idx);
5350                 if (field->system)
5351                         seq_printf(m, "      system: %s\n", field->system);
5352                 if (field->event_name)
5353                         seq_printf(m, "      event_name: %s\n", field->event_name);
5354         }
5355
5356         seq_printf(m, "      type: %s\n", field->type);
5357         seq_printf(m, "      size: %u\n", field->size);
5358         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5359
5360         return 0;
5361 }
5362
5363 static int field_var_debug_show(struct seq_file *m,
5364                                 struct field_var *field_var, unsigned int i,
5365                                 bool save_vars)
5366 {
5367         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5368         struct hist_field *field;
5369         int ret = 0;
5370
5371         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5372
5373         field = field_var->var;
5374
5375         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5376
5377         hist_field_debug_show_flags(m, field->flags);
5378         seq_printf(m, "      var.name: %s\n", field->var.name);
5379         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5380                    field->var.idx);
5381
5382         field = field_var->val;
5383
5384         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5385         if (field->field)
5386                 seq_printf(m, "      ftrace_event_field name: %s\n",
5387                            field->field->name);
5388         else {
5389                 ret = -EINVAL;
5390                 goto out;
5391         }
5392
5393         seq_printf(m, "      type: %s\n", field->type);
5394         seq_printf(m, "      size: %u\n", field->size);
5395         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5396 out:
5397         return ret;
5398 }
5399
5400 static int hist_action_debug_show(struct seq_file *m,
5401                                   struct action_data *data, int i)
5402 {
5403         int ret = 0;
5404
5405         if (data->handler == HANDLER_ONMAX ||
5406             data->handler == HANDLER_ONCHANGE) {
5407                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5408                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5409                                             HIST_FIELD_FL_VAR_REF);
5410                 if (ret)
5411                         goto out;
5412
5413                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5414                 ret = hist_field_debug_show(m, data->track_data.track_var,
5415                                             HIST_FIELD_FL_VAR);
5416                 if (ret)
5417                         goto out;
5418         }
5419
5420         if (data->handler == HANDLER_ONMATCH) {
5421                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5422                            i, data->match_data.event_system);
5423                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5424                            i, data->match_data.event);
5425         }
5426 out:
5427         return ret;
5428 }
5429
5430 static int hist_actions_debug_show(struct seq_file *m,
5431                                    struct hist_trigger_data *hist_data)
5432 {
5433         int i, ret = 0;
5434
5435         if (hist_data->n_actions)
5436                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5437
5438         for (i = 0; i < hist_data->n_actions; i++) {
5439                 struct action_data *action = hist_data->actions[i];
5440
5441                 ret = hist_action_debug_show(m, action, i);
5442                 if (ret)
5443                         goto out;
5444         }
5445
5446         if (hist_data->n_save_vars)
5447                 seq_puts(m, "\n  save action variables (save() params):\n");
5448
5449         for (i = 0; i < hist_data->n_save_vars; i++) {
5450                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5451                 if (ret)
5452                         goto out;
5453         }
5454 out:
5455         return ret;
5456 }
5457
5458 static void hist_trigger_debug_show(struct seq_file *m,
5459                                     struct event_trigger_data *data, int n)
5460 {
5461         struct hist_trigger_data *hist_data;
5462         int i, ret;
5463
5464         if (n > 0)
5465                 seq_puts(m, "\n\n");
5466
5467         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5468         data->ops->print(m, data->ops, data);
5469         seq_puts(m, "#\n\n");
5470
5471         hist_data = data->private_data;
5472
5473         seq_printf(m, "hist_data: %p\n\n", hist_data);
5474         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5475         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5476         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5477
5478         seq_puts(m, "\n  val fields:\n\n");
5479
5480         seq_puts(m, "    hist_data->fields[0]:\n");
5481         ret = hist_field_debug_show(m, hist_data->fields[0],
5482                                     HIST_FIELD_FL_HITCOUNT);
5483         if (ret)
5484                 return;
5485
5486         for (i = 1; i < hist_data->n_vals; i++) {
5487                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5488                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5489                 if (ret)
5490                         return;
5491         }
5492
5493         seq_puts(m, "\n  key fields:\n");
5494
5495         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5496                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5497                 ret = hist_field_debug_show(m, hist_data->fields[i],
5498                                             HIST_FIELD_FL_KEY);
5499                 if (ret)
5500                         return;
5501         }
5502
5503         if (hist_data->n_var_refs)
5504                 seq_puts(m, "\n  variable reference fields:\n");
5505
5506         for (i = 0; i < hist_data->n_var_refs; i++) {
5507                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5508                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5509                                             HIST_FIELD_FL_VAR_REF);
5510                 if (ret)
5511                         return;
5512         }
5513
5514         if (hist_data->n_field_vars)
5515                 seq_puts(m, "\n  field variables:\n");
5516
5517         for (i = 0; i < hist_data->n_field_vars; i++) {
5518                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5519                 if (ret)
5520                         return;
5521         }
5522
5523         ret = hist_actions_debug_show(m, hist_data);
5524         if (ret)
5525                 return;
5526 }
5527
5528 static int hist_debug_show(struct seq_file *m, void *v)
5529 {
5530         struct event_trigger_data *data;
5531         struct trace_event_file *event_file;
5532         int n = 0, ret = 0;
5533
5534         mutex_lock(&event_mutex);
5535
5536         event_file = event_file_data(m->private);
5537         if (unlikely(!event_file)) {
5538                 ret = -ENODEV;
5539                 goto out_unlock;
5540         }
5541
5542         list_for_each_entry(data, &event_file->triggers, list) {
5543                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5544                         hist_trigger_debug_show(m, data, n++);
5545         }
5546
5547  out_unlock:
5548         mutex_unlock(&event_mutex);
5549
5550         return ret;
5551 }
5552
5553 static int event_hist_debug_open(struct inode *inode, struct file *file)
5554 {
5555         int ret;
5556
5557         ret = security_locked_down(LOCKDOWN_TRACEFS);
5558         if (ret)
5559                 return ret;
5560
5561         return single_open(file, hist_debug_show, file);
5562 }
5563
5564 const struct file_operations event_hist_debug_fops = {
5565         .open = event_hist_debug_open,
5566         .read = seq_read,
5567         .llseek = seq_lseek,
5568         .release = single_release,
5569 };
5570 #endif
5571
5572 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5573 {
5574         const char *field_name = hist_field_name(hist_field, 0);
5575
5576         if (hist_field->var.name)
5577                 seq_printf(m, "%s=", hist_field->var.name);
5578
5579         if (hist_field->flags & HIST_FIELD_FL_CPU)
5580                 seq_puts(m, "common_cpu");
5581         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5582                 seq_printf(m, "%llu", hist_field->constant);
5583         else if (field_name) {
5584                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5585                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5586                         seq_putc(m, '$');
5587                 seq_printf(m, "%s", field_name);
5588         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5589                 seq_puts(m, "common_timestamp");
5590
5591         if (hist_field->flags) {
5592                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5593                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5594                         const char *flags = get_hist_field_flags(hist_field);
5595
5596                         if (flags)
5597                                 seq_printf(m, ".%s", flags);
5598                 }
5599         }
5600         if (hist_field->buckets)
5601                 seq_printf(m, "=%ld", hist_field->buckets);
5602 }
5603
5604 static int event_hist_trigger_print(struct seq_file *m,
5605                                     struct event_trigger_ops *ops,
5606                                     struct event_trigger_data *data)
5607 {
5608         struct hist_trigger_data *hist_data = data->private_data;
5609         struct hist_field *field;
5610         bool have_var = false;
5611         unsigned int i;
5612
5613         seq_puts(m, "hist:");
5614
5615         if (data->name)
5616                 seq_printf(m, "%s:", data->name);
5617
5618         seq_puts(m, "keys=");
5619
5620         for_each_hist_key_field(i, hist_data) {
5621                 field = hist_data->fields[i];
5622
5623                 if (i > hist_data->n_vals)
5624                         seq_puts(m, ",");
5625
5626                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5627                         seq_puts(m, "stacktrace");
5628                 else
5629                         hist_field_print(m, field);
5630         }
5631
5632         seq_puts(m, ":vals=");
5633
5634         for_each_hist_val_field(i, hist_data) {
5635                 field = hist_data->fields[i];
5636                 if (field->flags & HIST_FIELD_FL_VAR) {
5637                         have_var = true;
5638                         continue;
5639                 }
5640
5641                 if (i == HITCOUNT_IDX)
5642                         seq_puts(m, "hitcount");
5643                 else {
5644                         seq_puts(m, ",");
5645                         hist_field_print(m, field);
5646                 }
5647         }
5648
5649         if (have_var) {
5650                 unsigned int n = 0;
5651
5652                 seq_puts(m, ":");
5653
5654                 for_each_hist_val_field(i, hist_data) {
5655                         field = hist_data->fields[i];
5656
5657                         if (field->flags & HIST_FIELD_FL_VAR) {
5658                                 if (n++)
5659                                         seq_puts(m, ",");
5660                                 hist_field_print(m, field);
5661                         }
5662                 }
5663         }
5664
5665         seq_puts(m, ":sort=");
5666
5667         for (i = 0; i < hist_data->n_sort_keys; i++) {
5668                 struct tracing_map_sort_key *sort_key;
5669                 unsigned int idx, first_key_idx;
5670
5671                 /* skip VAR vals */
5672                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5673
5674                 sort_key = &hist_data->sort_keys[i];
5675                 idx = sort_key->field_idx;
5676
5677                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5678                         return -EINVAL;
5679
5680                 if (i > 0)
5681                         seq_puts(m, ",");
5682
5683                 if (idx == HITCOUNT_IDX)
5684                         seq_puts(m, "hitcount");
5685                 else {
5686                         if (idx >= first_key_idx)
5687                                 idx += hist_data->n_vars;
5688                         hist_field_print(m, hist_data->fields[idx]);
5689                 }
5690
5691                 if (sort_key->descending)
5692                         seq_puts(m, ".descending");
5693         }
5694         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5695         if (hist_data->enable_timestamps)
5696                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5697
5698         print_actions_spec(m, hist_data);
5699
5700         if (data->filter_str)
5701                 seq_printf(m, " if %s", data->filter_str);
5702
5703         if (data->paused)
5704                 seq_puts(m, " [paused]");
5705         else
5706                 seq_puts(m, " [active]");
5707
5708         seq_putc(m, '\n');
5709
5710         return 0;
5711 }
5712
5713 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5714                                    struct event_trigger_data *data)
5715 {
5716         struct hist_trigger_data *hist_data = data->private_data;
5717
5718         if (!data->ref && hist_data->attrs->name)
5719                 save_named_trigger(hist_data->attrs->name, data);
5720
5721         data->ref++;
5722
5723         return 0;
5724 }
5725
5726 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5727 {
5728         struct trace_event_file *file;
5729         unsigned int i;
5730         char *cmd;
5731         int ret;
5732
5733         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5734                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5735                 cmd = hist_data->field_var_hists[i]->cmd;
5736                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5737                                                "!hist", "hist", cmd);
5738                 WARN_ON_ONCE(ret < 0);
5739         }
5740 }
5741
5742 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5743                                     struct event_trigger_data *data)
5744 {
5745         struct hist_trigger_data *hist_data = data->private_data;
5746
5747         if (WARN_ON_ONCE(data->ref <= 0))
5748                 return;
5749
5750         data->ref--;
5751         if (!data->ref) {
5752                 if (data->name)
5753                         del_named_trigger(data);
5754
5755                 trigger_data_free(data);
5756
5757                 remove_hist_vars(hist_data);
5758
5759                 unregister_field_var_hists(hist_data);
5760
5761                 destroy_hist_data(hist_data);
5762         }
5763 }
5764
5765 static struct event_trigger_ops event_hist_trigger_ops = {
5766         .trigger                = event_hist_trigger,
5767         .print                  = event_hist_trigger_print,
5768         .init                   = event_hist_trigger_init,
5769         .free                   = event_hist_trigger_free,
5770 };
5771
5772 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5773                                          struct event_trigger_data *data)
5774 {
5775         data->ref++;
5776
5777         save_named_trigger(data->named_data->name, data);
5778
5779         event_hist_trigger_init(ops, data->named_data);
5780
5781         return 0;
5782 }
5783
5784 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5785                                           struct event_trigger_data *data)
5786 {
5787         if (WARN_ON_ONCE(data->ref <= 0))
5788                 return;
5789
5790         event_hist_trigger_free(ops, data->named_data);
5791
5792         data->ref--;
5793         if (!data->ref) {
5794                 del_named_trigger(data);
5795                 trigger_data_free(data);
5796         }
5797 }
5798
5799 static struct event_trigger_ops event_hist_trigger_named_ops = {
5800         .trigger                = event_hist_trigger,
5801         .print                  = event_hist_trigger_print,
5802         .init                   = event_hist_trigger_named_init,
5803         .free                   = event_hist_trigger_named_free,
5804 };
5805
5806 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5807                                                             char *param)
5808 {
5809         return &event_hist_trigger_ops;
5810 }
5811
5812 static void hist_clear(struct event_trigger_data *data)
5813 {
5814         struct hist_trigger_data *hist_data = data->private_data;
5815
5816         if (data->name)
5817                 pause_named_trigger(data);
5818
5819         tracepoint_synchronize_unregister();
5820
5821         tracing_map_clear(hist_data->map);
5822
5823         if (data->name)
5824                 unpause_named_trigger(data);
5825 }
5826
5827 static bool compatible_field(struct ftrace_event_field *field,
5828                              struct ftrace_event_field *test_field)
5829 {
5830         if (field == test_field)
5831                 return true;
5832         if (field == NULL || test_field == NULL)
5833                 return false;
5834         if (strcmp(field->name, test_field->name) != 0)
5835                 return false;
5836         if (strcmp(field->type, test_field->type) != 0)
5837                 return false;
5838         if (field->size != test_field->size)
5839                 return false;
5840         if (field->is_signed != test_field->is_signed)
5841                 return false;
5842
5843         return true;
5844 }
5845
5846 static bool hist_trigger_match(struct event_trigger_data *data,
5847                                struct event_trigger_data *data_test,
5848                                struct event_trigger_data *named_data,
5849                                bool ignore_filter)
5850 {
5851         struct tracing_map_sort_key *sort_key, *sort_key_test;
5852         struct hist_trigger_data *hist_data, *hist_data_test;
5853         struct hist_field *key_field, *key_field_test;
5854         unsigned int i;
5855
5856         if (named_data && (named_data != data_test) &&
5857             (named_data != data_test->named_data))
5858                 return false;
5859
5860         if (!named_data && is_named_trigger(data_test))
5861                 return false;
5862
5863         hist_data = data->private_data;
5864         hist_data_test = data_test->private_data;
5865
5866         if (hist_data->n_vals != hist_data_test->n_vals ||
5867             hist_data->n_fields != hist_data_test->n_fields ||
5868             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5869                 return false;
5870
5871         if (!ignore_filter) {
5872                 if ((data->filter_str && !data_test->filter_str) ||
5873                    (!data->filter_str && data_test->filter_str))
5874                         return false;
5875         }
5876
5877         for_each_hist_field(i, hist_data) {
5878                 key_field = hist_data->fields[i];
5879                 key_field_test = hist_data_test->fields[i];
5880
5881                 if (key_field->flags != key_field_test->flags)
5882                         return false;
5883                 if (!compatible_field(key_field->field, key_field_test->field))
5884                         return false;
5885                 if (key_field->offset != key_field_test->offset)
5886                         return false;
5887                 if (key_field->size != key_field_test->size)
5888                         return false;
5889                 if (key_field->is_signed != key_field_test->is_signed)
5890                         return false;
5891                 if (!!key_field->var.name != !!key_field_test->var.name)
5892                         return false;
5893                 if (key_field->var.name &&
5894                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5895                         return false;
5896         }
5897
5898         for (i = 0; i < hist_data->n_sort_keys; i++) {
5899                 sort_key = &hist_data->sort_keys[i];
5900                 sort_key_test = &hist_data_test->sort_keys[i];
5901
5902                 if (sort_key->field_idx != sort_key_test->field_idx ||
5903                     sort_key->descending != sort_key_test->descending)
5904                         return false;
5905         }
5906
5907         if (!ignore_filter && data->filter_str &&
5908             (strcmp(data->filter_str, data_test->filter_str) != 0))
5909                 return false;
5910
5911         if (!actions_match(hist_data, hist_data_test))
5912                 return false;
5913
5914         return true;
5915 }
5916
5917 static int hist_register_trigger(char *glob,
5918                                  struct event_trigger_data *data,
5919                                  struct trace_event_file *file)
5920 {
5921         struct hist_trigger_data *hist_data = data->private_data;
5922         struct event_trigger_data *test, *named_data = NULL;
5923         struct trace_array *tr = file->tr;
5924         int ret = 0;
5925
5926         if (hist_data->attrs->name) {
5927                 named_data = find_named_trigger(hist_data->attrs->name);
5928                 if (named_data) {
5929                         if (!hist_trigger_match(data, named_data, named_data,
5930                                                 true)) {
5931                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5932                                 ret = -EINVAL;
5933                                 goto out;
5934                         }
5935                 }
5936         }
5937
5938         if (hist_data->attrs->name && !named_data)
5939                 goto new;
5940
5941         lockdep_assert_held(&event_mutex);
5942
5943         list_for_each_entry(test, &file->triggers, list) {
5944                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5945                         if (!hist_trigger_match(data, test, named_data, false))
5946                                 continue;
5947                         if (hist_data->attrs->pause)
5948                                 test->paused = true;
5949                         else if (hist_data->attrs->cont)
5950                                 test->paused = false;
5951                         else if (hist_data->attrs->clear)
5952                                 hist_clear(test);
5953                         else {
5954                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5955                                 ret = -EEXIST;
5956                         }
5957                         goto out;
5958                 }
5959         }
5960  new:
5961         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5962                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5963                 ret = -ENOENT;
5964                 goto out;
5965         }
5966
5967         if (hist_data->attrs->pause)
5968                 data->paused = true;
5969
5970         if (named_data) {
5971                 data->private_data = named_data->private_data;
5972                 set_named_trigger_data(data, named_data);
5973                 data->ops = &event_hist_trigger_named_ops;
5974         }
5975
5976         if (data->ops->init) {
5977                 ret = data->ops->init(data->ops, data);
5978                 if (ret < 0)
5979                         goto out;
5980         }
5981
5982         if (hist_data->enable_timestamps) {
5983                 char *clock = hist_data->attrs->clock;
5984
5985                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5986                 if (ret) {
5987                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5988                         goto out;
5989                 }
5990
5991                 tracing_set_filter_buffering(file->tr, true);
5992         }
5993
5994         if (named_data)
5995                 destroy_hist_data(hist_data);
5996
5997         ret++;
5998  out:
5999         return ret;
6000 }
6001
6002 static int hist_trigger_enable(struct event_trigger_data *data,
6003                                struct trace_event_file *file)
6004 {
6005         int ret = 0;
6006
6007         list_add_tail_rcu(&data->list, &file->triggers);
6008
6009         update_cond_flag(file);
6010
6011         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6012                 list_del_rcu(&data->list);
6013                 update_cond_flag(file);
6014                 ret--;
6015         }
6016
6017         return ret;
6018 }
6019
6020 static bool have_hist_trigger_match(struct event_trigger_data *data,
6021                                     struct trace_event_file *file)
6022 {
6023         struct hist_trigger_data *hist_data = data->private_data;
6024         struct event_trigger_data *test, *named_data = NULL;
6025         bool match = false;
6026
6027         lockdep_assert_held(&event_mutex);
6028
6029         if (hist_data->attrs->name)
6030                 named_data = find_named_trigger(hist_data->attrs->name);
6031
6032         list_for_each_entry(test, &file->triggers, list) {
6033                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6034                         if (hist_trigger_match(data, test, named_data, false)) {
6035                                 match = true;
6036                                 break;
6037                         }
6038                 }
6039         }
6040
6041         return match;
6042 }
6043
6044 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6045                                     struct trace_event_file *file)
6046 {
6047         struct hist_trigger_data *hist_data = data->private_data;
6048         struct event_trigger_data *test, *named_data = NULL;
6049
6050         lockdep_assert_held(&event_mutex);
6051
6052         if (hist_data->attrs->name)
6053                 named_data = find_named_trigger(hist_data->attrs->name);
6054
6055         list_for_each_entry(test, &file->triggers, list) {
6056                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6057                         if (!hist_trigger_match(data, test, named_data, false))
6058                                 continue;
6059                         hist_data = test->private_data;
6060                         if (check_var_refs(hist_data))
6061                                 return true;
6062                         break;
6063                 }
6064         }
6065
6066         return false;
6067 }
6068
6069 static void hist_unregister_trigger(char *glob,
6070                                     struct event_trigger_data *data,
6071                                     struct trace_event_file *file)
6072 {
6073         struct hist_trigger_data *hist_data = data->private_data;
6074         struct event_trigger_data *test, *named_data = NULL;
6075         bool unregistered = false;
6076
6077         lockdep_assert_held(&event_mutex);
6078
6079         if (hist_data->attrs->name)
6080                 named_data = find_named_trigger(hist_data->attrs->name);
6081
6082         list_for_each_entry(test, &file->triggers, list) {
6083                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6084                         if (!hist_trigger_match(data, test, named_data, false))
6085                                 continue;
6086                         unregistered = true;
6087                         list_del_rcu(&test->list);
6088                         trace_event_trigger_enable_disable(file, 0);
6089                         update_cond_flag(file);
6090                         break;
6091                 }
6092         }
6093
6094         if (unregistered && test->ops->free)
6095                 test->ops->free(test->ops, test);
6096
6097         if (hist_data->enable_timestamps) {
6098                 if (!hist_data->remove || unregistered)
6099                         tracing_set_filter_buffering(file->tr, false);
6100         }
6101 }
6102
6103 static bool hist_file_check_refs(struct trace_event_file *file)
6104 {
6105         struct hist_trigger_data *hist_data;
6106         struct event_trigger_data *test;
6107
6108         lockdep_assert_held(&event_mutex);
6109
6110         list_for_each_entry(test, &file->triggers, list) {
6111                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6112                         hist_data = test->private_data;
6113                         if (check_var_refs(hist_data))
6114                                 return true;
6115                 }
6116         }
6117
6118         return false;
6119 }
6120
6121 static void hist_unreg_all(struct trace_event_file *file)
6122 {
6123         struct event_trigger_data *test, *n;
6124         struct hist_trigger_data *hist_data;
6125         struct synth_event *se;
6126         const char *se_name;
6127
6128         lockdep_assert_held(&event_mutex);
6129
6130         if (hist_file_check_refs(file))
6131                 return;
6132
6133         list_for_each_entry_safe(test, n, &file->triggers, list) {
6134                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6135                         hist_data = test->private_data;
6136                         list_del_rcu(&test->list);
6137                         trace_event_trigger_enable_disable(file, 0);
6138
6139                         se_name = trace_event_name(file->event_call);
6140                         se = find_synth_event(se_name);
6141                         if (se)
6142                                 se->ref--;
6143
6144                         update_cond_flag(file);
6145                         if (hist_data->enable_timestamps)
6146                                 tracing_set_filter_buffering(file->tr, false);
6147                         if (test->ops->free)
6148                                 test->ops->free(test->ops, test);
6149                 }
6150         }
6151 }
6152
6153 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6154                                     struct trace_event_file *file,
6155                                     char *glob, char *cmd, char *param)
6156 {
6157         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6158         struct event_trigger_data *trigger_data;
6159         struct hist_trigger_attrs *attrs;
6160         struct event_trigger_ops *trigger_ops;
6161         struct hist_trigger_data *hist_data;
6162         struct synth_event *se;
6163         const char *se_name;
6164         bool remove = false;
6165         char *trigger, *p, *start;
6166         int ret = 0;
6167
6168         lockdep_assert_held(&event_mutex);
6169
6170         WARN_ON(!glob);
6171
6172         if (strlen(glob)) {
6173                 hist_err_clear();
6174                 last_cmd_set(file, param);
6175         }
6176
6177         if (!param)
6178                 return -EINVAL;
6179
6180         if (glob[0] == '!')
6181                 remove = true;
6182
6183         /*
6184          * separate the trigger from the filter (k:v [if filter])
6185          * allowing for whitespace in the trigger
6186          */
6187         p = trigger = param;
6188         do {
6189                 p = strstr(p, "if");
6190                 if (!p)
6191                         break;
6192                 if (p == param)
6193                         return -EINVAL;
6194                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6195                         p++;
6196                         continue;
6197                 }
6198                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6199                         return -EINVAL;
6200                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6201                         p++;
6202                         continue;
6203                 }
6204                 break;
6205         } while (1);
6206
6207         if (!p)
6208                 param = NULL;
6209         else {
6210                 *(p - 1) = '\0';
6211                 param = strstrip(p);
6212                 trigger = strstrip(trigger);
6213         }
6214
6215         /*
6216          * To simplify arithmetic expression parsing, replace occurrences of
6217          * '.sym-offset' modifier with '.symXoffset'
6218          */
6219         start = strstr(trigger, ".sym-offset");
6220         while (start) {
6221                 *(start + 4) = 'X';
6222                 start = strstr(start + 11, ".sym-offset");
6223         }
6224
6225         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6226         if (IS_ERR(attrs))
6227                 return PTR_ERR(attrs);
6228
6229         if (attrs->map_bits)
6230                 hist_trigger_bits = attrs->map_bits;
6231
6232         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6233         if (IS_ERR(hist_data)) {
6234                 destroy_hist_trigger_attrs(attrs);
6235                 return PTR_ERR(hist_data);
6236         }
6237
6238         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6239
6240         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6241         if (!trigger_data) {
6242                 ret = -ENOMEM;
6243                 goto out_free;
6244         }
6245
6246         trigger_data->count = -1;
6247         trigger_data->ops = trigger_ops;
6248         trigger_data->cmd_ops = cmd_ops;
6249
6250         INIT_LIST_HEAD(&trigger_data->list);
6251         RCU_INIT_POINTER(trigger_data->filter, NULL);
6252
6253         trigger_data->private_data = hist_data;
6254
6255         /* if param is non-empty, it's supposed to be a filter */
6256         if (param && cmd_ops->set_filter) {
6257                 ret = cmd_ops->set_filter(param, trigger_data, file);
6258                 if (ret < 0)
6259                         goto out_free;
6260         }
6261
6262         if (remove) {
6263                 if (!have_hist_trigger_match(trigger_data, file))
6264                         goto out_free;
6265
6266                 if (hist_trigger_check_refs(trigger_data, file)) {
6267                         ret = -EBUSY;
6268                         goto out_free;
6269                 }
6270
6271                 cmd_ops->unreg(glob+1, trigger_data, file);
6272                 se_name = trace_event_name(file->event_call);
6273                 se = find_synth_event(se_name);
6274                 if (se)
6275                         se->ref--;
6276                 ret = 0;
6277                 goto out_free;
6278         }
6279
6280         ret = cmd_ops->reg(glob, trigger_data, file);
6281         /*
6282          * The above returns on success the # of triggers registered,
6283          * but if it didn't register any it returns zero.  Consider no
6284          * triggers registered a failure too.
6285          */
6286         if (!ret) {
6287                 if (!(attrs->pause || attrs->cont || attrs->clear))
6288                         ret = -ENOENT;
6289                 goto out_free;
6290         } else if (ret < 0)
6291                 goto out_free;
6292
6293         if (get_named_trigger_data(trigger_data))
6294                 goto enable;
6295
6296         if (has_hist_vars(hist_data))
6297                 save_hist_vars(hist_data);
6298
6299         ret = create_actions(hist_data);
6300         if (ret)
6301                 goto out_unreg;
6302
6303         ret = tracing_map_init(hist_data->map);
6304         if (ret)
6305                 goto out_unreg;
6306 enable:
6307         ret = hist_trigger_enable(trigger_data, file);
6308         if (ret)
6309                 goto out_unreg;
6310
6311         se_name = trace_event_name(file->event_call);
6312         se = find_synth_event(se_name);
6313         if (se)
6314                 se->ref++;
6315         /* Just return zero, not the number of registered triggers */
6316         ret = 0;
6317  out:
6318         if (ret == 0)
6319                 hist_err_clear();
6320
6321         return ret;
6322  out_unreg:
6323         cmd_ops->unreg(glob+1, trigger_data, file);
6324  out_free:
6325         if (cmd_ops->set_filter)
6326                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6327
6328         remove_hist_vars(hist_data);
6329
6330         kfree(trigger_data);
6331
6332         destroy_hist_data(hist_data);
6333         goto out;
6334 }
6335
6336 static struct event_command trigger_hist_cmd = {
6337         .name                   = "hist",
6338         .trigger_type           = ETT_EVENT_HIST,
6339         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6340         .parse                  = event_hist_trigger_parse,
6341         .reg                    = hist_register_trigger,
6342         .unreg                  = hist_unregister_trigger,
6343         .unreg_all              = hist_unreg_all,
6344         .get_trigger_ops        = event_hist_get_trigger_ops,
6345         .set_filter             = set_trigger_filter,
6346 };
6347
6348 __init int register_trigger_hist_cmd(void)
6349 {
6350         int ret;
6351
6352         ret = register_event_command(&trigger_hist_cmd);
6353         WARN_ON(ret < 0);
6354
6355         return ret;
6356 }
6357
6358 static void
6359 hist_enable_trigger(struct event_trigger_data *data,
6360                     struct trace_buffer *buffer,  void *rec,
6361                     struct ring_buffer_event *event)
6362 {
6363         struct enable_trigger_data *enable_data = data->private_data;
6364         struct event_trigger_data *test;
6365
6366         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6367                                 lockdep_is_held(&event_mutex)) {
6368                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6369                         if (enable_data->enable)
6370                                 test->paused = false;
6371                         else
6372                                 test->paused = true;
6373                 }
6374         }
6375 }
6376
6377 static void
6378 hist_enable_count_trigger(struct event_trigger_data *data,
6379                           struct trace_buffer *buffer,  void *rec,
6380                           struct ring_buffer_event *event)
6381 {
6382         if (!data->count)
6383                 return;
6384
6385         if (data->count != -1)
6386                 (data->count)--;
6387
6388         hist_enable_trigger(data, buffer, rec, event);
6389 }
6390
6391 static struct event_trigger_ops hist_enable_trigger_ops = {
6392         .trigger                = hist_enable_trigger,
6393         .print                  = event_enable_trigger_print,
6394         .init                   = event_trigger_init,
6395         .free                   = event_enable_trigger_free,
6396 };
6397
6398 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6399         .trigger                = hist_enable_count_trigger,
6400         .print                  = event_enable_trigger_print,
6401         .init                   = event_trigger_init,
6402         .free                   = event_enable_trigger_free,
6403 };
6404
6405 static struct event_trigger_ops hist_disable_trigger_ops = {
6406         .trigger                = hist_enable_trigger,
6407         .print                  = event_enable_trigger_print,
6408         .init                   = event_trigger_init,
6409         .free                   = event_enable_trigger_free,
6410 };
6411
6412 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6413         .trigger                = hist_enable_count_trigger,
6414         .print                  = event_enable_trigger_print,
6415         .init                   = event_trigger_init,
6416         .free                   = event_enable_trigger_free,
6417 };
6418
6419 static struct event_trigger_ops *
6420 hist_enable_get_trigger_ops(char *cmd, char *param)
6421 {
6422         struct event_trigger_ops *ops;
6423         bool enable;
6424
6425         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6426
6427         if (enable)
6428                 ops = param ? &hist_enable_count_trigger_ops :
6429                         &hist_enable_trigger_ops;
6430         else
6431                 ops = param ? &hist_disable_count_trigger_ops :
6432                         &hist_disable_trigger_ops;
6433
6434         return ops;
6435 }
6436
6437 static void hist_enable_unreg_all(struct trace_event_file *file)
6438 {
6439         struct event_trigger_data *test, *n;
6440
6441         list_for_each_entry_safe(test, n, &file->triggers, list) {
6442                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6443                         list_del_rcu(&test->list);
6444                         update_cond_flag(file);
6445                         trace_event_trigger_enable_disable(file, 0);
6446                         if (test->ops->free)
6447                                 test->ops->free(test->ops, test);
6448                 }
6449         }
6450 }
6451
6452 static struct event_command trigger_hist_enable_cmd = {
6453         .name                   = ENABLE_HIST_STR,
6454         .trigger_type           = ETT_HIST_ENABLE,
6455         .parse                  = event_enable_trigger_parse,
6456         .reg                    = event_enable_register_trigger,
6457         .unreg                  = event_enable_unregister_trigger,
6458         .unreg_all              = hist_enable_unreg_all,
6459         .get_trigger_ops        = hist_enable_get_trigger_ops,
6460         .set_filter             = set_trigger_filter,
6461 };
6462
6463 static struct event_command trigger_hist_disable_cmd = {
6464         .name                   = DISABLE_HIST_STR,
6465         .trigger_type           = ETT_HIST_ENABLE,
6466         .parse                  = event_enable_trigger_parse,
6467         .reg                    = event_enable_register_trigger,
6468         .unreg                  = event_enable_unregister_trigger,
6469         .unreg_all              = hist_enable_unreg_all,
6470         .get_trigger_ops        = hist_enable_get_trigger_ops,
6471         .set_filter             = set_trigger_filter,
6472 };
6473
6474 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6475 {
6476         unregister_event_command(&trigger_hist_enable_cmd);
6477         unregister_event_command(&trigger_hist_disable_cmd);
6478 }
6479
6480 __init int register_trigger_hist_enable_disable_cmds(void)
6481 {
6482         int ret;
6483
6484         ret = register_event_command(&trigger_hist_enable_cmd);
6485         if (WARN_ON(ret < 0))
6486                 return ret;
6487         ret = register_event_command(&trigger_hist_disable_cmd);
6488         if (WARN_ON(ret < 0))
6489                 unregister_trigger_hist_enable_disable_cmds();
6490
6491         return ret;
6492 }