kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),
  73
  74 #undef C
  75 #define C(a, b)         HIST_ERR_##a
  76
  77 enum { ERRORS };
  78
  79 #undef C
  80 #define C(a, b)         b
  81
  82 static const char *err_text[] = { ERRORS };
  83
  84 struct hist_field;
  85
  86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  87                                 struct tracing_map_elt *elt,
  88                                 struct trace_buffer *buffer,
  89                                 struct ring_buffer_event *rbe,
  90                                 void *event);
  91
  92 #define HIST_FIELD_OPERANDS_MAX 2
  93 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  94 #define HIST_ACTIONS_MAX        8
  95 #define HIST_CONST_DIGITS_MAX   21
  96 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  97
  98 enum field_op_id {
  99         FIELD_OP_NONE,
 100         FIELD_OP_PLUS,
 101         FIELD_OP_MINUS,
 102         FIELD_OP_UNARY_MINUS,
 103         FIELD_OP_DIV,
 104         FIELD_OP_MULT,
 105 };
 106
 107 /*
 108  * A hist_var (histogram variable) contains variable information for
 109  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 110  * flag set.  A hist_var has a variable name e.g. ts0, and is
 111  * associated with a given histogram trigger, as specified by
 112  * hist_data.  The hist_var idx is the unique index assigned to the
 113  * variable by the hist trigger's tracing_map.  The idx is what is
 114  * used to set a variable's value and, by a variable reference, to
 115  * retrieve it.
 116  */
 117 struct hist_var {
 118         char                            *name;
 119         struct hist_trigger_data        *hist_data;
 120         unsigned int                    idx;
 121 };
 122
 123 struct hist_field {
 124         struct ftrace_event_field       *field;
 125         unsigned long                   flags;
 126         hist_field_fn_t                 fn;
 127         unsigned int                    ref;
 128         unsigned int                    size;
 129         unsigned int                    offset;
 130         unsigned int                    is_signed;
 131         unsigned long                   buckets;
 132         const char                      *type;
 133         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 134         struct hist_trigger_data        *hist_data;
 135
 136         /*
 137          * Variable fields contain variable-specific info in var.
 138          */
 139         struct hist_var                 var;
 140         enum field_op_id                operator;
 141         char                            *system;
 142         char                            *event_name;
 143
 144         /*
 145          * The name field is used for EXPR and VAR_REF fields.  VAR
 146          * fields contain the variable name in var.name.
 147          */
 148         char                            *name;
 149
 150         /*
 151          * When a histogram trigger is hit, if it has any references
 152          * to variables, the values of those variables are collected
 153          * into a var_ref_vals array by resolve_var_refs().  The
 154          * current value of each variable is read from the tracing_map
 155          * using the hist field's hist_var.idx and entered into the
 156          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 157          */
 158         unsigned int                    var_ref_idx;
 159         bool                            read_once;
 160
 161         unsigned int                    var_str_idx;
 162
 163         /* Numeric literals are represented as u64 */
 164         u64                             constant;
 165         /* Used to optimize division by constants */
 166         u64                             div_multiplier;
 167 };
 168
 169 static u64 hist_field_none(struct hist_field *field,
 170                            struct tracing_map_elt *elt,
 171                            struct trace_buffer *buffer,
 172                            struct ring_buffer_event *rbe,
 173                            void *event)
 174 {
 175         return 0;
 176 }
 177
 178 static u64 hist_field_const(struct hist_field *field,
 179                            struct tracing_map_elt *elt,
 180                            struct trace_buffer *buffer,
 181                            struct ring_buffer_event *rbe,
 182                            void *event)
 183 {
 184         return field->constant;
 185 }
 186
 187 static u64 hist_field_counter(struct hist_field *field,
 188                               struct tracing_map_elt *elt,
 189                               struct trace_buffer *buffer,
 190                               struct ring_buffer_event *rbe,
 191                               void *event)
 192 {
 193         return 1;
 194 }
 195
 196 static u64 hist_field_string(struct hist_field *hist_field,
 197                              struct tracing_map_elt *elt,
 198                              struct trace_buffer *buffer,
 199                              struct ring_buffer_event *rbe,
 200                              void *event)
 201 {
 202         char *addr = (char *)(event + hist_field->field->offset);
 203
 204         return (u64)(unsigned long)addr;
 205 }
 206
 207 static u64 hist_field_dynstring(struct hist_field *hist_field,
 208                                 struct tracing_map_elt *elt,
 209                                 struct trace_buffer *buffer,
 210                                 struct ring_buffer_event *rbe,
 211                                 void *event)
 212 {
 213         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 214         int str_loc = str_item & 0xffff;
 215         char *addr = (char *)(event + str_loc);
 216
 217         return (u64)(unsigned long)addr;
 218 }
 219
 220 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 221                                    struct tracing_map_elt *elt,
 222                                    struct trace_buffer *buffer,
 223                                    struct ring_buffer_event *rbe,
 224                                    void *event)
 225 {
 226         u32 *item = event + hist_field->field->offset;
 227         u32 str_item = *item;
 228         int str_loc = str_item & 0xffff;
 229         char *addr = (char *)&item[1] + str_loc;
 230
 231         return (u64)(unsigned long)addr;
 232 }
 233
 234 static u64 hist_field_pstring(struct hist_field *hist_field,
 235                               struct tracing_map_elt *elt,
 236                               struct trace_buffer *buffer,
 237                               struct ring_buffer_event *rbe,
 238                               void *event)
 239 {
 240         char **addr = (char **)(event + hist_field->field->offset);
 241
 242         return (u64)(unsigned long)*addr;
 243 }
 244
 245 static u64 hist_field_log2(struct hist_field *hist_field,
 246                            struct tracing_map_elt *elt,
 247                            struct trace_buffer *buffer,
 248                            struct ring_buffer_event *rbe,
 249                            void *event)
 250 {
 251         struct hist_field *operand = hist_field->operands[0];
 252
 253         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 254
 255         return (u64) ilog2(roundup_pow_of_two(val));
 256 }
 257
 258 static u64 hist_field_bucket(struct hist_field *hist_field,
 259                              struct tracing_map_elt *elt,
 260                              struct trace_buffer *buffer,
 261                              struct ring_buffer_event *rbe,
 262                              void *event)
 263 {
 264         struct hist_field *operand = hist_field->operands[0];
 265         unsigned long buckets = hist_field->buckets;
 266
 267         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 268
 269         if (WARN_ON_ONCE(!buckets))
 270                 return val;
 271
 272         if (val >= LONG_MAX)
 273                 val = div64_ul(val, buckets);
 274         else
 275                 val = (u64)((unsigned long)val / buckets);
 276         return val * buckets;
 277 }
 278
 279 static u64 hist_field_plus(struct hist_field *hist_field,
 280                            struct tracing_map_elt *elt,
 281                            struct trace_buffer *buffer,
 282                            struct ring_buffer_event *rbe,
 283                            void *event)
 284 {
 285         struct hist_field *operand1 = hist_field->operands[0];
 286         struct hist_field *operand2 = hist_field->operands[1];
 287
 288         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 289         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 290
 291         return val1 + val2;
 292 }
 293
 294 static u64 hist_field_minus(struct hist_field *hist_field,
 295                             struct tracing_map_elt *elt,
 296                             struct trace_buffer *buffer,
 297                             struct ring_buffer_event *rbe,
 298                             void *event)
 299 {
 300         struct hist_field *operand1 = hist_field->operands[0];
 301         struct hist_field *operand2 = hist_field->operands[1];
 302
 303         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 304         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 305
 306         return val1 - val2;
 307 }
 308
 309 static u64 hist_field_div(struct hist_field *hist_field,
 310                            struct tracing_map_elt *elt,
 311                            struct trace_buffer *buffer,
 312                            struct ring_buffer_event *rbe,
 313                            void *event)
 314 {
 315         struct hist_field *operand1 = hist_field->operands[0];
 316         struct hist_field *operand2 = hist_field->operands[1];
 317
 318         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 319         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 320
 321         /* Return -1 for the undefined case */
 322         if (!val2)
 323                 return -1;
 324
 325         /* Use shift if the divisor is a power of 2 */
 326         if (!(val2 & (val2 - 1)))
 327                 return val1 >> __ffs64(val2);
 328
 329         return div64_u64(val1, val2);
 330 }
 331
 332 static u64 div_by_power_of_two(struct hist_field *hist_field,
 333                                 struct tracing_map_elt *elt,
 334                                 struct trace_buffer *buffer,
 335                                 struct ring_buffer_event *rbe,
 336                                 void *event)
 337 {
 338         struct hist_field *operand1 = hist_field->operands[0];
 339         struct hist_field *operand2 = hist_field->operands[1];
 340
 341         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 342
 343         return val1 >> __ffs64(operand2->constant);
 344 }
 345
 346 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 347                                 struct tracing_map_elt *elt,
 348                                 struct trace_buffer *buffer,
 349                                 struct ring_buffer_event *rbe,
 350                                 void *event)
 351 {
 352         struct hist_field *operand1 = hist_field->operands[0];
 353         struct hist_field *operand2 = hist_field->operands[1];
 354
 355         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 356
 357         return div64_u64(val1, operand2->constant);
 358 }
 359
 360 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 361                                 struct tracing_map_elt *elt,
 362                                 struct trace_buffer *buffer,
 363                                 struct ring_buffer_event *rbe,
 364                                 void *event)
 365 {
 366         struct hist_field *operand1 = hist_field->operands[0];
 367         struct hist_field *operand2 = hist_field->operands[1];
 368
 369         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 370
 371         /*
 372          * If the divisor is a constant, do a multiplication and shift instead.
 373          *
 374          * Choose Z = some power of 2. If Y <= Z, then:
 375          *     X / Y = (X * (Z / Y)) / Z
 376          *
 377          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 378          *     X / Y = (X * mult) / Z
 379          *
 380          * The division by Z can be replaced by a shift since Z is a power of 2:
 381          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 382          *
 383          * As long, as X < Z the results will not be off by more than 1.
 384          */
 385         if (val1 < (1 << HIST_DIV_SHIFT)) {
 386                 u64 mult = operand2->div_multiplier;
 387
 388                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 389         }
 390
 391         return div64_u64(val1, operand2->constant);
 392 }
 393
 394 static u64 hist_field_mult(struct hist_field *hist_field,
 395                            struct tracing_map_elt *elt,
 396                            struct trace_buffer *buffer,
 397                            struct ring_buffer_event *rbe,
 398                            void *event)
 399 {
 400         struct hist_field *operand1 = hist_field->operands[0];
 401         struct hist_field *operand2 = hist_field->operands[1];
 402
 403         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 404         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 405
 406         return val1 * val2;
 407 }
 408
 409 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 410                                   struct tracing_map_elt *elt,
 411                                   struct trace_buffer *buffer,
 412                                   struct ring_buffer_event *rbe,
 413                                   void *event)
 414 {
 415         struct hist_field *operand = hist_field->operands[0];
 416
 417         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 418         u64 val = (u64)-sval;
 419
 420         return val;
 421 }
 422
 423 #define DEFINE_HIST_FIELD_FN(type)                                      \
 424         static u64 hist_field_##type(struct hist_field *hist_field,     \
 425                                      struct tracing_map_elt *elt,       \
 426                                      struct trace_buffer *buffer,       \
 427                                      struct ring_buffer_event *rbe,     \
 428                                      void *event)                       \
 429 {                                                                       \
 430         type *addr = (type *)(event + hist_field->field->offset);       \
 431                                                                         \
 432         return (u64)(unsigned long)*addr;                               \
 433 }
 434
 435 DEFINE_HIST_FIELD_FN(s64);
 436 DEFINE_HIST_FIELD_FN(u64);
 437 DEFINE_HIST_FIELD_FN(s32);
 438 DEFINE_HIST_FIELD_FN(u32);
 439 DEFINE_HIST_FIELD_FN(s16);
 440 DEFINE_HIST_FIELD_FN(u16);
 441 DEFINE_HIST_FIELD_FN(s8);
 442 DEFINE_HIST_FIELD_FN(u8);
 443
 444 #define for_each_hist_field(i, hist_data)       \
 445         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 446
 447 #define for_each_hist_val_field(i, hist_data)   \
 448         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 449
 450 #define for_each_hist_key_field(i, hist_data)   \
 451         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 452
 453 #define HIST_STACKTRACE_DEPTH   16
 454 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 455 #define HIST_STACKTRACE_SKIP    5
 456
 457 #define HITCOUNT_IDX            0
 458 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 459
 460 enum hist_field_flags {
 461         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 462         HIST_FIELD_FL_KEY               = 1 << 1,
 463         HIST_FIELD_FL_STRING            = 1 << 2,
 464         HIST_FIELD_FL_HEX               = 1 << 3,
 465         HIST_FIELD_FL_SYM               = 1 << 4,
 466         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 467         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 468         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 469         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 470         HIST_FIELD_FL_LOG2              = 1 << 9,
 471         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 472         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 473         HIST_FIELD_FL_VAR               = 1 << 12,
 474         HIST_FIELD_FL_EXPR              = 1 << 13,
 475         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 476         HIST_FIELD_FL_CPU               = 1 << 15,
 477         HIST_FIELD_FL_ALIAS             = 1 << 16,
 478         HIST_FIELD_FL_BUCKET            = 1 << 17,
 479         HIST_FIELD_FL_CONST             = 1 << 18,
 480 };
 481
 482 struct var_defs {
 483         unsigned int    n_vars;
 484         char            *name[TRACING_MAP_VARS_MAX];
 485         char            *expr[TRACING_MAP_VARS_MAX];
 486 };
 487
 488 struct hist_trigger_attrs {
 489         char            *keys_str;
 490         char            *vals_str;
 491         char            *sort_key_str;
 492         char            *name;
 493         char            *clock;
 494         bool            pause;
 495         bool            cont;
 496         bool            clear;
 497         bool            ts_in_usecs;
 498         unsigned int    map_bits;
 499
 500         char            *assignment_str[TRACING_MAP_VARS_MAX];
 501         unsigned int    n_assignments;
 502
 503         char            *action_str[HIST_ACTIONS_MAX];
 504         unsigned int    n_actions;
 505
 506         struct var_defs var_defs;
 507 };
 508
 509 struct field_var {
 510         struct hist_field       *var;
 511         struct hist_field       *val;
 512 };
 513
 514 struct field_var_hist {
 515         struct hist_trigger_data        *hist_data;
 516         char                            *cmd;
 517 };
 518
 519 struct hist_trigger_data {
 520         struct hist_field               *fields[HIST_FIELDS_MAX];
 521         unsigned int                    n_vals;
 522         unsigned int                    n_keys;
 523         unsigned int                    n_fields;
 524         unsigned int                    n_vars;
 525         unsigned int                    n_var_str;
 526         unsigned int                    key_size;
 527         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 528         unsigned int                    n_sort_keys;
 529         struct trace_event_file         *event_file;
 530         struct hist_trigger_attrs       *attrs;
 531         struct tracing_map              *map;
 532         bool                            enable_timestamps;
 533         bool                            remove;
 534         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 535         unsigned int                    n_var_refs;
 536
 537         struct action_data              *actions[HIST_ACTIONS_MAX];
 538         unsigned int                    n_actions;
 539
 540         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 541         unsigned int                    n_field_vars;
 542         unsigned int                    n_field_var_str;
 543         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 544         unsigned int                    n_field_var_hists;
 545
 546         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 547         unsigned int                    n_save_vars;
 548         unsigned int                    n_save_var_str;
 549 };
 550
 551 struct action_data;
 552
 553 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 554                              struct tracing_map_elt *elt,
 555                              struct trace_buffer *buffer, void *rec,
 556                              struct ring_buffer_event *rbe, void *key,
 557                              struct action_data *data, u64 *var_ref_vals);
 558
 559 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 560
 561 enum handler_id {
 562         HANDLER_ONMATCH = 1,
 563         HANDLER_ONMAX,
 564         HANDLER_ONCHANGE,
 565 };
 566
 567 enum action_id {
 568         ACTION_SAVE = 1,
 569         ACTION_TRACE,
 570         ACTION_SNAPSHOT,
 571 };
 572
 573 struct action_data {
 574         enum handler_id         handler;
 575         enum action_id          action;
 576         char                    *action_name;
 577         action_fn_t             fn;
 578
 579         unsigned int            n_params;
 580         char                    *params[SYNTH_FIELDS_MAX];
 581
 582         /*
 583          * When a histogram trigger is hit, the values of any
 584          * references to variables, including variables being passed
 585          * as parameters to synthetic events, are collected into a
 586          * var_ref_vals array.  This var_ref_idx array is an array of
 587          * indices into the var_ref_vals array, one for each synthetic
 588          * event param, and is passed to the synthetic event
 589          * invocation.
 590          */
 591         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 592         struct synth_event      *synth_event;
 593         bool                    use_trace_keyword;
 594         char                    *synth_event_name;
 595
 596         union {
 597                 struct {
 598                         char                    *event;
 599                         char                    *event_system;
 600                 } match_data;
 601
 602                 struct {
 603                         /*
 604                          * var_str contains the $-unstripped variable
 605                          * name referenced by var_ref, and used when
 606                          * printing the action.  Because var_ref
 607                          * creation is deferred to create_actions(),
 608                          * we need a per-action way to save it until
 609                          * then, thus var_str.
 610                          */
 611                         char                    *var_str;
 612
 613                         /*
 614                          * var_ref refers to the variable being
 615                          * tracked e.g onmax($var).
 616                          */
 617                         struct hist_field       *var_ref;
 618
 619                         /*
 620                          * track_var contains the 'invisible' tracking
 621                          * variable created to keep the current
 622                          * e.g. max value.
 623                          */
 624                         struct hist_field       *track_var;
 625
 626                         check_track_val_fn_t    check_val;
 627                         action_fn_t             save_data;
 628                 } track_data;
 629         };
 630 };
 631
 632 struct track_data {
 633         u64                             track_val;
 634         bool                            updated;
 635
 636         unsigned int                    key_len;
 637         void                            *key;
 638         struct tracing_map_elt          elt;
 639
 640         struct action_data              *action_data;
 641         struct hist_trigger_data        *hist_data;
 642 };
 643
 644 struct hist_elt_data {
 645         char *comm;
 646         u64 *var_ref_vals;
 647         char **field_var_str;
 648         int n_field_var_str;
 649 };
 650
 651 struct snapshot_context {
 652         struct tracing_map_elt  *elt;
 653         void                    *key;
 654 };
 655
 656 /*
 657  * Returns the specific division function to use if the divisor
 658  * is constant. This avoids extra branches when the trigger is hit.
 659  */
 660 static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
 661 {
 662         u64 div = divisor->constant;
 663
 664         if (!(div & (div - 1)))
 665                 return div_by_power_of_two;
 666
 667         /* If the divisor is too large, do a regular division */
 668         if (div > (1 << HIST_DIV_SHIFT))
 669                 return div_by_not_power_of_two;
 670
 671         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 672         return div_by_mult_and_shift;
 673 }
 674
 675 static void track_data_free(struct track_data *track_data)
 676 {
 677         struct hist_elt_data *elt_data;
 678
 679         if (!track_data)
 680                 return;
 681
 682         kfree(track_data->key);
 683
 684         elt_data = track_data->elt.private_data;
 685         if (elt_data) {
 686                 kfree(elt_data->comm);
 687                 kfree(elt_data);
 688         }
 689
 690         kfree(track_data);
 691 }
 692
 693 static struct track_data *track_data_alloc(unsigned int key_len,
 694                                            struct action_data *action_data,
 695                                            struct hist_trigger_data *hist_data)
 696 {
 697         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 698         struct hist_elt_data *elt_data;
 699
 700         if (!data)
 701                 return ERR_PTR(-ENOMEM);
 702
 703         data->key = kzalloc(key_len, GFP_KERNEL);
 704         if (!data->key) {
 705                 track_data_free(data);
 706                 return ERR_PTR(-ENOMEM);
 707         }
 708
 709         data->key_len = key_len;
 710         data->action_data = action_data;
 711         data->hist_data = hist_data;
 712
 713         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 714         if (!elt_data) {
 715                 track_data_free(data);
 716                 return ERR_PTR(-ENOMEM);
 717         }
 718
 719         data->elt.private_data = elt_data;
 720
 721         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 722         if (!elt_data->comm) {
 723                 track_data_free(data);
 724                 return ERR_PTR(-ENOMEM);
 725         }
 726
 727         return data;
 728 }
 729
 730 static char last_cmd[MAX_FILTER_STR_VAL];
 731 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 732
 733 static int errpos(char *str)
 734 {
 735         return err_pos(last_cmd, str);
 736 }
 737
 738 static void last_cmd_set(struct trace_event_file *file, char *str)
 739 {
 740         const char *system = NULL, *name = NULL;
 741         struct trace_event_call *call;
 742
 743         if (!str)
 744                 return;
 745
 746         strcpy(last_cmd, "hist:");
 747         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 748
 749         if (file) {
 750                 call = file->event_call;
 751                 system = call->class->system;
 752                 if (system) {
 753                         name = trace_event_name(call);
 754                         if (!name)
 755                                 system = NULL;
 756                 }
 757         }
 758
 759         if (system)
 760                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 761 }
 762
 763 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 764 {
 765         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 766                         err_type, err_pos);
 767 }
 768
 769 static void hist_err_clear(void)
 770 {
 771         last_cmd[0] = '\0';
 772         last_cmd_loc[0] = '\0';
 773 }
 774
 775 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 776                                     unsigned int *var_ref_idx);
 777
 778 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 779                                unsigned int *var_ref_idx)
 780 {
 781         struct tracepoint *tp = event->tp;
 782
 783         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 784                 struct tracepoint_func *probe_func_ptr;
 785                 synth_probe_func_t probe_func;
 786                 void *__data;
 787
 788                 if (!(cpu_online(raw_smp_processor_id())))
 789                         return;
 790
 791                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 792                 if (probe_func_ptr) {
 793                         do {
 794                                 probe_func = probe_func_ptr->func;
 795                                 __data = probe_func_ptr->data;
 796                                 probe_func(__data, var_ref_vals, var_ref_idx);
 797                         } while ((++probe_func_ptr)->func);
 798                 }
 799         }
 800 }
 801
 802 static void action_trace(struct hist_trigger_data *hist_data,
 803                          struct tracing_map_elt *elt,
 804                          struct trace_buffer *buffer, void *rec,
 805                          struct ring_buffer_event *rbe, void *key,
 806                          struct action_data *data, u64 *var_ref_vals)
 807 {
 808         struct synth_event *event = data->synth_event;
 809
 810         trace_synth(event, var_ref_vals, data->var_ref_idx);
 811 }
 812
 813 struct hist_var_data {
 814         struct list_head list;
 815         struct hist_trigger_data *hist_data;
 816 };
 817
 818 static u64 hist_field_timestamp(struct hist_field *hist_field,
 819                                 struct tracing_map_elt *elt,
 820                                 struct trace_buffer *buffer,
 821                                 struct ring_buffer_event *rbe,
 822                                 void *event)
 823 {
 824         struct hist_trigger_data *hist_data = hist_field->hist_data;
 825         struct trace_array *tr = hist_data->event_file->tr;
 826
 827         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 828
 829         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 830                 ts = ns2usecs(ts);
 831
 832         return ts;
 833 }
 834
 835 static u64 hist_field_cpu(struct hist_field *hist_field,
 836                           struct tracing_map_elt *elt,
 837                           struct trace_buffer *buffer,
 838                           struct ring_buffer_event *rbe,
 839                           void *event)
 840 {
 841         int cpu = smp_processor_id();
 842
 843         return cpu;
 844 }
 845
 846 /**
 847  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 848  * @hist_field: The VAR_REF field to check
 849  * @var_data: The hist trigger that owns the variable
 850  * @var_idx: The trigger variable identifier
 851  *
 852  * Check the given VAR_REF field to see whether or not it references
 853  * the given variable associated with the given trigger.
 854  *
 855  * Return: The VAR_REF field if it does reference the variable, NULL if not
 856  */
 857 static struct hist_field *
 858 check_field_for_var_ref(struct hist_field *hist_field,
 859                         struct hist_trigger_data *var_data,
 860                         unsigned int var_idx)
 861 {
 862         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 863
 864         if (hist_field && hist_field->var.idx == var_idx &&
 865             hist_field->var.hist_data == var_data)
 866                 return hist_field;
 867
 868         return NULL;
 869 }
 870
 871 /**
 872  * find_var_ref - Check if a trigger has a reference to a trigger variable
 873  * @hist_data: The hist trigger that might have a reference to the variable
 874  * @var_data: The hist trigger that owns the variable
 875  * @var_idx: The trigger variable identifier
 876  *
 877  * Check the list of var_refs[] on the first hist trigger to see
 878  * whether any of them are references to the variable on the second
 879  * trigger.
 880  *
 881  * Return: The VAR_REF field referencing the variable if so, NULL if not
 882  */
 883 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 884                                        struct hist_trigger_data *var_data,
 885                                        unsigned int var_idx)
 886 {
 887         struct hist_field *hist_field;
 888         unsigned int i;
 889
 890         for (i = 0; i < hist_data->n_var_refs; i++) {
 891                 hist_field = hist_data->var_refs[i];
 892                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 893                         return hist_field;
 894         }
 895
 896         return NULL;
 897 }
 898
 899 /**
 900  * find_any_var_ref - Check if there is a reference to a given trigger variable
 901  * @hist_data: The hist trigger
 902  * @var_idx: The trigger variable identifier
 903  *
 904  * Check to see whether the given variable is currently referenced by
 905  * any other trigger.
 906  *
 907  * The trigger the variable is defined on is explicitly excluded - the
 908  * assumption being that a self-reference doesn't prevent a trigger
 909  * from being removed.
 910  *
 911  * Return: The VAR_REF field referencing the variable if so, NULL if not
 912  */
 913 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 914                                            unsigned int var_idx)
 915 {
 916         struct trace_array *tr = hist_data->event_file->tr;
 917         struct hist_field *found = NULL;
 918         struct hist_var_data *var_data;
 919
 920         list_for_each_entry(var_data, &tr->hist_vars, list) {
 921                 if (var_data->hist_data == hist_data)
 922                         continue;
 923                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 924                 if (found)
 925                         break;
 926         }
 927
 928         return found;
 929 }
 930
 931 /**
 932  * check_var_refs - Check if there is a reference to any of trigger's variables
 933  * @hist_data: The hist trigger
 934  *
 935  * A trigger can define one or more variables.  If any one of them is
 936  * currently referenced by any other trigger, this function will
 937  * determine that.
 938
 939  * Typically used to determine whether or not a trigger can be removed
 940  * - if there are any references to a trigger's variables, it cannot.
 941  *
 942  * Return: True if there is a reference to any of trigger's variables
 943  */
 944 static bool check_var_refs(struct hist_trigger_data *hist_data)
 945 {
 946         struct hist_field *field;
 947         bool found = false;
 948         int i;
 949
 950         for_each_hist_field(i, hist_data) {
 951                 field = hist_data->fields[i];
 952                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 953                         if (find_any_var_ref(hist_data, field->var.idx)) {
 954                                 found = true;
 955                                 break;
 956                         }
 957                 }
 958         }
 959
 960         return found;
 961 }
 962
 963 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 964 {
 965         struct trace_array *tr = hist_data->event_file->tr;
 966         struct hist_var_data *var_data, *found = NULL;
 967
 968         list_for_each_entry(var_data, &tr->hist_vars, list) {
 969                 if (var_data->hist_data == hist_data) {
 970                         found = var_data;
 971                         break;
 972                 }
 973         }
 974
 975         return found;
 976 }
 977
 978 static bool field_has_hist_vars(struct hist_field *hist_field,
 979                                 unsigned int level)
 980 {
 981         int i;
 982
 983         if (level > 3)
 984                 return false;
 985
 986         if (!hist_field)
 987                 return false;
 988
 989         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 990             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 991                 return true;
 992
 993         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 994                 struct hist_field *operand;
 995
 996                 operand = hist_field->operands[i];
 997                 if (field_has_hist_vars(operand, level + 1))
 998                         return true;
 999         }
1000
1001         return false;
1002 }
1003
1004 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1005 {
1006         struct hist_field *hist_field;
1007         int i;
1008
1009         for_each_hist_field(i, hist_data) {
1010                 hist_field = hist_data->fields[i];
1011                 if (field_has_hist_vars(hist_field, 0))
1012                         return true;
1013         }
1014
1015         return false;
1016 }
1017
1018 static int save_hist_vars(struct hist_trigger_data *hist_data)
1019 {
1020         struct trace_array *tr = hist_data->event_file->tr;
1021         struct hist_var_data *var_data;
1022
1023         var_data = find_hist_vars(hist_data);
1024         if (var_data)
1025                 return 0;
1026
1027         if (tracing_check_open_get_tr(tr))
1028                 return -ENODEV;
1029
1030         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1031         if (!var_data) {
1032                 trace_array_put(tr);
1033                 return -ENOMEM;
1034         }
1035
1036         var_data->hist_data = hist_data;
1037         list_add(&var_data->list, &tr->hist_vars);
1038
1039         return 0;
1040 }
1041
1042 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1043 {
1044         struct trace_array *tr = hist_data->event_file->tr;
1045         struct hist_var_data *var_data;
1046
1047         var_data = find_hist_vars(hist_data);
1048         if (!var_data)
1049                 return;
1050
1051         if (WARN_ON(check_var_refs(hist_data)))
1052                 return;
1053
1054         list_del(&var_data->list);
1055
1056         kfree(var_data);
1057
1058         trace_array_put(tr);
1059 }
1060
1061 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1062                                          const char *var_name)
1063 {
1064         struct hist_field *hist_field, *found = NULL;
1065         int i;
1066
1067         for_each_hist_field(i, hist_data) {
1068                 hist_field = hist_data->fields[i];
1069                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1070                     strcmp(hist_field->var.name, var_name) == 0) {
1071                         found = hist_field;
1072                         break;
1073                 }
1074         }
1075
1076         return found;
1077 }
1078
1079 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1080                                    struct trace_event_file *file,
1081                                    const char *var_name)
1082 {
1083         struct hist_trigger_data *test_data;
1084         struct event_trigger_data *test;
1085         struct hist_field *hist_field;
1086
1087         lockdep_assert_held(&event_mutex);
1088
1089         hist_field = find_var_field(hist_data, var_name);
1090         if (hist_field)
1091                 return hist_field;
1092
1093         list_for_each_entry(test, &file->triggers, list) {
1094                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1095                         test_data = test->private_data;
1096                         hist_field = find_var_field(test_data, var_name);
1097                         if (hist_field)
1098                                 return hist_field;
1099                 }
1100         }
1101
1102         return NULL;
1103 }
1104
1105 static struct trace_event_file *find_var_file(struct trace_array *tr,
1106                                               char *system,
1107                                               char *event_name,
1108                                               char *var_name)
1109 {
1110         struct hist_trigger_data *var_hist_data;
1111         struct hist_var_data *var_data;
1112         struct trace_event_file *file, *found = NULL;
1113
1114         if (system)
1115                 return find_event_file(tr, system, event_name);
1116
1117         list_for_each_entry(var_data, &tr->hist_vars, list) {
1118                 var_hist_data = var_data->hist_data;
1119                 file = var_hist_data->event_file;
1120                 if (file == found)
1121                         continue;
1122
1123                 if (find_var_field(var_hist_data, var_name)) {
1124                         if (found) {
1125                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1126                                 return NULL;
1127                         }
1128
1129                         found = file;
1130                 }
1131         }
1132
1133         return found;
1134 }
1135
1136 static struct hist_field *find_file_var(struct trace_event_file *file,
1137                                         const char *var_name)
1138 {
1139         struct hist_trigger_data *test_data;
1140         struct event_trigger_data *test;
1141         struct hist_field *hist_field;
1142
1143         lockdep_assert_held(&event_mutex);
1144
1145         list_for_each_entry(test, &file->triggers, list) {
1146                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1147                         test_data = test->private_data;
1148                         hist_field = find_var_field(test_data, var_name);
1149                         if (hist_field)
1150                                 return hist_field;
1151                 }
1152         }
1153
1154         return NULL;
1155 }
1156
1157 static struct hist_field *
1158 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1159 {
1160         struct trace_array *tr = hist_data->event_file->tr;
1161         struct hist_field *hist_field, *found = NULL;
1162         struct trace_event_file *file;
1163         unsigned int i;
1164
1165         for (i = 0; i < hist_data->n_actions; i++) {
1166                 struct action_data *data = hist_data->actions[i];
1167
1168                 if (data->handler == HANDLER_ONMATCH) {
1169                         char *system = data->match_data.event_system;
1170                         char *event_name = data->match_data.event;
1171
1172                         file = find_var_file(tr, system, event_name, var_name);
1173                         if (!file)
1174                                 continue;
1175                         hist_field = find_file_var(file, var_name);
1176                         if (hist_field) {
1177                                 if (found) {
1178                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1179                                                  errpos(var_name));
1180                                         return ERR_PTR(-EINVAL);
1181                                 }
1182
1183                                 found = hist_field;
1184                         }
1185                 }
1186         }
1187         return found;
1188 }
1189
1190 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1191                                          char *system,
1192                                          char *event_name,
1193                                          char *var_name)
1194 {
1195         struct trace_array *tr = hist_data->event_file->tr;
1196         struct hist_field *hist_field = NULL;
1197         struct trace_event_file *file;
1198
1199         if (!system || !event_name) {
1200                 hist_field = find_match_var(hist_data, var_name);
1201                 if (IS_ERR(hist_field))
1202                         return NULL;
1203                 if (hist_field)
1204                         return hist_field;
1205         }
1206
1207         file = find_var_file(tr, system, event_name, var_name);
1208         if (!file)
1209                 return NULL;
1210
1211         hist_field = find_file_var(file, var_name);
1212
1213         return hist_field;
1214 }
1215
1216 static u64 hist_field_var_ref(struct hist_field *hist_field,
1217                               struct tracing_map_elt *elt,
1218                               struct trace_buffer *buffer,
1219                               struct ring_buffer_event *rbe,
1220                               void *event)
1221 {
1222         struct hist_elt_data *elt_data;
1223         u64 var_val = 0;
1224
1225         if (WARN_ON_ONCE(!elt))
1226                 return var_val;
1227
1228         elt_data = elt->private_data;
1229         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1230
1231         return var_val;
1232 }
1233
1234 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1235                              u64 *var_ref_vals, bool self)
1236 {
1237         struct hist_trigger_data *var_data;
1238         struct tracing_map_elt *var_elt;
1239         struct hist_field *hist_field;
1240         unsigned int i, var_idx;
1241         bool resolved = true;
1242         u64 var_val = 0;
1243
1244         for (i = 0; i < hist_data->n_var_refs; i++) {
1245                 hist_field = hist_data->var_refs[i];
1246                 var_idx = hist_field->var.idx;
1247                 var_data = hist_field->var.hist_data;
1248
1249                 if (var_data == NULL) {
1250                         resolved = false;
1251                         break;
1252                 }
1253
1254                 if ((self && var_data != hist_data) ||
1255                     (!self && var_data == hist_data))
1256                         continue;
1257
1258                 var_elt = tracing_map_lookup(var_data->map, key);
1259                 if (!var_elt) {
1260                         resolved = false;
1261                         break;
1262                 }
1263
1264                 if (!tracing_map_var_set(var_elt, var_idx)) {
1265                         resolved = false;
1266                         break;
1267                 }
1268
1269                 if (self || !hist_field->read_once)
1270                         var_val = tracing_map_read_var(var_elt, var_idx);
1271                 else
1272                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1273
1274                 var_ref_vals[i] = var_val;
1275         }
1276
1277         return resolved;
1278 }
1279
1280 static const char *hist_field_name(struct hist_field *field,
1281                                    unsigned int level)
1282 {
1283         const char *field_name = "";
1284
1285         if (level > 1)
1286                 return field_name;
1287
1288         if (field->field)
1289                 field_name = field->field->name;
1290         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1291                  field->flags & HIST_FIELD_FL_ALIAS ||
1292                  field->flags & HIST_FIELD_FL_BUCKET)
1293                 field_name = hist_field_name(field->operands[0], ++level);
1294         else if (field->flags & HIST_FIELD_FL_CPU)
1295                 field_name = "common_cpu";
1296         else if (field->flags & HIST_FIELD_FL_EXPR ||
1297                  field->flags & HIST_FIELD_FL_VAR_REF) {
1298                 if (field->system) {
1299                         static char full_name[MAX_FILTER_STR_VAL];
1300
1301                         strcat(full_name, field->system);
1302                         strcat(full_name, ".");
1303                         strcat(full_name, field->event_name);
1304                         strcat(full_name, ".");
1305                         strcat(full_name, field->name);
1306                         field_name = full_name;
1307                 } else
1308                         field_name = field->name;
1309         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1310                 field_name = "common_timestamp";
1311
1312         if (field_name == NULL)
1313                 field_name = "";
1314
1315         return field_name;
1316 }
1317
1318 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1319 {
1320         hist_field_fn_t fn = NULL;
1321
1322         switch (field_size) {
1323         case 8:
1324                 if (field_is_signed)
1325                         fn = hist_field_s64;
1326                 else
1327                         fn = hist_field_u64;
1328                 break;
1329         case 4:
1330                 if (field_is_signed)
1331                         fn = hist_field_s32;
1332                 else
1333                         fn = hist_field_u32;
1334                 break;
1335         case 2:
1336                 if (field_is_signed)
1337                         fn = hist_field_s16;
1338                 else
1339                         fn = hist_field_u16;
1340                 break;
1341         case 1:
1342                 if (field_is_signed)
1343                         fn = hist_field_s8;
1344                 else
1345                         fn = hist_field_u8;
1346                 break;
1347         }
1348
1349         return fn;
1350 }
1351
1352 static int parse_map_size(char *str)
1353 {
1354         unsigned long size, map_bits;
1355         int ret;
1356
1357         ret = kstrtoul(str, 0, &size);
1358         if (ret)
1359                 goto out;
1360
1361         map_bits = ilog2(roundup_pow_of_two(size));
1362         if (map_bits < TRACING_MAP_BITS_MIN ||
1363             map_bits > TRACING_MAP_BITS_MAX)
1364                 ret = -EINVAL;
1365         else
1366                 ret = map_bits;
1367  out:
1368         return ret;
1369 }
1370
1371 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1372 {
1373         unsigned int i;
1374
1375         if (!attrs)
1376                 return;
1377
1378         for (i = 0; i < attrs->n_assignments; i++)
1379                 kfree(attrs->assignment_str[i]);
1380
1381         for (i = 0; i < attrs->n_actions; i++)
1382                 kfree(attrs->action_str[i]);
1383
1384         kfree(attrs->name);
1385         kfree(attrs->sort_key_str);
1386         kfree(attrs->keys_str);
1387         kfree(attrs->vals_str);
1388         kfree(attrs->clock);
1389         kfree(attrs);
1390 }
1391
1392 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1393 {
1394         int ret = -EINVAL;
1395
1396         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1397                 return ret;
1398
1399         if ((str_has_prefix(str, "onmatch(")) ||
1400             (str_has_prefix(str, "onmax(")) ||
1401             (str_has_prefix(str, "onchange("))) {
1402                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1403                 if (!attrs->action_str[attrs->n_actions]) {
1404                         ret = -ENOMEM;
1405                         return ret;
1406                 }
1407                 attrs->n_actions++;
1408                 ret = 0;
1409         }
1410         return ret;
1411 }
1412
1413 static int parse_assignment(struct trace_array *tr,
1414                             char *str, struct hist_trigger_attrs *attrs)
1415 {
1416         int len, ret = 0;
1417
1418         if ((len = str_has_prefix(str, "key=")) ||
1419             (len = str_has_prefix(str, "keys="))) {
1420                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1421                 if (!attrs->keys_str) {
1422                         ret = -ENOMEM;
1423                         goto out;
1424                 }
1425         } else if ((len = str_has_prefix(str, "val=")) ||
1426                    (len = str_has_prefix(str, "vals=")) ||
1427                    (len = str_has_prefix(str, "values="))) {
1428                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1429                 if (!attrs->vals_str) {
1430                         ret = -ENOMEM;
1431                         goto out;
1432                 }
1433         } else if ((len = str_has_prefix(str, "sort="))) {
1434                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1435                 if (!attrs->sort_key_str) {
1436                         ret = -ENOMEM;
1437                         goto out;
1438                 }
1439         } else if (str_has_prefix(str, "name=")) {
1440                 attrs->name = kstrdup(str, GFP_KERNEL);
1441                 if (!attrs->name) {
1442                         ret = -ENOMEM;
1443                         goto out;
1444                 }
1445         } else if ((len = str_has_prefix(str, "clock="))) {
1446                 str += len;
1447
1448                 str = strstrip(str);
1449                 attrs->clock = kstrdup(str, GFP_KERNEL);
1450                 if (!attrs->clock) {
1451                         ret = -ENOMEM;
1452                         goto out;
1453                 }
1454         } else if ((len = str_has_prefix(str, "size="))) {
1455                 int map_bits = parse_map_size(str + len);
1456
1457                 if (map_bits < 0) {
1458                         ret = map_bits;
1459                         goto out;
1460                 }
1461                 attrs->map_bits = map_bits;
1462         } else {
1463                 char *assignment;
1464
1465                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1466                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1467                         ret = -EINVAL;
1468                         goto out;
1469                 }
1470
1471                 assignment = kstrdup(str, GFP_KERNEL);
1472                 if (!assignment) {
1473                         ret = -ENOMEM;
1474                         goto out;
1475                 }
1476
1477                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1478         }
1479  out:
1480         return ret;
1481 }
1482
1483 static struct hist_trigger_attrs *
1484 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1485 {
1486         struct hist_trigger_attrs *attrs;
1487         int ret = 0;
1488
1489         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1490         if (!attrs)
1491                 return ERR_PTR(-ENOMEM);
1492
1493         while (trigger_str) {
1494                 char *str = strsep(&trigger_str, ":");
1495                 char *rhs;
1496
1497                 rhs = strchr(str, '=');
1498                 if (rhs) {
1499                         if (!strlen(++rhs)) {
1500                                 ret = -EINVAL;
1501                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1502                                 goto free;
1503                         }
1504                         ret = parse_assignment(tr, str, attrs);
1505                         if (ret)
1506                                 goto free;
1507                 } else if (strcmp(str, "pause") == 0)
1508                         attrs->pause = true;
1509                 else if ((strcmp(str, "cont") == 0) ||
1510                          (strcmp(str, "continue") == 0))
1511                         attrs->cont = true;
1512                 else if (strcmp(str, "clear") == 0)
1513                         attrs->clear = true;
1514                 else {
1515                         ret = parse_action(str, attrs);
1516                         if (ret)
1517                                 goto free;
1518                 }
1519         }
1520
1521         if (!attrs->keys_str) {
1522                 ret = -EINVAL;
1523                 goto free;
1524         }
1525
1526         if (!attrs->clock) {
1527                 attrs->clock = kstrdup("global", GFP_KERNEL);
1528                 if (!attrs->clock) {
1529                         ret = -ENOMEM;
1530                         goto free;
1531                 }
1532         }
1533
1534         return attrs;
1535  free:
1536         destroy_hist_trigger_attrs(attrs);
1537
1538         return ERR_PTR(ret);
1539 }
1540
1541 static inline void save_comm(char *comm, struct task_struct *task)
1542 {
1543         if (!task->pid) {
1544                 strcpy(comm, "<idle>");
1545                 return;
1546         }
1547
1548         if (WARN_ON_ONCE(task->pid < 0)) {
1549                 strcpy(comm, "<XXX>");
1550                 return;
1551         }
1552
1553         strncpy(comm, task->comm, TASK_COMM_LEN);
1554 }
1555
1556 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1557 {
1558         unsigned int i;
1559
1560         for (i = 0; i < elt_data->n_field_var_str; i++)
1561                 kfree(elt_data->field_var_str[i]);
1562
1563         kfree(elt_data->field_var_str);
1564
1565         kfree(elt_data->comm);
1566         kfree(elt_data);
1567 }
1568
1569 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1570 {
1571         struct hist_elt_data *elt_data = elt->private_data;
1572
1573         hist_elt_data_free(elt_data);
1574 }
1575
1576 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1577 {
1578         struct hist_trigger_data *hist_data = elt->map->private_data;
1579         unsigned int size = TASK_COMM_LEN;
1580         struct hist_elt_data *elt_data;
1581         struct hist_field *hist_field;
1582         unsigned int i, n_str;
1583
1584         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1585         if (!elt_data)
1586                 return -ENOMEM;
1587
1588         for_each_hist_field(i, hist_data) {
1589                 hist_field = hist_data->fields[i];
1590
1591                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1592                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1593                         if (!elt_data->comm) {
1594                                 kfree(elt_data);
1595                                 return -ENOMEM;
1596                         }
1597                         break;
1598                 }
1599         }
1600
1601         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1602                 hist_data->n_var_str;
1603         if (n_str > SYNTH_FIELDS_MAX) {
1604                 hist_elt_data_free(elt_data);
1605                 return -EINVAL;
1606         }
1607
1608         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1609
1610         size = STR_VAR_LEN_MAX;
1611
1612         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1613         if (!elt_data->field_var_str) {
1614                 hist_elt_data_free(elt_data);
1615                 return -EINVAL;
1616         }
1617         elt_data->n_field_var_str = n_str;
1618
1619         for (i = 0; i < n_str; i++) {
1620                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1621                 if (!elt_data->field_var_str[i]) {
1622                         hist_elt_data_free(elt_data);
1623                         return -ENOMEM;
1624                 }
1625         }
1626
1627         elt->private_data = elt_data;
1628
1629         return 0;
1630 }
1631
1632 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1633 {
1634         struct hist_elt_data *elt_data = elt->private_data;
1635
1636         if (elt_data->comm)
1637                 save_comm(elt_data->comm, current);
1638 }
1639
1640 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1641         .elt_alloc      = hist_trigger_elt_data_alloc,
1642         .elt_free       = hist_trigger_elt_data_free,
1643         .elt_init       = hist_trigger_elt_data_init,
1644 };
1645
1646 static const char *get_hist_field_flags(struct hist_field *hist_field)
1647 {
1648         const char *flags_str = NULL;
1649
1650         if (hist_field->flags & HIST_FIELD_FL_HEX)
1651                 flags_str = "hex";
1652         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1653                 flags_str = "sym";
1654         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1655                 flags_str = "sym-offset";
1656         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1657                 flags_str = "execname";
1658         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1659                 flags_str = "syscall";
1660         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1661                 flags_str = "log2";
1662         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1663                 flags_str = "buckets";
1664         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1665                 flags_str = "usecs";
1666
1667         return flags_str;
1668 }
1669
1670 static void expr_field_str(struct hist_field *field, char *expr)
1671 {
1672         if (field->flags & HIST_FIELD_FL_VAR_REF)
1673                 strcat(expr, "$");
1674         else if (field->flags & HIST_FIELD_FL_CONST) {
1675                 char str[HIST_CONST_DIGITS_MAX];
1676
1677                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1678                 strcat(expr, str);
1679         }
1680
1681         strcat(expr, hist_field_name(field, 0));
1682
1683         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1684                 const char *flags_str = get_hist_field_flags(field);
1685
1686                 if (flags_str) {
1687                         strcat(expr, ".");
1688                         strcat(expr, flags_str);
1689                 }
1690         }
1691 }
1692
1693 static char *expr_str(struct hist_field *field, unsigned int level)
1694 {
1695         char *expr;
1696
1697         if (level > 1)
1698                 return NULL;
1699
1700         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1701         if (!expr)
1702                 return NULL;
1703
1704         if (!field->operands[0]) {
1705                 expr_field_str(field, expr);
1706                 return expr;
1707         }
1708
1709         if (field->operator == FIELD_OP_UNARY_MINUS) {
1710                 char *subexpr;
1711
1712                 strcat(expr, "-(");
1713                 subexpr = expr_str(field->operands[0], ++level);
1714                 if (!subexpr) {
1715                         kfree(expr);
1716                         return NULL;
1717                 }
1718                 strcat(expr, subexpr);
1719                 strcat(expr, ")");
1720
1721                 kfree(subexpr);
1722
1723                 return expr;
1724         }
1725
1726         expr_field_str(field->operands[0], expr);
1727
1728         switch (field->operator) {
1729         case FIELD_OP_MINUS:
1730                 strcat(expr, "-");
1731                 break;
1732         case FIELD_OP_PLUS:
1733                 strcat(expr, "+");
1734                 break;
1735         case FIELD_OP_DIV:
1736                 strcat(expr, "/");
1737                 break;
1738         case FIELD_OP_MULT:
1739                 strcat(expr, "*");
1740                 break;
1741         default:
1742                 kfree(expr);
1743                 return NULL;
1744         }
1745
1746         expr_field_str(field->operands[1], expr);
1747
1748         return expr;
1749 }
1750
1751 /*
1752  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1753  * of the expression tree to be evaluated.
1754  */
1755 static int contains_operator(char *str, char **sep)
1756 {
1757         enum field_op_id field_op = FIELD_OP_NONE;
1758         char *minus_op, *plus_op, *div_op, *mult_op;
1759
1760
1761         /*
1762          * Report the last occurrence of the operators first, so that the
1763          * expression is evaluated left to right. This is important since
1764          * subtraction and division are not associative.
1765          *
1766          *      e.g
1767          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1768          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1769          */
1770
1771         /*
1772          * First, find lower precedence addition and subtraction
1773          * since the expression will be evaluated recursively.
1774          */
1775         minus_op = strrchr(str, '-');
1776         if (minus_op) {
1777                 /*
1778                  * Unary minus is not supported in sub-expressions. If
1779                  * present, it is always the next root operator.
1780                  */
1781                 if (minus_op == str) {
1782                         field_op = FIELD_OP_UNARY_MINUS;
1783                         goto out;
1784                 }
1785
1786                 field_op = FIELD_OP_MINUS;
1787         }
1788
1789         plus_op = strrchr(str, '+');
1790         if (plus_op || minus_op) {
1791                 /*
1792                  * For operators of the same precedence use to rightmost as the
1793                  * root, so that the expression is evaluated left to right.
1794                  */
1795                 if (plus_op > minus_op)
1796                         field_op = FIELD_OP_PLUS;
1797                 goto out;
1798         }
1799
1800         /*
1801          * Multiplication and division have higher precedence than addition and
1802          * subtraction.
1803          */
1804         div_op = strrchr(str, '/');
1805         if (div_op)
1806                 field_op = FIELD_OP_DIV;
1807
1808         mult_op = strrchr(str, '*');
1809         /*
1810          * For operators of the same precedence use to rightmost as the
1811          * root, so that the expression is evaluated left to right.
1812          */
1813         if (mult_op > div_op)
1814                 field_op = FIELD_OP_MULT;
1815
1816 out:
1817         if (sep) {
1818                 switch (field_op) {
1819                 case FIELD_OP_UNARY_MINUS:
1820                 case FIELD_OP_MINUS:
1821                         *sep = minus_op;
1822                         break;
1823                 case FIELD_OP_PLUS:
1824                         *sep = plus_op;
1825                         break;
1826                 case FIELD_OP_DIV:
1827                         *sep = div_op;
1828                         break;
1829                 case FIELD_OP_MULT:
1830                         *sep = mult_op;
1831                         break;
1832                 case FIELD_OP_NONE:
1833                 default:
1834                         *sep = NULL;
1835                         break;
1836                 }
1837         }
1838
1839         return field_op;
1840 }
1841
1842 static void get_hist_field(struct hist_field *hist_field)
1843 {
1844         hist_field->ref++;
1845 }
1846
1847 static void __destroy_hist_field(struct hist_field *hist_field)
1848 {
1849         if (--hist_field->ref > 1)
1850                 return;
1851
1852         kfree(hist_field->var.name);
1853         kfree(hist_field->name);
1854
1855         /* Can likely be a const */
1856         kfree_const(hist_field->type);
1857
1858         kfree(hist_field->system);
1859         kfree(hist_field->event_name);
1860
1861         kfree(hist_field);
1862 }
1863
1864 static void destroy_hist_field(struct hist_field *hist_field,
1865                                unsigned int level)
1866 {
1867         unsigned int i;
1868
1869         if (level > 3)
1870                 return;
1871
1872         if (!hist_field)
1873                 return;
1874
1875         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1876                 return; /* var refs will be destroyed separately */
1877
1878         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1879                 destroy_hist_field(hist_field->operands[i], level + 1);
1880
1881         __destroy_hist_field(hist_field);
1882 }
1883
1884 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1885                                             struct ftrace_event_field *field,
1886                                             unsigned long flags,
1887                                             char *var_name)
1888 {
1889         struct hist_field *hist_field;
1890
1891         if (field && is_function_field(field))
1892                 return NULL;
1893
1894         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1895         if (!hist_field)
1896                 return NULL;
1897
1898         hist_field->ref = 1;
1899
1900         hist_field->hist_data = hist_data;
1901
1902         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1903                 goto out; /* caller will populate */
1904
1905         if (flags & HIST_FIELD_FL_VAR_REF) {
1906                 hist_field->fn = hist_field_var_ref;
1907                 goto out;
1908         }
1909
1910         if (flags & HIST_FIELD_FL_HITCOUNT) {
1911                 hist_field->fn = hist_field_counter;
1912                 hist_field->size = sizeof(u64);
1913                 hist_field->type = "u64";
1914                 goto out;
1915         }
1916
1917         if (flags & HIST_FIELD_FL_CONST) {
1918                 hist_field->fn = hist_field_const;
1919                 hist_field->size = sizeof(u64);
1920                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1921                 if (!hist_field->type)
1922                         goto free;
1923                 goto out;
1924         }
1925
1926         if (flags & HIST_FIELD_FL_STACKTRACE) {
1927                 hist_field->fn = hist_field_none;
1928                 goto out;
1929         }
1930
1931         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1932                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1933                 hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1934                         hist_field_bucket;
1935                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1936                 hist_field->size = hist_field->operands[0]->size;
1937                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1938                 if (!hist_field->type)
1939                         goto free;
1940                 goto out;
1941         }
1942
1943         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1944                 hist_field->fn = hist_field_timestamp;
1945                 hist_field->size = sizeof(u64);
1946                 hist_field->type = "u64";
1947                 goto out;
1948         }
1949
1950         if (flags & HIST_FIELD_FL_CPU) {
1951                 hist_field->fn = hist_field_cpu;
1952                 hist_field->size = sizeof(int);
1953                 hist_field->type = "unsigned int";
1954                 goto out;
1955         }
1956
1957         if (WARN_ON_ONCE(!field))
1958                 goto out;
1959
1960         /* Pointers to strings are just pointers and dangerous to dereference */
1961         if (is_string_field(field) &&
1962             (field->filter_type != FILTER_PTR_STRING)) {
1963                 flags |= HIST_FIELD_FL_STRING;
1964
1965                 hist_field->size = MAX_FILTER_STR_VAL;
1966                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1967                 if (!hist_field->type)
1968                         goto free;
1969
1970                 if (field->filter_type == FILTER_STATIC_STRING) {
1971                         hist_field->fn = hist_field_string;
1972                         hist_field->size = field->size;
1973                 } else if (field->filter_type == FILTER_DYN_STRING) {
1974                         hist_field->fn = hist_field_dynstring;
1975                 } else if (field->filter_type == FILTER_RDYN_STRING)
1976                         hist_field->fn = hist_field_reldynstring;
1977                 else
1978                         hist_field->fn = hist_field_pstring;
1979         } else {
1980                 hist_field->size = field->size;
1981                 hist_field->is_signed = field->is_signed;
1982                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1983                 if (!hist_field->type)
1984                         goto free;
1985
1986                 hist_field->fn = select_value_fn(field->size,
1987                                                  field->is_signed);
1988                 if (!hist_field->fn) {
1989                         destroy_hist_field(hist_field, 0);
1990                         return NULL;
1991                 }
1992         }
1993  out:
1994         hist_field->field = field;
1995         hist_field->flags = flags;
1996
1997         if (var_name) {
1998                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1999                 if (!hist_field->var.name)
2000                         goto free;
2001         }
2002
2003         return hist_field;
2004  free:
2005         destroy_hist_field(hist_field, 0);
2006         return NULL;
2007 }
2008
2009 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2010 {
2011         unsigned int i;
2012
2013         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2014                 if (hist_data->fields[i]) {
2015                         destroy_hist_field(hist_data->fields[i], 0);
2016                         hist_data->fields[i] = NULL;
2017                 }
2018         }
2019
2020         for (i = 0; i < hist_data->n_var_refs; i++) {
2021                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2022                 __destroy_hist_field(hist_data->var_refs[i]);
2023                 hist_data->var_refs[i] = NULL;
2024         }
2025 }
2026
2027 static int init_var_ref(struct hist_field *ref_field,
2028                         struct hist_field *var_field,
2029                         char *system, char *event_name)
2030 {
2031         int err = 0;
2032
2033         ref_field->var.idx = var_field->var.idx;
2034         ref_field->var.hist_data = var_field->hist_data;
2035         ref_field->size = var_field->size;
2036         ref_field->is_signed = var_field->is_signed;
2037         ref_field->flags |= var_field->flags &
2038                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2039
2040         if (system) {
2041                 ref_field->system = kstrdup(system, GFP_KERNEL);
2042                 if (!ref_field->system)
2043                         return -ENOMEM;
2044         }
2045
2046         if (event_name) {
2047                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2048                 if (!ref_field->event_name) {
2049                         err = -ENOMEM;
2050                         goto free;
2051                 }
2052         }
2053
2054         if (var_field->var.name) {
2055                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2056                 if (!ref_field->name) {
2057                         err = -ENOMEM;
2058                         goto free;
2059                 }
2060         } else if (var_field->name) {
2061                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2062                 if (!ref_field->name) {
2063                         err = -ENOMEM;
2064                         goto free;
2065                 }
2066         }
2067
2068         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2069         if (!ref_field->type) {
2070                 err = -ENOMEM;
2071                 goto free;
2072         }
2073  out:
2074         return err;
2075  free:
2076         kfree(ref_field->system);
2077         kfree(ref_field->event_name);
2078         kfree(ref_field->name);
2079
2080         goto out;
2081 }
2082
2083 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2084                             struct hist_field *var_field)
2085 {
2086         struct hist_field *ref_field;
2087         int i;
2088
2089         for (i = 0; i < hist_data->n_var_refs; i++) {
2090                 ref_field = hist_data->var_refs[i];
2091                 if (ref_field->var.idx == var_field->var.idx &&
2092                     ref_field->var.hist_data == var_field->hist_data)
2093                         return i;
2094         }
2095
2096         return -ENOENT;
2097 }
2098
2099 /**
2100  * create_var_ref - Create a variable reference and attach it to trigger
2101  * @hist_data: The trigger that will be referencing the variable
2102  * @var_field: The VAR field to create a reference to
2103  * @system: The optional system string
2104  * @event_name: The optional event_name string
2105  *
2106  * Given a variable hist_field, create a VAR_REF hist_field that
2107  * represents a reference to it.
2108  *
2109  * This function also adds the reference to the trigger that
2110  * now references the variable.
2111  *
2112  * Return: The VAR_REF field if successful, NULL if not
2113  */
2114 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2115                                          struct hist_field *var_field,
2116                                          char *system, char *event_name)
2117 {
2118         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2119         struct hist_field *ref_field;
2120         int i;
2121
2122         /* Check if the variable already exists */
2123         for (i = 0; i < hist_data->n_var_refs; i++) {
2124                 ref_field = hist_data->var_refs[i];
2125                 if (ref_field->var.idx == var_field->var.idx &&
2126                     ref_field->var.hist_data == var_field->hist_data) {
2127                         get_hist_field(ref_field);
2128                         return ref_field;
2129                 }
2130         }
2131
2132         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2133         if (ref_field) {
2134                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2135                         destroy_hist_field(ref_field, 0);
2136                         return NULL;
2137                 }
2138
2139                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2140                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2141         }
2142
2143         return ref_field;
2144 }
2145
2146 static bool is_var_ref(char *var_name)
2147 {
2148         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2149                 return false;
2150
2151         return true;
2152 }
2153
2154 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2155                                  char *var_name)
2156 {
2157         char *name, *field;
2158         unsigned int i;
2159
2160         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2161                 name = hist_data->attrs->var_defs.name[i];
2162
2163                 if (strcmp(var_name, name) == 0) {
2164                         field = hist_data->attrs->var_defs.expr[i];
2165                         if (contains_operator(field, NULL) || is_var_ref(field))
2166                                 continue;
2167                         return field;
2168                 }
2169         }
2170
2171         return NULL;
2172 }
2173
2174 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2175                                  char *system, char *event_name,
2176                                  char *var_name)
2177 {
2178         struct trace_event_call *call;
2179
2180         if (system && event_name) {
2181                 call = hist_data->event_file->event_call;
2182
2183                 if (strcmp(system, call->class->system) != 0)
2184                         return NULL;
2185
2186                 if (strcmp(event_name, trace_event_name(call)) != 0)
2187                         return NULL;
2188         }
2189
2190         if (!!system != !!event_name)
2191                 return NULL;
2192
2193         if (!is_var_ref(var_name))
2194                 return NULL;
2195
2196         var_name++;
2197
2198         return field_name_from_var(hist_data, var_name);
2199 }
2200
2201 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2202                                         char *system, char *event_name,
2203                                         char *var_name)
2204 {
2205         struct hist_field *var_field = NULL, *ref_field = NULL;
2206         struct trace_array *tr = hist_data->event_file->tr;
2207
2208         if (!is_var_ref(var_name))
2209                 return NULL;
2210
2211         var_name++;
2212
2213         var_field = find_event_var(hist_data, system, event_name, var_name);
2214         if (var_field)
2215                 ref_field = create_var_ref(hist_data, var_field,
2216                                            system, event_name);
2217
2218         if (!ref_field)
2219                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2220
2221         return ref_field;
2222 }
2223
2224 static struct ftrace_event_field *
2225 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2226             char *field_str, unsigned long *flags, unsigned long *buckets)
2227 {
2228         struct ftrace_event_field *field = NULL;
2229         char *field_name, *modifier, *str;
2230         struct trace_array *tr = file->tr;
2231
2232         modifier = str = kstrdup(field_str, GFP_KERNEL);
2233         if (!modifier)
2234                 return ERR_PTR(-ENOMEM);
2235
2236         field_name = strsep(&modifier, ".");
2237         if (modifier) {
2238                 if (strcmp(modifier, "hex") == 0)
2239                         *flags |= HIST_FIELD_FL_HEX;
2240                 else if (strcmp(modifier, "sym") == 0)
2241                         *flags |= HIST_FIELD_FL_SYM;
2242                 /*
2243                  * 'sym-offset' occurrences in the trigger string are modified
2244                  * to 'symXoffset' to simplify arithmetic expression parsing.
2245                  */
2246                 else if (strcmp(modifier, "symXoffset") == 0)
2247                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2248                 else if ((strcmp(modifier, "execname") == 0) &&
2249                          (strcmp(field_name, "common_pid") == 0))
2250                         *flags |= HIST_FIELD_FL_EXECNAME;
2251                 else if (strcmp(modifier, "syscall") == 0)
2252                         *flags |= HIST_FIELD_FL_SYSCALL;
2253                 else if (strcmp(modifier, "log2") == 0)
2254                         *flags |= HIST_FIELD_FL_LOG2;
2255                 else if (strcmp(modifier, "usecs") == 0)
2256                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2257                 else if (strncmp(modifier, "bucket", 6) == 0) {
2258                         int ret;
2259
2260                         modifier += 6;
2261
2262                         if (*modifier == 's')
2263                                 modifier++;
2264                         if (*modifier != '=')
2265                                 goto error;
2266                         modifier++;
2267                         ret = kstrtoul(modifier, 0, buckets);
2268                         if (ret || !(*buckets))
2269                                 goto error;
2270                         *flags |= HIST_FIELD_FL_BUCKET;
2271                 } else {
2272  error:
2273                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2274                         field = ERR_PTR(-EINVAL);
2275                         goto out;
2276                 }
2277         }
2278
2279         if (strcmp(field_name, "common_timestamp") == 0) {
2280                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2281                 hist_data->enable_timestamps = true;
2282                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2283                         hist_data->attrs->ts_in_usecs = true;
2284         } else if (strcmp(field_name, "common_cpu") == 0)
2285                 *flags |= HIST_FIELD_FL_CPU;
2286         else {
2287                 field = trace_find_event_field(file->event_call, field_name);
2288                 if (!field || !field->size) {
2289                         /*
2290                          * For backward compatibility, if field_name
2291                          * was "cpu", then we treat this the same as
2292                          * common_cpu.
2293                          */
2294                         if (strcmp(field_name, "cpu") == 0) {
2295                                 *flags |= HIST_FIELD_FL_CPU;
2296                         } else {
2297                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2298                                          errpos(field_name));
2299                                 field = ERR_PTR(-EINVAL);
2300                                 goto out;
2301                         }
2302                 }
2303         }
2304  out:
2305         kfree(str);
2306
2307         return field;
2308 }
2309
2310 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2311                                        struct hist_field *var_ref,
2312                                        char *var_name)
2313 {
2314         struct hist_field *alias = NULL;
2315         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2316
2317         alias = create_hist_field(hist_data, NULL, flags, var_name);
2318         if (!alias)
2319                 return NULL;
2320
2321         alias->fn = var_ref->fn;
2322         alias->operands[0] = var_ref;
2323
2324         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2325                 destroy_hist_field(alias, 0);
2326                 return NULL;
2327         }
2328
2329         alias->var_ref_idx = var_ref->var_ref_idx;
2330
2331         return alias;
2332 }
2333
2334 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2335                                       char *str, char *var_name,
2336                                       unsigned long *flags)
2337 {
2338         struct trace_array *tr = hist_data->event_file->tr;
2339         struct hist_field *field = NULL;
2340         u64 constant;
2341
2342         if (kstrtoull(str, 0, &constant)) {
2343                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2344                 return NULL;
2345         }
2346
2347         *flags |= HIST_FIELD_FL_CONST;
2348         field = create_hist_field(hist_data, NULL, *flags, var_name);
2349         if (!field)
2350                 return NULL;
2351
2352         field->constant = constant;
2353
2354         return field;
2355 }
2356
2357 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2358                                      struct trace_event_file *file, char *str,
2359                                      unsigned long *flags, char *var_name)
2360 {
2361         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2362         struct ftrace_event_field *field = NULL;
2363         struct hist_field *hist_field = NULL;
2364         unsigned long buckets = 0;
2365         int ret = 0;
2366
2367         if (isdigit(str[0])) {
2368                 hist_field = parse_const(hist_data, str, var_name, flags);
2369                 if (!hist_field) {
2370                         ret = -EINVAL;
2371                         goto out;
2372                 }
2373                 return hist_field;
2374         }
2375
2376         s = strchr(str, '.');
2377         if (s) {
2378                 s = strchr(++s, '.');
2379                 if (s) {
2380                         ref_system = strsep(&str, ".");
2381                         if (!str) {
2382                                 ret = -EINVAL;
2383                                 goto out;
2384                         }
2385                         ref_event = strsep(&str, ".");
2386                         if (!str) {
2387                                 ret = -EINVAL;
2388                                 goto out;
2389                         }
2390                         ref_var = str;
2391                 }
2392         }
2393
2394         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2395         if (!s) {
2396                 hist_field = parse_var_ref(hist_data, ref_system,
2397                                            ref_event, ref_var);
2398                 if (hist_field) {
2399                         if (var_name) {
2400                                 hist_field = create_alias(hist_data, hist_field, var_name);
2401                                 if (!hist_field) {
2402                                         ret = -ENOMEM;
2403                                         goto out;
2404                                 }
2405                         }
2406                         return hist_field;
2407                 }
2408         } else
2409                 str = s;
2410
2411         field = parse_field(hist_data, file, str, flags, &buckets);
2412         if (IS_ERR(field)) {
2413                 ret = PTR_ERR(field);
2414                 goto out;
2415         }
2416
2417         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2418         if (!hist_field) {
2419                 ret = -ENOMEM;
2420                 goto out;
2421         }
2422         hist_field->buckets = buckets;
2423
2424         return hist_field;
2425  out:
2426         return ERR_PTR(ret);
2427 }
2428
2429 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2430                                      struct trace_event_file *file,
2431                                      char *str, unsigned long flags,
2432                                      char *var_name, unsigned int *n_subexprs);
2433
2434 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2435                                       struct trace_event_file *file,
2436                                       char *str, unsigned long flags,
2437                                       char *var_name, unsigned int *n_subexprs)
2438 {
2439         struct hist_field *operand1, *expr = NULL;
2440         unsigned long operand_flags;
2441         int ret = 0;
2442         char *s;
2443
2444         /* Unary minus operator, increment n_subexprs */
2445         ++*n_subexprs;
2446
2447         /* we support only -(xxx) i.e. explicit parens required */
2448
2449         if (*n_subexprs > 3) {
2450                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2451                 ret = -EINVAL;
2452                 goto free;
2453         }
2454
2455         str++; /* skip leading '-' */
2456
2457         s = strchr(str, '(');
2458         if (s)
2459                 str++;
2460         else {
2461                 ret = -EINVAL;
2462                 goto free;
2463         }
2464
2465         s = strrchr(str, ')');
2466         if (s) {
2467                  /* unary minus not supported in sub-expressions */
2468                 if (*(s+1) != '\0') {
2469                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2470                                  errpos(str));
2471                         ret = -EINVAL;
2472                         goto free;
2473                 }
2474                 *s = '\0';
2475         }
2476         else {
2477                 ret = -EINVAL; /* no closing ')' */
2478                 goto free;
2479         }
2480
2481         flags |= HIST_FIELD_FL_EXPR;
2482         expr = create_hist_field(hist_data, NULL, flags, var_name);
2483         if (!expr) {
2484                 ret = -ENOMEM;
2485                 goto free;
2486         }
2487
2488         operand_flags = 0;
2489         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2490         if (IS_ERR(operand1)) {
2491                 ret = PTR_ERR(operand1);
2492                 goto free;
2493         }
2494         if (operand1->flags & HIST_FIELD_FL_STRING) {
2495                 /* String type can not be the operand of unary operator. */
2496                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2497                 destroy_hist_field(operand1, 0);
2498                 ret = -EINVAL;
2499                 goto free;
2500         }
2501
2502         expr->flags |= operand1->flags &
2503                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2504         expr->fn = hist_field_unary_minus;
2505         expr->operands[0] = operand1;
2506         expr->operator = FIELD_OP_UNARY_MINUS;
2507         expr->name = expr_str(expr, 0);
2508         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2509         if (!expr->type) {
2510                 ret = -ENOMEM;
2511                 goto free;
2512         }
2513
2514         return expr;
2515  free:
2516         destroy_hist_field(expr, 0);
2517         return ERR_PTR(ret);
2518 }
2519
2520 /*
2521  * If the operands are var refs, return pointers the
2522  * variable(s) referenced in var1 and var2, else NULL.
2523  */
2524 static int check_expr_operands(struct trace_array *tr,
2525                                struct hist_field *operand1,
2526                                struct hist_field *operand2,
2527                                struct hist_field **var1,
2528                                struct hist_field **var2)
2529 {
2530         unsigned long operand1_flags = operand1->flags;
2531         unsigned long operand2_flags = operand2->flags;
2532
2533         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2534             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2535                 struct hist_field *var;
2536
2537                 var = find_var_field(operand1->var.hist_data, operand1->name);
2538                 if (!var)
2539                         return -EINVAL;
2540                 operand1_flags = var->flags;
2541                 *var1 = var;
2542         }
2543
2544         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2545             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2546                 struct hist_field *var;
2547
2548                 var = find_var_field(operand2->var.hist_data, operand2->name);
2549                 if (!var)
2550                         return -EINVAL;
2551                 operand2_flags = var->flags;
2552                 *var2 = var;
2553         }
2554
2555         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2556             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2557                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2558                 return -EINVAL;
2559         }
2560
2561         return 0;
2562 }
2563
2564 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2565                                      struct trace_event_file *file,
2566                                      char *str, unsigned long flags,
2567                                      char *var_name, unsigned int *n_subexprs)
2568 {
2569         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2570         struct hist_field *var1 = NULL, *var2 = NULL;
2571         unsigned long operand_flags, operand2_flags;
2572         int field_op, ret = -EINVAL;
2573         char *sep, *operand1_str;
2574         hist_field_fn_t op_fn;
2575         bool combine_consts;
2576
2577         if (*n_subexprs > 3) {
2578                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2579                 return ERR_PTR(-EINVAL);
2580         }
2581
2582         field_op = contains_operator(str, &sep);
2583
2584         if (field_op == FIELD_OP_NONE)
2585                 return parse_atom(hist_data, file, str, &flags, var_name);
2586
2587         if (field_op == FIELD_OP_UNARY_MINUS)
2588                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2589
2590         /* Binary operator found, increment n_subexprs */
2591         ++*n_subexprs;
2592
2593         /* Split the expression string at the root operator */
2594         if (!sep)
2595                 return ERR_PTR(-EINVAL);
2596
2597         *sep = '\0';
2598         operand1_str = str;
2599         str = sep+1;
2600
2601         /* Binary operator requires both operands */
2602         if (*operand1_str == '\0' || *str == '\0')
2603                 return ERR_PTR(-EINVAL);
2604
2605         operand_flags = 0;
2606
2607         /* LHS of string is an expression e.g. a+b in a+b+c */
2608         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2609         if (IS_ERR(operand1))
2610                 return ERR_CAST(operand1);
2611
2612         if (operand1->flags & HIST_FIELD_FL_STRING) {
2613                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2614                 ret = -EINVAL;
2615                 goto free_op1;
2616         }
2617
2618         /* RHS of string is another expression e.g. c in a+b+c */
2619         operand_flags = 0;
2620         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2621         if (IS_ERR(operand2)) {
2622                 ret = PTR_ERR(operand2);
2623                 goto free_op1;
2624         }
2625         if (operand2->flags & HIST_FIELD_FL_STRING) {
2626                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2627                 ret = -EINVAL;
2628                 goto free_operands;
2629         }
2630
2631         switch (field_op) {
2632         case FIELD_OP_MINUS:
2633                 op_fn = hist_field_minus;
2634                 break;
2635         case FIELD_OP_PLUS:
2636                 op_fn = hist_field_plus;
2637                 break;
2638         case FIELD_OP_DIV:
2639                 op_fn = hist_field_div;
2640                 break;
2641         case FIELD_OP_MULT:
2642                 op_fn = hist_field_mult;
2643                 break;
2644         default:
2645                 ret = -EINVAL;
2646                 goto free_operands;
2647         }
2648
2649         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2650         if (ret)
2651                 goto free_operands;
2652
2653         operand_flags = var1 ? var1->flags : operand1->flags;
2654         operand2_flags = var2 ? var2->flags : operand2->flags;
2655
2656         /*
2657          * If both operands are constant, the expression can be
2658          * collapsed to a single constant.
2659          */
2660         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2661
2662         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2663
2664         flags |= operand1->flags &
2665                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2666
2667         expr = create_hist_field(hist_data, NULL, flags, var_name);
2668         if (!expr) {
2669                 ret = -ENOMEM;
2670                 goto free_operands;
2671         }
2672
2673         operand1->read_once = true;
2674         operand2->read_once = true;
2675
2676         /* The operands are now owned and free'd by 'expr' */
2677         expr->operands[0] = operand1;
2678         expr->operands[1] = operand2;
2679
2680         if (field_op == FIELD_OP_DIV &&
2681                         operand2_flags & HIST_FIELD_FL_CONST) {
2682                 u64 divisor = var2 ? var2->constant : operand2->constant;
2683
2684                 if (!divisor) {
2685                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2686                         ret = -EDOM;
2687                         goto free_expr;
2688                 }
2689
2690                 /*
2691                  * Copy the divisor here so we don't have to look it up
2692                  * later if this is a var ref
2693                  */
2694                 operand2->constant = divisor;
2695                 op_fn = hist_field_get_div_fn(operand2);
2696         }
2697
2698         if (combine_consts) {
2699                 if (var1)
2700                         expr->operands[0] = var1;
2701                 if (var2)
2702                         expr->operands[1] = var2;
2703
2704                 expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2705
2706                 expr->operands[0] = NULL;
2707                 expr->operands[1] = NULL;
2708
2709                 /*
2710                  * var refs won't be destroyed immediately
2711                  * See: destroy_hist_field()
2712                  */
2713                 destroy_hist_field(operand2, 0);
2714                 destroy_hist_field(operand1, 0);
2715
2716                 expr->name = expr_str(expr, 0);
2717         } else {
2718                 expr->fn = op_fn;
2719
2720                 /* The operand sizes should be the same, so just pick one */
2721                 expr->size = operand1->size;
2722
2723                 expr->operator = field_op;
2724                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2725                 if (!expr->type) {
2726                         ret = -ENOMEM;
2727                         goto free_expr;
2728                 }
2729
2730                 expr->name = expr_str(expr, 0);
2731         }
2732
2733         return expr;
2734
2735 free_operands:
2736         destroy_hist_field(operand2, 0);
2737 free_op1:
2738         destroy_hist_field(operand1, 0);
2739         return ERR_PTR(ret);
2740
2741 free_expr:
2742         destroy_hist_field(expr, 0);
2743         return ERR_PTR(ret);
2744 }
2745
2746 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2747                                  struct trace_event_file *file)
2748 {
2749         struct event_trigger_data *test;
2750
2751         lockdep_assert_held(&event_mutex);
2752
2753         list_for_each_entry(test, &file->triggers, list) {
2754                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2755                         if (test->private_data == hist_data)
2756                                 return test->filter_str;
2757                 }
2758         }
2759
2760         return NULL;
2761 }
2762
2763 static struct event_command trigger_hist_cmd;
2764 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2765                                     struct trace_event_file *file,
2766                                     char *glob, char *cmd, char *param);
2767
2768 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2769                             struct hist_trigger_data *hist_data,
2770                             unsigned int n_keys)
2771 {
2772         struct hist_field *target_hist_field, *hist_field;
2773         unsigned int n, i, j;
2774
2775         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2776                 return false;
2777
2778         i = hist_data->n_vals;
2779         j = target_hist_data->n_vals;
2780
2781         for (n = 0; n < n_keys; n++) {
2782                 hist_field = hist_data->fields[i + n];
2783                 target_hist_field = target_hist_data->fields[j + n];
2784
2785                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2786                         return false;
2787                 if (hist_field->size != target_hist_field->size)
2788                         return false;
2789                 if (hist_field->is_signed != target_hist_field->is_signed)
2790                         return false;
2791         }
2792
2793         return true;
2794 }
2795
2796 static struct hist_trigger_data *
2797 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2798                      struct trace_event_file *file)
2799 {
2800         struct hist_trigger_data *hist_data;
2801         struct event_trigger_data *test;
2802         unsigned int n_keys;
2803
2804         lockdep_assert_held(&event_mutex);
2805
2806         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2807
2808         list_for_each_entry(test, &file->triggers, list) {
2809                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2810                         hist_data = test->private_data;
2811
2812                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2813                                 return hist_data;
2814                 }
2815         }
2816
2817         return NULL;
2818 }
2819
2820 static struct trace_event_file *event_file(struct trace_array *tr,
2821                                            char *system, char *event_name)
2822 {
2823         struct trace_event_file *file;
2824
2825         file = __find_event_file(tr, system, event_name);
2826         if (!file)
2827                 return ERR_PTR(-EINVAL);
2828
2829         return file;
2830 }
2831
2832 static struct hist_field *
2833 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2834                          char *system, char *event_name, char *field_name)
2835 {
2836         struct hist_field *event_var;
2837         char *synthetic_name;
2838
2839         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2840         if (!synthetic_name)
2841                 return ERR_PTR(-ENOMEM);
2842
2843         strcpy(synthetic_name, "synthetic_");
2844         strcat(synthetic_name, field_name);
2845
2846         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2847
2848         kfree(synthetic_name);
2849
2850         return event_var;
2851 }
2852
2853 /**
2854  * create_field_var_hist - Automatically create a histogram and var for a field
2855  * @target_hist_data: The target hist trigger
2856  * @subsys_name: Optional subsystem name
2857  * @event_name: Optional event name
2858  * @field_name: The name of the field (and the resulting variable)
2859  *
2860  * Hist trigger actions fetch data from variables, not directly from
2861  * events.  However, for convenience, users are allowed to directly
2862  * specify an event field in an action, which will be automatically
2863  * converted into a variable on their behalf.
2864  *
2865  * If a user specifies a field on an event that isn't the event the
2866  * histogram currently being defined (the target event histogram), the
2867  * only way that can be accomplished is if a new hist trigger is
2868  * created and the field variable defined on that.
2869  *
2870  * This function creates a new histogram compatible with the target
2871  * event (meaning a histogram with the same key as the target
2872  * histogram), and creates a variable for the specified field, but
2873  * with 'synthetic_' prepended to the variable name in order to avoid
2874  * collision with normal field variables.
2875  *
2876  * Return: The variable created for the field.
2877  */
2878 static struct hist_field *
2879 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2880                       char *subsys_name, char *event_name, char *field_name)
2881 {
2882         struct trace_array *tr = target_hist_data->event_file->tr;
2883         struct hist_trigger_data *hist_data;
2884         unsigned int i, n, first = true;
2885         struct field_var_hist *var_hist;
2886         struct trace_event_file *file;
2887         struct hist_field *key_field;
2888         struct hist_field *event_var;
2889         char *saved_filter;
2890         char *cmd;
2891         int ret;
2892
2893         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2894                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2895                 return ERR_PTR(-EINVAL);
2896         }
2897
2898         file = event_file(tr, subsys_name, event_name);
2899
2900         if (IS_ERR(file)) {
2901                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2902                 ret = PTR_ERR(file);
2903                 return ERR_PTR(ret);
2904         }
2905
2906         /*
2907          * Look for a histogram compatible with target.  We'll use the
2908          * found histogram specification to create a new matching
2909          * histogram with our variable on it.  target_hist_data is not
2910          * yet a registered histogram so we can't use that.
2911          */
2912         hist_data = find_compatible_hist(target_hist_data, file);
2913         if (!hist_data) {
2914                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2915                 return ERR_PTR(-EINVAL);
2916         }
2917
2918         /* See if a synthetic field variable has already been created */
2919         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2920                                              event_name, field_name);
2921         if (!IS_ERR_OR_NULL(event_var))
2922                 return event_var;
2923
2924         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2925         if (!var_hist)
2926                 return ERR_PTR(-ENOMEM);
2927
2928         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2929         if (!cmd) {
2930                 kfree(var_hist);
2931                 return ERR_PTR(-ENOMEM);
2932         }
2933
2934         /* Use the same keys as the compatible histogram */
2935         strcat(cmd, "keys=");
2936
2937         for_each_hist_key_field(i, hist_data) {
2938                 key_field = hist_data->fields[i];
2939                 if (!first)
2940                         strcat(cmd, ",");
2941                 strcat(cmd, key_field->field->name);
2942                 first = false;
2943         }
2944
2945         /* Create the synthetic field variable specification */
2946         strcat(cmd, ":synthetic_");
2947         strcat(cmd, field_name);
2948         strcat(cmd, "=");
2949         strcat(cmd, field_name);
2950
2951         /* Use the same filter as the compatible histogram */
2952         saved_filter = find_trigger_filter(hist_data, file);
2953         if (saved_filter) {
2954                 strcat(cmd, " if ");
2955                 strcat(cmd, saved_filter);
2956         }
2957
2958         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2959         if (!var_hist->cmd) {
2960                 kfree(cmd);
2961                 kfree(var_hist);
2962                 return ERR_PTR(-ENOMEM);
2963         }
2964
2965         /* Save the compatible histogram information */
2966         var_hist->hist_data = hist_data;
2967
2968         /* Create the new histogram with our variable */
2969         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
2970                                        "", "hist", cmd);
2971         if (ret) {
2972                 kfree(cmd);
2973                 kfree(var_hist->cmd);
2974                 kfree(var_hist);
2975                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2976                 return ERR_PTR(ret);
2977         }
2978
2979         kfree(cmd);
2980
2981         /* If we can't find the variable, something went wrong */
2982         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2983                                              event_name, field_name);
2984         if (IS_ERR_OR_NULL(event_var)) {
2985                 kfree(var_hist->cmd);
2986                 kfree(var_hist);
2987                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2988                 return ERR_PTR(-EINVAL);
2989         }
2990
2991         n = target_hist_data->n_field_var_hists;
2992         target_hist_data->field_var_hists[n] = var_hist;
2993         target_hist_data->n_field_var_hists++;
2994
2995         return event_var;
2996 }
2997
2998 static struct hist_field *
2999 find_target_event_var(struct hist_trigger_data *hist_data,
3000                       char *subsys_name, char *event_name, char *var_name)
3001 {
3002         struct trace_event_file *file = hist_data->event_file;
3003         struct hist_field *hist_field = NULL;
3004
3005         if (subsys_name) {
3006                 struct trace_event_call *call;
3007
3008                 if (!event_name)
3009                         return NULL;
3010
3011                 call = file->event_call;
3012
3013                 if (strcmp(subsys_name, call->class->system) != 0)
3014                         return NULL;
3015
3016                 if (strcmp(event_name, trace_event_name(call)) != 0)
3017                         return NULL;
3018         }
3019
3020         hist_field = find_var_field(hist_data, var_name);
3021
3022         return hist_field;
3023 }
3024
3025 static inline void __update_field_vars(struct tracing_map_elt *elt,
3026                                        struct trace_buffer *buffer,
3027                                        struct ring_buffer_event *rbe,
3028                                        void *rec,
3029                                        struct field_var **field_vars,
3030                                        unsigned int n_field_vars,
3031                                        unsigned int field_var_str_start)
3032 {
3033         struct hist_elt_data *elt_data = elt->private_data;
3034         unsigned int i, j, var_idx;
3035         u64 var_val;
3036
3037         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3038                 struct field_var *field_var = field_vars[i];
3039                 struct hist_field *var = field_var->var;
3040                 struct hist_field *val = field_var->val;
3041
3042                 var_val = val->fn(val, elt, buffer, rbe, rec);
3043                 var_idx = var->var.idx;
3044
3045                 if (val->flags & HIST_FIELD_FL_STRING) {
3046                         char *str = elt_data->field_var_str[j++];
3047                         char *val_str = (char *)(uintptr_t)var_val;
3048                         unsigned int size;
3049
3050                         size = min(val->size, STR_VAR_LEN_MAX);
3051                         strscpy(str, val_str, size);
3052                         var_val = (u64)(uintptr_t)str;
3053                 }
3054                 tracing_map_set_var(elt, var_idx, var_val);
3055         }
3056 }
3057
3058 static void update_field_vars(struct hist_trigger_data *hist_data,
3059                               struct tracing_map_elt *elt,
3060                               struct trace_buffer *buffer,
3061                               struct ring_buffer_event *rbe,
3062                               void *rec)
3063 {
3064         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3065                             hist_data->n_field_vars, 0);
3066 }
3067
3068 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3069                                  struct tracing_map_elt *elt,
3070                                  struct trace_buffer *buffer,  void *rec,
3071                                  struct ring_buffer_event *rbe, void *key,
3072                                  struct action_data *data, u64 *var_ref_vals)
3073 {
3074         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3075                             hist_data->n_save_vars, hist_data->n_field_var_str);
3076 }
3077
3078 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3079                                      struct trace_event_file *file,
3080                                      char *name, int size, const char *type)
3081 {
3082         struct hist_field *var;
3083         int idx;
3084
3085         if (find_var(hist_data, file, name) && !hist_data->remove) {
3086                 var = ERR_PTR(-EINVAL);
3087                 goto out;
3088         }
3089
3090         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3091         if (!var) {
3092                 var = ERR_PTR(-ENOMEM);
3093                 goto out;
3094         }
3095
3096         idx = tracing_map_add_var(hist_data->map);
3097         if (idx < 0) {
3098                 kfree(var);
3099                 var = ERR_PTR(-EINVAL);
3100                 goto out;
3101         }
3102
3103         var->ref = 1;
3104         var->flags = HIST_FIELD_FL_VAR;
3105         var->var.idx = idx;
3106         var->var.hist_data = var->hist_data = hist_data;
3107         var->size = size;
3108         var->var.name = kstrdup(name, GFP_KERNEL);
3109         var->type = kstrdup_const(type, GFP_KERNEL);
3110         if (!var->var.name || !var->type) {
3111                 kfree_const(var->type);
3112                 kfree(var->var.name);
3113                 kfree(var);
3114                 var = ERR_PTR(-ENOMEM);
3115         }
3116  out:
3117         return var;
3118 }
3119
3120 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3121                                           struct trace_event_file *file,
3122                                           char *field_name)
3123 {
3124         struct hist_field *val = NULL, *var = NULL;
3125         unsigned long flags = HIST_FIELD_FL_VAR;
3126         struct trace_array *tr = file->tr;
3127         struct field_var *field_var;
3128         int ret = 0;
3129
3130         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3131                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3132                 ret = -EINVAL;
3133                 goto err;
3134         }
3135
3136         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3137         if (IS_ERR(val)) {
3138                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3139                 ret = PTR_ERR(val);
3140                 goto err;
3141         }
3142
3143         var = create_var(hist_data, file, field_name, val->size, val->type);
3144         if (IS_ERR(var)) {
3145                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3146                 kfree(val);
3147                 ret = PTR_ERR(var);
3148                 goto err;
3149         }
3150
3151         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3152         if (!field_var) {
3153                 kfree(val);
3154                 kfree(var);
3155                 ret =  -ENOMEM;
3156                 goto err;
3157         }
3158
3159         field_var->var = var;
3160         field_var->val = val;
3161  out:
3162         return field_var;
3163  err:
3164         field_var = ERR_PTR(ret);
3165         goto out;
3166 }
3167
3168 /**
3169  * create_target_field_var - Automatically create a variable for a field
3170  * @target_hist_data: The target hist trigger
3171  * @subsys_name: Optional subsystem name
3172  * @event_name: Optional event name
3173  * @var_name: The name of the field (and the resulting variable)
3174  *
3175  * Hist trigger actions fetch data from variables, not directly from
3176  * events.  However, for convenience, users are allowed to directly
3177  * specify an event field in an action, which will be automatically
3178  * converted into a variable on their behalf.
3179
3180  * This function creates a field variable with the name var_name on
3181  * the hist trigger currently being defined on the target event.  If
3182  * subsys_name and event_name are specified, this function simply
3183  * verifies that they do in fact match the target event subsystem and
3184  * event name.
3185  *
3186  * Return: The variable created for the field.
3187  */
3188 static struct field_var *
3189 create_target_field_var(struct hist_trigger_data *target_hist_data,
3190                         char *subsys_name, char *event_name, char *var_name)
3191 {
3192         struct trace_event_file *file = target_hist_data->event_file;
3193
3194         if (subsys_name) {
3195                 struct trace_event_call *call;
3196
3197                 if (!event_name)
3198                         return NULL;
3199
3200                 call = file->event_call;
3201
3202                 if (strcmp(subsys_name, call->class->system) != 0)
3203                         return NULL;
3204
3205                 if (strcmp(event_name, trace_event_name(call)) != 0)
3206                         return NULL;
3207         }
3208
3209         return create_field_var(target_hist_data, file, var_name);
3210 }
3211
3212 static bool check_track_val_max(u64 track_val, u64 var_val)
3213 {
3214         if (var_val <= track_val)
3215                 return false;
3216
3217         return true;
3218 }
3219
3220 static bool check_track_val_changed(u64 track_val, u64 var_val)
3221 {
3222         if (var_val == track_val)
3223                 return false;
3224
3225         return true;
3226 }
3227
3228 static u64 get_track_val(struct hist_trigger_data *hist_data,
3229                          struct tracing_map_elt *elt,
3230                          struct action_data *data)
3231 {
3232         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3233         u64 track_val;
3234
3235         track_val = tracing_map_read_var(elt, track_var_idx);
3236
3237         return track_val;
3238 }
3239
3240 static void save_track_val(struct hist_trigger_data *hist_data,
3241                            struct tracing_map_elt *elt,
3242                            struct action_data *data, u64 var_val)
3243 {
3244         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3245
3246         tracing_map_set_var(elt, track_var_idx, var_val);
3247 }
3248
3249 static void save_track_data(struct hist_trigger_data *hist_data,
3250                             struct tracing_map_elt *elt,
3251                             struct trace_buffer *buffer, void *rec,
3252                             struct ring_buffer_event *rbe, void *key,
3253                             struct action_data *data, u64 *var_ref_vals)
3254 {
3255         if (data->track_data.save_data)
3256                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3257                                            key, data, var_ref_vals);
3258 }
3259
3260 static bool check_track_val(struct tracing_map_elt *elt,
3261                             struct action_data *data,
3262                             u64 var_val)
3263 {
3264         struct hist_trigger_data *hist_data;
3265         u64 track_val;
3266
3267         hist_data = data->track_data.track_var->hist_data;
3268         track_val = get_track_val(hist_data, elt, data);
3269
3270         return data->track_data.check_val(track_val, var_val);
3271 }
3272
3273 #ifdef CONFIG_TRACER_SNAPSHOT
3274 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3275 {
3276         /* called with tr->max_lock held */
3277         struct track_data *track_data = tr->cond_snapshot->cond_data;
3278         struct hist_elt_data *elt_data, *track_elt_data;
3279         struct snapshot_context *context = cond_data;
3280         struct action_data *action;
3281         u64 track_val;
3282
3283         if (!track_data)
3284                 return false;
3285
3286         action = track_data->action_data;
3287
3288         track_val = get_track_val(track_data->hist_data, context->elt,
3289                                   track_data->action_data);
3290
3291         if (!action->track_data.check_val(track_data->track_val, track_val))
3292                 return false;
3293
3294         track_data->track_val = track_val;
3295         memcpy(track_data->key, context->key, track_data->key_len);
3296
3297         elt_data = context->elt->private_data;
3298         track_elt_data = track_data->elt.private_data;
3299         if (elt_data->comm)
3300                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3301
3302         track_data->updated = true;
3303
3304         return true;
3305 }
3306
3307 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3308                                      struct tracing_map_elt *elt,
3309                                      struct trace_buffer *buffer, void *rec,
3310                                      struct ring_buffer_event *rbe, void *key,
3311                                      struct action_data *data,
3312                                      u64 *var_ref_vals)
3313 {
3314         struct trace_event_file *file = hist_data->event_file;
3315         struct snapshot_context context;
3316
3317         context.elt = elt;
3318         context.key = key;
3319
3320         tracing_snapshot_cond(file->tr, &context);
3321 }
3322
3323 static void hist_trigger_print_key(struct seq_file *m,
3324                                    struct hist_trigger_data *hist_data,
3325                                    void *key,
3326                                    struct tracing_map_elt *elt);
3327
3328 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3329 {
3330         unsigned int i;
3331
3332         if (!hist_data->n_actions)
3333                 return NULL;
3334
3335         for (i = 0; i < hist_data->n_actions; i++) {
3336                 struct action_data *data = hist_data->actions[i];
3337
3338                 if (data->action == ACTION_SNAPSHOT)
3339                         return data;
3340         }
3341
3342         return NULL;
3343 }
3344
3345 static void track_data_snapshot_print(struct seq_file *m,
3346                                       struct hist_trigger_data *hist_data)
3347 {
3348         struct trace_event_file *file = hist_data->event_file;
3349         struct track_data *track_data;
3350         struct action_data *action;
3351
3352         track_data = tracing_cond_snapshot_data(file->tr);
3353         if (!track_data)
3354                 return;
3355
3356         if (!track_data->updated)
3357                 return;
3358
3359         action = snapshot_action(hist_data);
3360         if (!action)
3361                 return;
3362
3363         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3364         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3365                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3366                    action->track_data.var_str, track_data->track_val);
3367
3368         seq_puts(m, "\ttriggered by event with key: ");
3369         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3370         seq_putc(m, '\n');
3371 }
3372 #else
3373 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3374 {
3375         return false;
3376 }
3377 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3378                                      struct tracing_map_elt *elt,
3379                                      struct trace_buffer *buffer, void *rec,
3380                                      struct ring_buffer_event *rbe, void *key,
3381                                      struct action_data *data,
3382                                      u64 *var_ref_vals) {}
3383 static void track_data_snapshot_print(struct seq_file *m,
3384                                       struct hist_trigger_data *hist_data) {}
3385 #endif /* CONFIG_TRACER_SNAPSHOT */
3386
3387 static void track_data_print(struct seq_file *m,
3388                              struct hist_trigger_data *hist_data,
3389                              struct tracing_map_elt *elt,
3390                              struct action_data *data)
3391 {
3392         u64 track_val = get_track_val(hist_data, elt, data);
3393         unsigned int i, save_var_idx;
3394
3395         if (data->handler == HANDLER_ONMAX)
3396                 seq_printf(m, "\n\tmax: %10llu", track_val);
3397         else if (data->handler == HANDLER_ONCHANGE)
3398                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3399
3400         if (data->action == ACTION_SNAPSHOT)
3401                 return;
3402
3403         for (i = 0; i < hist_data->n_save_vars; i++) {
3404                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3405                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3406                 u64 val;
3407
3408                 save_var_idx = save_var->var.idx;
3409
3410                 val = tracing_map_read_var(elt, save_var_idx);
3411
3412                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3413                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3414                                    (char *)(uintptr_t)(val));
3415                 } else
3416                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3417         }
3418 }
3419
3420 static void ontrack_action(struct hist_trigger_data *hist_data,
3421                            struct tracing_map_elt *elt,
3422                            struct trace_buffer *buffer, void *rec,
3423                            struct ring_buffer_event *rbe, void *key,
3424                            struct action_data *data, u64 *var_ref_vals)
3425 {
3426         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3427
3428         if (check_track_val(elt, data, var_val)) {
3429                 save_track_val(hist_data, elt, data, var_val);
3430                 save_track_data(hist_data, elt, buffer, rec, rbe,
3431                                 key, data, var_ref_vals);
3432         }
3433 }
3434
3435 static void action_data_destroy(struct action_data *data)
3436 {
3437         unsigned int i;
3438
3439         lockdep_assert_held(&event_mutex);
3440
3441         kfree(data->action_name);
3442
3443         for (i = 0; i < data->n_params; i++)
3444                 kfree(data->params[i]);
3445
3446         if (data->synth_event)
3447                 data->synth_event->ref--;
3448
3449         kfree(data->synth_event_name);
3450
3451         kfree(data);
3452 }
3453
3454 static void track_data_destroy(struct hist_trigger_data *hist_data,
3455                                struct action_data *data)
3456 {
3457         struct trace_event_file *file = hist_data->event_file;
3458
3459         destroy_hist_field(data->track_data.track_var, 0);
3460
3461         if (data->action == ACTION_SNAPSHOT) {
3462                 struct track_data *track_data;
3463
3464                 track_data = tracing_cond_snapshot_data(file->tr);
3465                 if (track_data && track_data->hist_data == hist_data) {
3466                         tracing_snapshot_cond_disable(file->tr);
3467                         track_data_free(track_data);
3468                 }
3469         }
3470
3471         kfree(data->track_data.var_str);
3472
3473         action_data_destroy(data);
3474 }
3475
3476 static int action_create(struct hist_trigger_data *hist_data,
3477                          struct action_data *data);
3478
3479 static int track_data_create(struct hist_trigger_data *hist_data,
3480                              struct action_data *data)
3481 {
3482         struct hist_field *var_field, *ref_field, *track_var = NULL;
3483         struct trace_event_file *file = hist_data->event_file;
3484         struct trace_array *tr = file->tr;
3485         char *track_data_var_str;
3486         int ret = 0;
3487
3488         track_data_var_str = data->track_data.var_str;
3489         if (track_data_var_str[0] != '$') {
3490                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3491                 return -EINVAL;
3492         }
3493         track_data_var_str++;
3494
3495         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3496         if (!var_field) {
3497                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3498                 return -EINVAL;
3499         }
3500
3501         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3502         if (!ref_field)
3503                 return -ENOMEM;
3504
3505         data->track_data.var_ref = ref_field;
3506
3507         if (data->handler == HANDLER_ONMAX)
3508                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3509         if (IS_ERR(track_var)) {
3510                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3511                 ret = PTR_ERR(track_var);
3512                 goto out;
3513         }
3514
3515         if (data->handler == HANDLER_ONCHANGE)
3516                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3517         if (IS_ERR(track_var)) {
3518                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3519                 ret = PTR_ERR(track_var);
3520                 goto out;
3521         }
3522         data->track_data.track_var = track_var;
3523
3524         ret = action_create(hist_data, data);
3525  out:
3526         return ret;
3527 }
3528
3529 static int parse_action_params(struct trace_array *tr, char *params,
3530                                struct action_data *data)
3531 {
3532         char *param, *saved_param;
3533         bool first_param = true;
3534         int ret = 0;
3535
3536         while (params) {
3537                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3538                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3539                         goto out;
3540                 }
3541
3542                 param = strsep(&params, ",");
3543                 if (!param) {
3544                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3545                         ret = -EINVAL;
3546                         goto out;
3547                 }
3548
3549                 param = strstrip(param);
3550                 if (strlen(param) < 2) {
3551                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3552                         ret = -EINVAL;
3553                         goto out;
3554                 }
3555
3556                 saved_param = kstrdup(param, GFP_KERNEL);
3557                 if (!saved_param) {
3558                         ret = -ENOMEM;
3559                         goto out;
3560                 }
3561
3562                 if (first_param && data->use_trace_keyword) {
3563                         data->synth_event_name = saved_param;
3564                         first_param = false;
3565                         continue;
3566                 }
3567                 first_param = false;
3568
3569                 data->params[data->n_params++] = saved_param;
3570         }
3571  out:
3572         return ret;
3573 }
3574
3575 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3576                         enum handler_id handler)
3577 {
3578         char *action_name;
3579         int ret = 0;
3580
3581         strsep(&str, ".");
3582         if (!str) {
3583                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3584                 ret = -EINVAL;
3585                 goto out;
3586         }
3587
3588         action_name = strsep(&str, "(");
3589         if (!action_name || !str) {
3590                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3591                 ret = -EINVAL;
3592                 goto out;
3593         }
3594
3595         if (str_has_prefix(action_name, "save")) {
3596                 char *params = strsep(&str, ")");
3597
3598                 if (!params) {
3599                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3600                         ret = -EINVAL;
3601                         goto out;
3602                 }
3603
3604                 ret = parse_action_params(tr, params, data);
3605                 if (ret)
3606                         goto out;
3607
3608                 if (handler == HANDLER_ONMAX)
3609                         data->track_data.check_val = check_track_val_max;
3610                 else if (handler == HANDLER_ONCHANGE)
3611                         data->track_data.check_val = check_track_val_changed;
3612                 else {
3613                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3614                         ret = -EINVAL;
3615                         goto out;
3616                 }
3617
3618                 data->track_data.save_data = save_track_data_vars;
3619                 data->fn = ontrack_action;
3620                 data->action = ACTION_SAVE;
3621         } else if (str_has_prefix(action_name, "snapshot")) {
3622                 char *params = strsep(&str, ")");
3623
3624                 if (!str) {
3625                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3626                         ret = -EINVAL;
3627                         goto out;
3628                 }
3629
3630                 if (handler == HANDLER_ONMAX)
3631                         data->track_data.check_val = check_track_val_max;
3632                 else if (handler == HANDLER_ONCHANGE)
3633                         data->track_data.check_val = check_track_val_changed;
3634                 else {
3635                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3636                         ret = -EINVAL;
3637                         goto out;
3638                 }
3639
3640                 data->track_data.save_data = save_track_data_snapshot;
3641                 data->fn = ontrack_action;
3642                 data->action = ACTION_SNAPSHOT;
3643         } else {
3644                 char *params = strsep(&str, ")");
3645
3646                 if (str_has_prefix(action_name, "trace"))
3647                         data->use_trace_keyword = true;
3648
3649                 if (params) {
3650                         ret = parse_action_params(tr, params, data);
3651                         if (ret)
3652                                 goto out;
3653                 }
3654
3655                 if (handler == HANDLER_ONMAX)
3656                         data->track_data.check_val = check_track_val_max;
3657                 else if (handler == HANDLER_ONCHANGE)
3658                         data->track_data.check_val = check_track_val_changed;
3659
3660                 if (handler != HANDLER_ONMATCH) {
3661                         data->track_data.save_data = action_trace;
3662                         data->fn = ontrack_action;
3663                 } else
3664                         data->fn = action_trace;
3665
3666                 data->action = ACTION_TRACE;
3667         }
3668
3669         data->action_name = kstrdup(action_name, GFP_KERNEL);
3670         if (!data->action_name) {
3671                 ret = -ENOMEM;
3672                 goto out;
3673         }
3674
3675         data->handler = handler;
3676  out:
3677         return ret;
3678 }
3679
3680 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3681                                             char *str, enum handler_id handler)
3682 {
3683         struct action_data *data;
3684         int ret = -EINVAL;
3685         char *var_str;
3686
3687         data = kzalloc(sizeof(*data), GFP_KERNEL);
3688         if (!data)
3689                 return ERR_PTR(-ENOMEM);
3690
3691         var_str = strsep(&str, ")");
3692         if (!var_str || !str) {
3693                 ret = -EINVAL;
3694                 goto free;
3695         }
3696
3697         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3698         if (!data->track_data.var_str) {
3699                 ret = -ENOMEM;
3700                 goto free;
3701         }
3702
3703         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3704         if (ret)
3705                 goto free;
3706  out:
3707         return data;
3708  free:
3709         track_data_destroy(hist_data, data);
3710         data = ERR_PTR(ret);
3711         goto out;
3712 }
3713
3714 static void onmatch_destroy(struct action_data *data)
3715 {
3716         kfree(data->match_data.event);
3717         kfree(data->match_data.event_system);
3718
3719         action_data_destroy(data);
3720 }
3721
3722 static void destroy_field_var(struct field_var *field_var)
3723 {
3724         if (!field_var)
3725                 return;
3726
3727         destroy_hist_field(field_var->var, 0);
3728         destroy_hist_field(field_var->val, 0);
3729
3730         kfree(field_var);
3731 }
3732
3733 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3734 {
3735         unsigned int i;
3736
3737         for (i = 0; i < hist_data->n_field_vars; i++)
3738                 destroy_field_var(hist_data->field_vars[i]);
3739
3740         for (i = 0; i < hist_data->n_save_vars; i++)
3741                 destroy_field_var(hist_data->save_vars[i]);
3742 }
3743
3744 static void save_field_var(struct hist_trigger_data *hist_data,
3745                            struct field_var *field_var)
3746 {
3747         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3748
3749         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3750                 hist_data->n_field_var_str++;
3751 }
3752
3753
3754 static int check_synth_field(struct synth_event *event,
3755                              struct hist_field *hist_field,
3756                              unsigned int field_pos)
3757 {
3758         struct synth_field *field;
3759
3760         if (field_pos >= event->n_fields)
3761                 return -EINVAL;
3762
3763         field = event->fields[field_pos];
3764
3765         /*
3766          * A dynamic string synth field can accept static or
3767          * dynamic. A static string synth field can only accept a
3768          * same-sized static string, which is checked for later.
3769          */
3770         if (strstr(hist_field->type, "char[") && field->is_string
3771             && field->is_dynamic)
3772                 return 0;
3773
3774         if (strcmp(field->type, hist_field->type) != 0) {
3775                 if (field->size != hist_field->size ||
3776                     (!field->is_string && field->is_signed != hist_field->is_signed))
3777                         return -EINVAL;
3778         }
3779
3780         return 0;
3781 }
3782
3783 static struct hist_field *
3784 trace_action_find_var(struct hist_trigger_data *hist_data,
3785                       struct action_data *data,
3786                       char *system, char *event, char *var)
3787 {
3788         struct trace_array *tr = hist_data->event_file->tr;
3789         struct hist_field *hist_field;
3790
3791         var++; /* skip '$' */
3792
3793         hist_field = find_target_event_var(hist_data, system, event, var);
3794         if (!hist_field) {
3795                 if (!system && data->handler == HANDLER_ONMATCH) {
3796                         system = data->match_data.event_system;
3797                         event = data->match_data.event;
3798                 }
3799
3800                 hist_field = find_event_var(hist_data, system, event, var);
3801         }
3802
3803         if (!hist_field)
3804                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3805
3806         return hist_field;
3807 }
3808
3809 static struct hist_field *
3810 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3811                               struct action_data *data, char *system,
3812                               char *event, char *var)
3813 {
3814         struct hist_field *hist_field = NULL;
3815         struct field_var *field_var;
3816
3817         /*
3818          * First try to create a field var on the target event (the
3819          * currently being defined).  This will create a variable for
3820          * unqualified fields on the target event, or if qualified,
3821          * target fields that have qualified names matching the target.
3822          */
3823         field_var = create_target_field_var(hist_data, system, event, var);
3824
3825         if (field_var && !IS_ERR(field_var)) {
3826                 save_field_var(hist_data, field_var);
3827                 hist_field = field_var->var;
3828         } else {
3829                 field_var = NULL;
3830                 /*
3831                  * If no explicit system.event is specified, default to
3832                  * looking for fields on the onmatch(system.event.xxx)
3833                  * event.
3834                  */
3835                 if (!system && data->handler == HANDLER_ONMATCH) {
3836                         system = data->match_data.event_system;
3837                         event = data->match_data.event;
3838                 }
3839
3840                 if (!event)
3841                         goto free;
3842                 /*
3843                  * At this point, we're looking at a field on another
3844                  * event.  Because we can't modify a hist trigger on
3845                  * another event to add a variable for a field, we need
3846                  * to create a new trigger on that event and create the
3847                  * variable at the same time.
3848                  */
3849                 hist_field = create_field_var_hist(hist_data, system, event, var);
3850                 if (IS_ERR(hist_field))
3851                         goto free;
3852         }
3853  out:
3854         return hist_field;
3855  free:
3856         destroy_field_var(field_var);
3857         hist_field = NULL;
3858         goto out;
3859 }
3860
3861 static int trace_action_create(struct hist_trigger_data *hist_data,
3862                                struct action_data *data)
3863 {
3864         struct trace_array *tr = hist_data->event_file->tr;
3865         char *event_name, *param, *system = NULL;
3866         struct hist_field *hist_field, *var_ref;
3867         unsigned int i;
3868         unsigned int field_pos = 0;
3869         struct synth_event *event;
3870         char *synth_event_name;
3871         int var_ref_idx, ret = 0;
3872
3873         lockdep_assert_held(&event_mutex);
3874
3875         if (data->use_trace_keyword)
3876                 synth_event_name = data->synth_event_name;
3877         else
3878                 synth_event_name = data->action_name;
3879
3880         event = find_synth_event(synth_event_name);
3881         if (!event) {
3882                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3883                 return -EINVAL;
3884         }
3885
3886         event->ref++;
3887
3888         for (i = 0; i < data->n_params; i++) {
3889                 char *p;
3890
3891                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3892                 if (!param) {
3893                         ret = -ENOMEM;
3894                         goto err;
3895                 }
3896
3897                 system = strsep(&param, ".");
3898                 if (!param) {
3899                         param = (char *)system;
3900                         system = event_name = NULL;
3901                 } else {
3902                         event_name = strsep(&param, ".");
3903                         if (!param) {
3904                                 kfree(p);
3905                                 ret = -EINVAL;
3906                                 goto err;
3907                         }
3908                 }
3909
3910                 if (param[0] == '$')
3911                         hist_field = trace_action_find_var(hist_data, data,
3912                                                            system, event_name,
3913                                                            param);
3914                 else
3915                         hist_field = trace_action_create_field_var(hist_data,
3916                                                                    data,
3917                                                                    system,
3918                                                                    event_name,
3919                                                                    param);
3920
3921                 if (!hist_field) {
3922                         kfree(p);
3923                         ret = -EINVAL;
3924                         goto err;
3925                 }
3926
3927                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3928                         var_ref = create_var_ref(hist_data, hist_field,
3929                                                  system, event_name);
3930                         if (!var_ref) {
3931                                 kfree(p);
3932                                 ret = -ENOMEM;
3933                                 goto err;
3934                         }
3935
3936                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3937                         if (WARN_ON(var_ref_idx < 0)) {
3938                                 kfree(p);
3939                                 ret = var_ref_idx;
3940                                 goto err;
3941                         }
3942
3943                         data->var_ref_idx[i] = var_ref_idx;
3944
3945                         field_pos++;
3946                         kfree(p);
3947                         continue;
3948                 }
3949
3950                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3951                 kfree(p);
3952                 ret = -EINVAL;
3953                 goto err;
3954         }
3955
3956         if (field_pos != event->n_fields) {
3957                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3958                 ret = -EINVAL;
3959                 goto err;
3960         }
3961
3962         data->synth_event = event;
3963  out:
3964         return ret;
3965  err:
3966         event->ref--;
3967
3968         goto out;
3969 }
3970
3971 static int action_create(struct hist_trigger_data *hist_data,
3972                          struct action_data *data)
3973 {
3974         struct trace_event_file *file = hist_data->event_file;
3975         struct trace_array *tr = file->tr;
3976         struct track_data *track_data;
3977         struct field_var *field_var;
3978         unsigned int i;
3979         char *param;
3980         int ret = 0;
3981
3982         if (data->action == ACTION_TRACE)
3983                 return trace_action_create(hist_data, data);
3984
3985         if (data->action == ACTION_SNAPSHOT) {
3986                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3987                 if (IS_ERR(track_data)) {
3988                         ret = PTR_ERR(track_data);
3989                         goto out;
3990                 }
3991
3992                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3993                                                    cond_snapshot_update);
3994                 if (ret)
3995                         track_data_free(track_data);
3996
3997                 goto out;
3998         }
3999
4000         if (data->action == ACTION_SAVE) {
4001                 if (hist_data->n_save_vars) {
4002                         ret = -EEXIST;
4003                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4004                         goto out;
4005                 }
4006
4007                 for (i = 0; i < data->n_params; i++) {
4008                         param = kstrdup(data->params[i], GFP_KERNEL);
4009                         if (!param) {
4010                                 ret = -ENOMEM;
4011                                 goto out;
4012                         }
4013
4014                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4015                         if (IS_ERR(field_var)) {
4016                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4017                                          errpos(param));
4018                                 ret = PTR_ERR(field_var);
4019                                 kfree(param);
4020                                 goto out;
4021                         }
4022
4023                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4024                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4025                                 hist_data->n_save_var_str++;
4026                         kfree(param);
4027                 }
4028         }
4029  out:
4030         return ret;
4031 }
4032
4033 static int onmatch_create(struct hist_trigger_data *hist_data,
4034                           struct action_data *data)
4035 {
4036         return action_create(hist_data, data);
4037 }
4038
4039 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4040 {
4041         char *match_event, *match_event_system;
4042         struct action_data *data;
4043         int ret = -EINVAL;
4044
4045         data = kzalloc(sizeof(*data), GFP_KERNEL);
4046         if (!data)
4047                 return ERR_PTR(-ENOMEM);
4048
4049         match_event = strsep(&str, ")");
4050         if (!match_event || !str) {
4051                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4052                 goto free;
4053         }
4054
4055         match_event_system = strsep(&match_event, ".");
4056         if (!match_event) {
4057                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4058                 goto free;
4059         }
4060
4061         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4062                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4063                 goto free;
4064         }
4065
4066         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4067         if (!data->match_data.event) {
4068                 ret = -ENOMEM;
4069                 goto free;
4070         }
4071
4072         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4073         if (!data->match_data.event_system) {
4074                 ret = -ENOMEM;
4075                 goto free;
4076         }
4077
4078         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4079         if (ret)
4080                 goto free;
4081  out:
4082         return data;
4083  free:
4084         onmatch_destroy(data);
4085         data = ERR_PTR(ret);
4086         goto out;
4087 }
4088
4089 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4090 {
4091         hist_data->fields[HITCOUNT_IDX] =
4092                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4093         if (!hist_data->fields[HITCOUNT_IDX])
4094                 return -ENOMEM;
4095
4096         hist_data->n_vals++;
4097         hist_data->n_fields++;
4098
4099         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4100                 return -EINVAL;
4101
4102         return 0;
4103 }
4104
4105 static int __create_val_field(struct hist_trigger_data *hist_data,
4106                               unsigned int val_idx,
4107                               struct trace_event_file *file,
4108                               char *var_name, char *field_str,
4109                               unsigned long flags)
4110 {
4111         struct hist_field *hist_field;
4112         int ret = 0, n_subexprs = 0;
4113
4114         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4115         if (IS_ERR(hist_field)) {
4116                 ret = PTR_ERR(hist_field);
4117                 goto out;
4118         }
4119
4120         hist_data->fields[val_idx] = hist_field;
4121
4122         ++hist_data->n_vals;
4123         ++hist_data->n_fields;
4124
4125         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4126                 ret = -EINVAL;
4127  out:
4128         return ret;
4129 }
4130
4131 static int create_val_field(struct hist_trigger_data *hist_data,
4132                             unsigned int val_idx,
4133                             struct trace_event_file *file,
4134                             char *field_str)
4135 {
4136         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4137                 return -EINVAL;
4138
4139         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4140 }
4141
4142 static const char *no_comm = "(no comm)";
4143
4144 static u64 hist_field_execname(struct hist_field *hist_field,
4145                                struct tracing_map_elt *elt,
4146                                struct trace_buffer *buffer,
4147                                struct ring_buffer_event *rbe,
4148                                void *event)
4149 {
4150         struct hist_elt_data *elt_data;
4151
4152         if (WARN_ON_ONCE(!elt))
4153                 return (u64)(unsigned long)no_comm;
4154
4155         elt_data = elt->private_data;
4156
4157         if (WARN_ON_ONCE(!elt_data->comm))
4158                 return (u64)(unsigned long)no_comm;
4159
4160         return (u64)(unsigned long)(elt_data->comm);
4161 }
4162
4163 /* Convert a var that points to common_pid.execname to a string */
4164 static void update_var_execname(struct hist_field *hist_field)
4165 {
4166         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4167                 HIST_FIELD_FL_EXECNAME;
4168         hist_field->size = MAX_FILTER_STR_VAL;
4169         hist_field->is_signed = 0;
4170
4171         kfree_const(hist_field->type);
4172         hist_field->type = "char[]";
4173
4174         hist_field->fn = hist_field_execname;
4175 }
4176
4177 static int create_var_field(struct hist_trigger_data *hist_data,
4178                             unsigned int val_idx,
4179                             struct trace_event_file *file,
4180                             char *var_name, char *expr_str)
4181 {
4182         struct trace_array *tr = hist_data->event_file->tr;
4183         unsigned long flags = 0;
4184         int ret;
4185
4186         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4187                 return -EINVAL;
4188
4189         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4190                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4191                 return -EINVAL;
4192         }
4193
4194         flags |= HIST_FIELD_FL_VAR;
4195         hist_data->n_vars++;
4196         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4197                 return -EINVAL;
4198
4199         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4200
4201         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4202                 update_var_execname(hist_data->fields[val_idx]);
4203
4204         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4205                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4206
4207         return ret;
4208 }
4209
4210 static int create_val_fields(struct hist_trigger_data *hist_data,
4211                              struct trace_event_file *file)
4212 {
4213         char *fields_str, *field_str;
4214         unsigned int i, j = 1;
4215         int ret;
4216
4217         ret = create_hitcount_val(hist_data);
4218         if (ret)
4219                 goto out;
4220
4221         fields_str = hist_data->attrs->vals_str;
4222         if (!fields_str)
4223                 goto out;
4224
4225         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4226                      j < TRACING_MAP_VALS_MAX; i++) {
4227                 field_str = strsep(&fields_str, ",");
4228                 if (!field_str)
4229                         break;
4230
4231                 if (strcmp(field_str, "hitcount") == 0)
4232                         continue;
4233
4234                 ret = create_val_field(hist_data, j++, file, field_str);
4235                 if (ret)
4236                         goto out;
4237         }
4238
4239         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4240                 ret = -EINVAL;
4241  out:
4242         return ret;
4243 }
4244
4245 static int create_key_field(struct hist_trigger_data *hist_data,
4246                             unsigned int key_idx,
4247                             unsigned int key_offset,
4248                             struct trace_event_file *file,
4249                             char *field_str)
4250 {
4251         struct trace_array *tr = hist_data->event_file->tr;
4252         struct hist_field *hist_field = NULL;
4253         unsigned long flags = 0;
4254         unsigned int key_size;
4255         int ret = 0, n_subexprs = 0;
4256
4257         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4258                 return -EINVAL;
4259
4260         flags |= HIST_FIELD_FL_KEY;
4261
4262         if (strcmp(field_str, "stacktrace") == 0) {
4263                 flags |= HIST_FIELD_FL_STACKTRACE;
4264                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4265                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4266         } else {
4267                 hist_field = parse_expr(hist_data, file, field_str, flags,
4268                                         NULL, &n_subexprs);
4269                 if (IS_ERR(hist_field)) {
4270                         ret = PTR_ERR(hist_field);
4271                         goto out;
4272                 }
4273
4274                 if (field_has_hist_vars(hist_field, 0)) {
4275                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4276                         destroy_hist_field(hist_field, 0);
4277                         ret = -EINVAL;
4278                         goto out;
4279                 }
4280
4281                 key_size = hist_field->size;
4282         }
4283
4284         hist_data->fields[key_idx] = hist_field;
4285
4286         key_size = ALIGN(key_size, sizeof(u64));
4287         hist_data->fields[key_idx]->size = key_size;
4288         hist_data->fields[key_idx]->offset = key_offset;
4289
4290         hist_data->key_size += key_size;
4291
4292         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4293                 ret = -EINVAL;
4294                 goto out;
4295         }
4296
4297         hist_data->n_keys++;
4298         hist_data->n_fields++;
4299
4300         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4301                 return -EINVAL;
4302
4303         ret = key_size;
4304  out:
4305         return ret;
4306 }
4307
4308 static int create_key_fields(struct hist_trigger_data *hist_data,
4309                              struct trace_event_file *file)
4310 {
4311         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4312         char *fields_str, *field_str;
4313         int ret = -EINVAL;
4314
4315         fields_str = hist_data->attrs->keys_str;
4316         if (!fields_str)
4317                 goto out;
4318
4319         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4320                 field_str = strsep(&fields_str, ",");
4321                 if (!field_str)
4322                         break;
4323                 ret = create_key_field(hist_data, i, key_offset,
4324                                        file, field_str);
4325                 if (ret < 0)
4326                         goto out;
4327                 key_offset += ret;
4328         }
4329         if (fields_str) {
4330                 ret = -EINVAL;
4331                 goto out;
4332         }
4333         ret = 0;
4334  out:
4335         return ret;
4336 }
4337
4338 static int create_var_fields(struct hist_trigger_data *hist_data,
4339                              struct trace_event_file *file)
4340 {
4341         unsigned int i, j = hist_data->n_vals;
4342         int ret = 0;
4343
4344         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4345
4346         for (i = 0; i < n_vars; i++) {
4347                 char *var_name = hist_data->attrs->var_defs.name[i];
4348                 char *expr = hist_data->attrs->var_defs.expr[i];
4349
4350                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4351                 if (ret)
4352                         goto out;
4353         }
4354  out:
4355         return ret;
4356 }
4357
4358 static void free_var_defs(struct hist_trigger_data *hist_data)
4359 {
4360         unsigned int i;
4361
4362         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4363                 kfree(hist_data->attrs->var_defs.name[i]);
4364                 kfree(hist_data->attrs->var_defs.expr[i]);
4365         }
4366
4367         hist_data->attrs->var_defs.n_vars = 0;
4368 }
4369
4370 static int parse_var_defs(struct hist_trigger_data *hist_data)
4371 {
4372         struct trace_array *tr = hist_data->event_file->tr;
4373         char *s, *str, *var_name, *field_str;
4374         unsigned int i, j, n_vars = 0;
4375         int ret = 0;
4376
4377         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4378                 str = hist_data->attrs->assignment_str[i];
4379                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4380                         field_str = strsep(&str, ",");
4381                         if (!field_str)
4382                                 break;
4383
4384                         var_name = strsep(&field_str, "=");
4385                         if (!var_name || !field_str) {
4386                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4387                                          errpos(var_name));
4388                                 ret = -EINVAL;
4389                                 goto free;
4390                         }
4391
4392                         if (n_vars == TRACING_MAP_VARS_MAX) {
4393                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4394                                 ret = -EINVAL;
4395                                 goto free;
4396                         }
4397
4398                         s = kstrdup(var_name, GFP_KERNEL);
4399                         if (!s) {
4400                                 ret = -ENOMEM;
4401                                 goto free;
4402                         }
4403                         hist_data->attrs->var_defs.name[n_vars] = s;
4404
4405                         s = kstrdup(field_str, GFP_KERNEL);
4406                         if (!s) {
4407                                 ret = -ENOMEM;
4408                                 goto free;
4409                         }
4410                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4411
4412                         hist_data->attrs->var_defs.n_vars = n_vars;
4413                 }
4414         }
4415
4416         return ret;
4417  free:
4418         free_var_defs(hist_data);
4419
4420         return ret;
4421 }
4422
4423 static int create_hist_fields(struct hist_trigger_data *hist_data,
4424                               struct trace_event_file *file)
4425 {
4426         int ret;
4427
4428         ret = parse_var_defs(hist_data);
4429         if (ret)
4430                 goto out;
4431
4432         ret = create_val_fields(hist_data, file);
4433         if (ret)
4434                 goto out;
4435
4436         ret = create_var_fields(hist_data, file);
4437         if (ret)
4438                 goto out;
4439
4440         ret = create_key_fields(hist_data, file);
4441         if (ret)
4442                 goto out;
4443  out:
4444         free_var_defs(hist_data);
4445
4446         return ret;
4447 }
4448
4449 static int is_descending(struct trace_array *tr, const char *str)
4450 {
4451         if (!str)
4452                 return 0;
4453
4454         if (strcmp(str, "descending") == 0)
4455                 return 1;
4456
4457         if (strcmp(str, "ascending") == 0)
4458                 return 0;
4459
4460         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4461
4462         return -EINVAL;
4463 }
4464
4465 static int create_sort_keys(struct hist_trigger_data *hist_data)
4466 {
4467         struct trace_array *tr = hist_data->event_file->tr;
4468         char *fields_str = hist_data->attrs->sort_key_str;
4469         struct tracing_map_sort_key *sort_key;
4470         int descending, ret = 0;
4471         unsigned int i, j, k;
4472
4473         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4474
4475         if (!fields_str)
4476                 goto out;
4477
4478         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4479                 struct hist_field *hist_field;
4480                 char *field_str, *field_name;
4481                 const char *test_name;
4482
4483                 sort_key = &hist_data->sort_keys[i];
4484
4485                 field_str = strsep(&fields_str, ",");
4486                 if (!field_str)
4487                         break;
4488
4489                 if (!*field_str) {
4490                         ret = -EINVAL;
4491                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4492                         break;
4493                 }
4494
4495                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4496                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4497                         ret = -EINVAL;
4498                         break;
4499                 }
4500
4501                 field_name = strsep(&field_str, ".");
4502                 if (!field_name || !*field_name) {
4503                         ret = -EINVAL;
4504                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4505                         break;
4506                 }
4507
4508                 if (strcmp(field_name, "hitcount") == 0) {
4509                         descending = is_descending(tr, field_str);
4510                         if (descending < 0) {
4511                                 ret = descending;
4512                                 break;
4513                         }
4514                         sort_key->descending = descending;
4515                         continue;
4516                 }
4517
4518                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4519                         unsigned int idx;
4520
4521                         hist_field = hist_data->fields[j];
4522                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4523                                 continue;
4524
4525                         idx = k++;
4526
4527                         test_name = hist_field_name(hist_field, 0);
4528
4529                         if (strcmp(field_name, test_name) == 0) {
4530                                 sort_key->field_idx = idx;
4531                                 descending = is_descending(tr, field_str);
4532                                 if (descending < 0) {
4533                                         ret = descending;
4534                                         goto out;
4535                                 }
4536                                 sort_key->descending = descending;
4537                                 break;
4538                         }
4539                 }
4540                 if (j == hist_data->n_fields) {
4541                         ret = -EINVAL;
4542                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4543                         break;
4544                 }
4545         }
4546
4547         hist_data->n_sort_keys = i;
4548  out:
4549         return ret;
4550 }
4551
4552 static void destroy_actions(struct hist_trigger_data *hist_data)
4553 {
4554         unsigned int i;
4555
4556         for (i = 0; i < hist_data->n_actions; i++) {
4557                 struct action_data *data = hist_data->actions[i];
4558
4559                 if (data->handler == HANDLER_ONMATCH)
4560                         onmatch_destroy(data);
4561                 else if (data->handler == HANDLER_ONMAX ||
4562                          data->handler == HANDLER_ONCHANGE)
4563                         track_data_destroy(hist_data, data);
4564                 else
4565                         kfree(data);
4566         }
4567 }
4568
4569 static int parse_actions(struct hist_trigger_data *hist_data)
4570 {
4571         struct trace_array *tr = hist_data->event_file->tr;
4572         struct action_data *data;
4573         unsigned int i;
4574         int ret = 0;
4575         char *str;
4576         int len;
4577
4578         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4579                 str = hist_data->attrs->action_str[i];
4580
4581                 if ((len = str_has_prefix(str, "onmatch("))) {
4582                         char *action_str = str + len;
4583
4584                         data = onmatch_parse(tr, action_str);
4585                         if (IS_ERR(data)) {
4586                                 ret = PTR_ERR(data);
4587                                 break;
4588                         }
4589                 } else if ((len = str_has_prefix(str, "onmax("))) {
4590                         char *action_str = str + len;
4591
4592                         data = track_data_parse(hist_data, action_str,
4593                                                 HANDLER_ONMAX);
4594                         if (IS_ERR(data)) {
4595                                 ret = PTR_ERR(data);
4596                                 break;
4597                         }
4598                 } else if ((len = str_has_prefix(str, "onchange("))) {
4599                         char *action_str = str + len;
4600
4601                         data = track_data_parse(hist_data, action_str,
4602                                                 HANDLER_ONCHANGE);
4603                         if (IS_ERR(data)) {
4604                                 ret = PTR_ERR(data);
4605                                 break;
4606                         }
4607                 } else {
4608                         ret = -EINVAL;
4609                         break;
4610                 }
4611
4612                 hist_data->actions[hist_data->n_actions++] = data;
4613         }
4614
4615         return ret;
4616 }
4617
4618 static int create_actions(struct hist_trigger_data *hist_data)
4619 {
4620         struct action_data *data;
4621         unsigned int i;
4622         int ret = 0;
4623
4624         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4625                 data = hist_data->actions[i];
4626
4627                 if (data->handler == HANDLER_ONMATCH) {
4628                         ret = onmatch_create(hist_data, data);
4629                         if (ret)
4630                                 break;
4631                 } else if (data->handler == HANDLER_ONMAX ||
4632                            data->handler == HANDLER_ONCHANGE) {
4633                         ret = track_data_create(hist_data, data);
4634                         if (ret)
4635                                 break;
4636                 } else {
4637                         ret = -EINVAL;
4638                         break;
4639                 }
4640         }
4641
4642         return ret;
4643 }
4644
4645 static void print_actions(struct seq_file *m,
4646                           struct hist_trigger_data *hist_data,
4647                           struct tracing_map_elt *elt)
4648 {
4649         unsigned int i;
4650
4651         for (i = 0; i < hist_data->n_actions; i++) {
4652                 struct action_data *data = hist_data->actions[i];
4653
4654                 if (data->action == ACTION_SNAPSHOT)
4655                         continue;
4656
4657                 if (data->handler == HANDLER_ONMAX ||
4658                     data->handler == HANDLER_ONCHANGE)
4659                         track_data_print(m, hist_data, elt, data);
4660         }
4661 }
4662
4663 static void print_action_spec(struct seq_file *m,
4664                               struct hist_trigger_data *hist_data,
4665                               struct action_data *data)
4666 {
4667         unsigned int i;
4668
4669         if (data->action == ACTION_SAVE) {
4670                 for (i = 0; i < hist_data->n_save_vars; i++) {
4671                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4672                         if (i < hist_data->n_save_vars - 1)
4673                                 seq_puts(m, ",");
4674                 }
4675         } else if (data->action == ACTION_TRACE) {
4676                 if (data->use_trace_keyword)
4677                         seq_printf(m, "%s", data->synth_event_name);
4678                 for (i = 0; i < data->n_params; i++) {
4679                         if (i || data->use_trace_keyword)
4680                                 seq_puts(m, ",");
4681                         seq_printf(m, "%s", data->params[i]);
4682                 }
4683         }
4684 }
4685
4686 static void print_track_data_spec(struct seq_file *m,
4687                                   struct hist_trigger_data *hist_data,
4688                                   struct action_data *data)
4689 {
4690         if (data->handler == HANDLER_ONMAX)
4691                 seq_puts(m, ":onmax(");
4692         else if (data->handler == HANDLER_ONCHANGE)
4693                 seq_puts(m, ":onchange(");
4694         seq_printf(m, "%s", data->track_data.var_str);
4695         seq_printf(m, ").%s(", data->action_name);
4696
4697         print_action_spec(m, hist_data, data);
4698
4699         seq_puts(m, ")");
4700 }
4701
4702 static void print_onmatch_spec(struct seq_file *m,
4703                                struct hist_trigger_data *hist_data,
4704                                struct action_data *data)
4705 {
4706         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4707                    data->match_data.event);
4708
4709         seq_printf(m, "%s(", data->action_name);
4710
4711         print_action_spec(m, hist_data, data);
4712
4713         seq_puts(m, ")");
4714 }
4715
4716 static bool actions_match(struct hist_trigger_data *hist_data,
4717                           struct hist_trigger_data *hist_data_test)
4718 {
4719         unsigned int i, j;
4720
4721         if (hist_data->n_actions != hist_data_test->n_actions)
4722                 return false;
4723
4724         for (i = 0; i < hist_data->n_actions; i++) {
4725                 struct action_data *data = hist_data->actions[i];
4726                 struct action_data *data_test = hist_data_test->actions[i];
4727                 char *action_name, *action_name_test;
4728
4729                 if (data->handler != data_test->handler)
4730                         return false;
4731                 if (data->action != data_test->action)
4732                         return false;
4733
4734                 if (data->n_params != data_test->n_params)
4735                         return false;
4736
4737                 for (j = 0; j < data->n_params; j++) {
4738                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4739                                 return false;
4740                 }
4741
4742                 if (data->use_trace_keyword)
4743                         action_name = data->synth_event_name;
4744                 else
4745                         action_name = data->action_name;
4746
4747                 if (data_test->use_trace_keyword)
4748                         action_name_test = data_test->synth_event_name;
4749                 else
4750                         action_name_test = data_test->action_name;
4751
4752                 if (strcmp(action_name, action_name_test) != 0)
4753                         return false;
4754
4755                 if (data->handler == HANDLER_ONMATCH) {
4756                         if (strcmp(data->match_data.event_system,
4757                                    data_test->match_data.event_system) != 0)
4758                                 return false;
4759                         if (strcmp(data->match_data.event,
4760                                    data_test->match_data.event) != 0)
4761                                 return false;
4762                 } else if (data->handler == HANDLER_ONMAX ||
4763                            data->handler == HANDLER_ONCHANGE) {
4764                         if (strcmp(data->track_data.var_str,
4765                                    data_test->track_data.var_str) != 0)
4766                                 return false;
4767                 }
4768         }
4769
4770         return true;
4771 }
4772
4773
4774 static void print_actions_spec(struct seq_file *m,
4775                                struct hist_trigger_data *hist_data)
4776 {
4777         unsigned int i;
4778
4779         for (i = 0; i < hist_data->n_actions; i++) {
4780                 struct action_data *data = hist_data->actions[i];
4781
4782                 if (data->handler == HANDLER_ONMATCH)
4783                         print_onmatch_spec(m, hist_data, data);
4784                 else if (data->handler == HANDLER_ONMAX ||
4785                          data->handler == HANDLER_ONCHANGE)
4786                         print_track_data_spec(m, hist_data, data);
4787         }
4788 }
4789
4790 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4791 {
4792         unsigned int i;
4793
4794         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4795                 kfree(hist_data->field_var_hists[i]->cmd);
4796                 kfree(hist_data->field_var_hists[i]);
4797         }
4798 }
4799
4800 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4801 {
4802         if (!hist_data)
4803                 return;
4804
4805         destroy_hist_trigger_attrs(hist_data->attrs);
4806         destroy_hist_fields(hist_data);
4807         tracing_map_destroy(hist_data->map);
4808
4809         destroy_actions(hist_data);
4810         destroy_field_vars(hist_data);
4811         destroy_field_var_hists(hist_data);
4812
4813         kfree(hist_data);
4814 }
4815
4816 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4817 {
4818         struct tracing_map *map = hist_data->map;
4819         struct ftrace_event_field *field;
4820         struct hist_field *hist_field;
4821         int i, idx = 0;
4822
4823         for_each_hist_field(i, hist_data) {
4824                 hist_field = hist_data->fields[i];
4825                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4826                         tracing_map_cmp_fn_t cmp_fn;
4827
4828                         field = hist_field->field;
4829
4830                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4831                                 cmp_fn = tracing_map_cmp_none;
4832                         else if (!field)
4833                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4834                                                              hist_field->is_signed);
4835                         else if (is_string_field(field))
4836                                 cmp_fn = tracing_map_cmp_string;
4837                         else
4838                                 cmp_fn = tracing_map_cmp_num(field->size,
4839                                                              field->is_signed);
4840                         idx = tracing_map_add_key_field(map,
4841                                                         hist_field->offset,
4842                                                         cmp_fn);
4843                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4844                         idx = tracing_map_add_sum_field(map);
4845
4846                 if (idx < 0)
4847                         return idx;
4848
4849                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4850                         idx = tracing_map_add_var(map);
4851                         if (idx < 0)
4852                                 return idx;
4853                         hist_field->var.idx = idx;
4854                         hist_field->var.hist_data = hist_data;
4855                 }
4856         }
4857
4858         return 0;
4859 }
4860
4861 static struct hist_trigger_data *
4862 create_hist_data(unsigned int map_bits,
4863                  struct hist_trigger_attrs *attrs,
4864                  struct trace_event_file *file,
4865                  bool remove)
4866 {
4867         const struct tracing_map_ops *map_ops = NULL;
4868         struct hist_trigger_data *hist_data;
4869         int ret = 0;
4870
4871         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4872         if (!hist_data)
4873                 return ERR_PTR(-ENOMEM);
4874
4875         hist_data->attrs = attrs;
4876         hist_data->remove = remove;
4877         hist_data->event_file = file;
4878
4879         ret = parse_actions(hist_data);
4880         if (ret)
4881                 goto free;
4882
4883         ret = create_hist_fields(hist_data, file);
4884         if (ret)
4885                 goto free;
4886
4887         ret = create_sort_keys(hist_data);
4888         if (ret)
4889                 goto free;
4890
4891         map_ops = &hist_trigger_elt_data_ops;
4892
4893         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4894                                             map_ops, hist_data);
4895         if (IS_ERR(hist_data->map)) {
4896                 ret = PTR_ERR(hist_data->map);
4897                 hist_data->map = NULL;
4898                 goto free;
4899         }
4900
4901         ret = create_tracing_map_fields(hist_data);
4902         if (ret)
4903                 goto free;
4904  out:
4905         return hist_data;
4906  free:
4907         hist_data->attrs = NULL;
4908
4909         destroy_hist_data(hist_data);
4910
4911         hist_data = ERR_PTR(ret);
4912
4913         goto out;
4914 }
4915
4916 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4917                                     struct tracing_map_elt *elt,
4918                                     struct trace_buffer *buffer, void *rec,
4919                                     struct ring_buffer_event *rbe,
4920                                     u64 *var_ref_vals)
4921 {
4922         struct hist_elt_data *elt_data;
4923         struct hist_field *hist_field;
4924         unsigned int i, var_idx;
4925         u64 hist_val;
4926
4927         elt_data = elt->private_data;
4928         elt_data->var_ref_vals = var_ref_vals;
4929
4930         for_each_hist_val_field(i, hist_data) {
4931                 hist_field = hist_data->fields[i];
4932                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4933                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4934                         var_idx = hist_field->var.idx;
4935
4936                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4937                                 unsigned int str_start, var_str_idx, idx;
4938                                 char *str, *val_str;
4939                                 unsigned int size;
4940
4941                                 str_start = hist_data->n_field_var_str +
4942                                         hist_data->n_save_var_str;
4943                                 var_str_idx = hist_field->var_str_idx;
4944                                 idx = str_start + var_str_idx;
4945
4946                                 str = elt_data->field_var_str[idx];
4947                                 val_str = (char *)(uintptr_t)hist_val;
4948
4949                                 size = min(hist_field->size, STR_VAR_LEN_MAX);
4950                                 strscpy(str, val_str, size);
4951
4952                                 hist_val = (u64)(uintptr_t)str;
4953                         }
4954                         tracing_map_set_var(elt, var_idx, hist_val);
4955                         continue;
4956                 }
4957                 tracing_map_update_sum(elt, i, hist_val);
4958         }
4959
4960         for_each_hist_key_field(i, hist_data) {
4961                 hist_field = hist_data->fields[i];
4962                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4963                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4964                         var_idx = hist_field->var.idx;
4965                         tracing_map_set_var(elt, var_idx, hist_val);
4966                 }
4967         }
4968
4969         update_field_vars(hist_data, elt, buffer, rbe, rec);
4970 }
4971
4972 static inline void add_to_key(char *compound_key, void *key,
4973                               struct hist_field *key_field, void *rec)
4974 {
4975         size_t size = key_field->size;
4976
4977         if (key_field->flags & HIST_FIELD_FL_STRING) {
4978                 struct ftrace_event_field *field;
4979
4980                 field = key_field->field;
4981                 if (field->filter_type == FILTER_DYN_STRING ||
4982                     field->filter_type == FILTER_RDYN_STRING)
4983                         size = *(u32 *)(rec + field->offset) >> 16;
4984                 else if (field->filter_type == FILTER_STATIC_STRING)
4985                         size = field->size;
4986
4987                 /* ensure NULL-termination */
4988                 if (size > key_field->size - 1)
4989                         size = key_field->size - 1;
4990
4991                 strncpy(compound_key + key_field->offset, (char *)key, size);
4992         } else
4993                 memcpy(compound_key + key_field->offset, key, size);
4994 }
4995
4996 static void
4997 hist_trigger_actions(struct hist_trigger_data *hist_data,
4998                      struct tracing_map_elt *elt,
4999                      struct trace_buffer *buffer, void *rec,
5000                      struct ring_buffer_event *rbe, void *key,
5001                      u64 *var_ref_vals)
5002 {
5003         struct action_data *data;
5004         unsigned int i;
5005
5006         for (i = 0; i < hist_data->n_actions; i++) {
5007                 data = hist_data->actions[i];
5008                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5009         }
5010 }
5011
5012 static void event_hist_trigger(struct event_trigger_data *data,
5013                                struct trace_buffer *buffer, void *rec,
5014                                struct ring_buffer_event *rbe)
5015 {
5016         struct hist_trigger_data *hist_data = data->private_data;
5017         bool use_compound_key = (hist_data->n_keys > 1);
5018         unsigned long entries[HIST_STACKTRACE_DEPTH];
5019         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5020         char compound_key[HIST_KEY_SIZE_MAX];
5021         struct tracing_map_elt *elt = NULL;
5022         struct hist_field *key_field;
5023         u64 field_contents;
5024         void *key = NULL;
5025         unsigned int i;
5026
5027         memset(compound_key, 0, hist_data->key_size);
5028
5029         for_each_hist_key_field(i, hist_data) {
5030                 key_field = hist_data->fields[i];
5031
5032                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5033                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5034                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5035                                          HIST_STACKTRACE_SKIP);
5036                         key = entries;
5037                 } else {
5038                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5039                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5040                                 key = (void *)(unsigned long)field_contents;
5041                                 use_compound_key = true;
5042                         } else
5043                                 key = (void *)&field_contents;
5044                 }
5045
5046                 if (use_compound_key)
5047                         add_to_key(compound_key, key, key_field, rec);
5048         }
5049
5050         if (use_compound_key)
5051                 key = compound_key;
5052
5053         if (hist_data->n_var_refs &&
5054             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5055                 return;
5056
5057         elt = tracing_map_insert(hist_data->map, key);
5058         if (!elt)
5059                 return;
5060
5061         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5062
5063         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5064                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5065 }
5066
5067 static void hist_trigger_stacktrace_print(struct seq_file *m,
5068                                           unsigned long *stacktrace_entries,
5069                                           unsigned int max_entries)
5070 {
5071         unsigned int spaces = 8;
5072         unsigned int i;
5073
5074         for (i = 0; i < max_entries; i++) {
5075                 if (!stacktrace_entries[i])
5076                         return;
5077
5078                 seq_printf(m, "%*c", 1 + spaces, ' ');
5079                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5080         }
5081 }
5082
5083 static void hist_trigger_print_key(struct seq_file *m,
5084                                    struct hist_trigger_data *hist_data,
5085                                    void *key,
5086                                    struct tracing_map_elt *elt)
5087 {
5088         struct hist_field *key_field;
5089         bool multiline = false;
5090         const char *field_name;
5091         unsigned int i;
5092         u64 uval;
5093
5094         seq_puts(m, "{ ");
5095
5096         for_each_hist_key_field(i, hist_data) {
5097                 key_field = hist_data->fields[i];
5098
5099                 if (i > hist_data->n_vals)
5100                         seq_puts(m, ", ");
5101
5102                 field_name = hist_field_name(key_field, 0);
5103
5104                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5105                         uval = *(u64 *)(key + key_field->offset);
5106                         seq_printf(m, "%s: %llx", field_name, uval);
5107                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5108                         uval = *(u64 *)(key + key_field->offset);
5109                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5110                                    uval, (void *)(uintptr_t)uval);
5111                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5112                         uval = *(u64 *)(key + key_field->offset);
5113                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5114                                    uval, (void *)(uintptr_t)uval);
5115                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5116                         struct hist_elt_data *elt_data = elt->private_data;
5117                         char *comm;
5118
5119                         if (WARN_ON_ONCE(!elt_data))
5120                                 return;
5121
5122                         comm = elt_data->comm;
5123
5124                         uval = *(u64 *)(key + key_field->offset);
5125                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5126                                    comm, uval);
5127                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5128                         const char *syscall_name;
5129
5130                         uval = *(u64 *)(key + key_field->offset);
5131                         syscall_name = get_syscall_name(uval);
5132                         if (!syscall_name)
5133                                 syscall_name = "unknown_syscall";
5134
5135                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5136                                    syscall_name, uval);
5137                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5138                         seq_puts(m, "stacktrace:\n");
5139                         hist_trigger_stacktrace_print(m,
5140                                                       key + key_field->offset,
5141                                                       HIST_STACKTRACE_DEPTH);
5142                         multiline = true;
5143                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5144                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5145                                    *(u64 *)(key + key_field->offset));
5146                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5147                         unsigned long buckets = key_field->buckets;
5148                         uval = *(u64 *)(key + key_field->offset);
5149                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5150                                    uval, uval + buckets -1);
5151                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5152                         seq_printf(m, "%s: %-50s", field_name,
5153                                    (char *)(key + key_field->offset));
5154                 } else {
5155                         uval = *(u64 *)(key + key_field->offset);
5156                         seq_printf(m, "%s: %10llu", field_name, uval);
5157                 }
5158         }
5159
5160         if (!multiline)
5161                 seq_puts(m, " ");
5162
5163         seq_puts(m, "}");
5164 }
5165
5166 static void hist_trigger_entry_print(struct seq_file *m,
5167                                      struct hist_trigger_data *hist_data,
5168                                      void *key,
5169                                      struct tracing_map_elt *elt)
5170 {
5171         const char *field_name;
5172         unsigned int i;
5173
5174         hist_trigger_print_key(m, hist_data, key, elt);
5175
5176         seq_printf(m, " hitcount: %10llu",
5177                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5178
5179         for (i = 1; i < hist_data->n_vals; i++) {
5180                 field_name = hist_field_name(hist_data->fields[i], 0);
5181
5182                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5183                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5184                         continue;
5185
5186                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5187                         seq_printf(m, "  %s: %10llx", field_name,
5188                                    tracing_map_read_sum(elt, i));
5189                 } else {
5190                         seq_printf(m, "  %s: %10llu", field_name,
5191                                    tracing_map_read_sum(elt, i));
5192                 }
5193         }
5194
5195         print_actions(m, hist_data, elt);
5196
5197         seq_puts(m, "\n");
5198 }
5199
5200 static int print_entries(struct seq_file *m,
5201                          struct hist_trigger_data *hist_data)
5202 {
5203         struct tracing_map_sort_entry **sort_entries = NULL;
5204         struct tracing_map *map = hist_data->map;
5205         int i, n_entries;
5206
5207         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5208                                              hist_data->n_sort_keys,
5209                                              &sort_entries);
5210         if (n_entries < 0)
5211                 return n_entries;
5212
5213         for (i = 0; i < n_entries; i++)
5214                 hist_trigger_entry_print(m, hist_data,
5215                                          sort_entries[i]->key,
5216                                          sort_entries[i]->elt);
5217
5218         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5219
5220         return n_entries;
5221 }
5222
5223 static void hist_trigger_show(struct seq_file *m,
5224                               struct event_trigger_data *data, int n)
5225 {
5226         struct hist_trigger_data *hist_data;
5227         int n_entries;
5228
5229         if (n > 0)
5230                 seq_puts(m, "\n\n");
5231
5232         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5233         data->ops->print(m, data->ops, data);
5234         seq_puts(m, "#\n\n");
5235
5236         hist_data = data->private_data;
5237         n_entries = print_entries(m, hist_data);
5238         if (n_entries < 0)
5239                 n_entries = 0;
5240
5241         track_data_snapshot_print(m, hist_data);
5242
5243         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5244                    (u64)atomic64_read(&hist_data->map->hits),
5245                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5246 }
5247
5248 static int hist_show(struct seq_file *m, void *v)
5249 {
5250         struct event_trigger_data *data;
5251         struct trace_event_file *event_file;
5252         int n = 0, ret = 0;
5253
5254         mutex_lock(&event_mutex);
5255
5256         event_file = event_file_data(m->private);
5257         if (unlikely(!event_file)) {
5258                 ret = -ENODEV;
5259                 goto out_unlock;
5260         }
5261
5262         list_for_each_entry(data, &event_file->triggers, list) {
5263                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5264                         hist_trigger_show(m, data, n++);
5265         }
5266
5267  out_unlock:
5268         mutex_unlock(&event_mutex);
5269
5270         return ret;
5271 }
5272
5273 static int event_hist_open(struct inode *inode, struct file *file)
5274 {
5275         int ret;
5276
5277         ret = security_locked_down(LOCKDOWN_TRACEFS);
5278         if (ret)
5279                 return ret;
5280
5281         return single_open(file, hist_show, file);
5282 }
5283
5284 const struct file_operations event_hist_fops = {
5285         .open = event_hist_open,
5286         .read = seq_read,
5287         .llseek = seq_lseek,
5288         .release = single_release,
5289 };
5290
5291 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5292 static void hist_field_debug_show_flags(struct seq_file *m,
5293                                         unsigned long flags)
5294 {
5295         seq_puts(m, "      flags:\n");
5296
5297         if (flags & HIST_FIELD_FL_KEY)
5298                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5299         else if (flags & HIST_FIELD_FL_HITCOUNT)
5300                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5301         else if (flags & HIST_FIELD_FL_VAR)
5302                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5303         else if (flags & HIST_FIELD_FL_VAR_REF)
5304                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5305         else
5306                 seq_puts(m, "        VAL: normal u64 value\n");
5307
5308         if (flags & HIST_FIELD_FL_ALIAS)
5309                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5310         else if (flags & HIST_FIELD_FL_CONST)
5311                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5312 }
5313
5314 static int hist_field_debug_show(struct seq_file *m,
5315                                  struct hist_field *field, unsigned long flags)
5316 {
5317         if ((field->flags & flags) != flags) {
5318                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5319                 return -EINVAL;
5320         }
5321
5322         hist_field_debug_show_flags(m, field->flags);
5323         if (field->field)
5324                 seq_printf(m, "      ftrace_event_field name: %s\n",
5325                            field->field->name);
5326
5327         if (field->flags & HIST_FIELD_FL_VAR) {
5328                 seq_printf(m, "      var.name: %s\n", field->var.name);
5329                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5330                            field->var.idx);
5331         }
5332
5333         if (field->flags & HIST_FIELD_FL_CONST)
5334                 seq_printf(m, "      constant: %llu\n", field->constant);
5335
5336         if (field->flags & HIST_FIELD_FL_ALIAS)
5337                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5338                            field->var_ref_idx);
5339
5340         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5341                 seq_printf(m, "      name: %s\n", field->name);
5342                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5343                            field->var.idx);
5344                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5345                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5346                            field->var_ref_idx);
5347                 if (field->system)
5348                         seq_printf(m, "      system: %s\n", field->system);
5349                 if (field->event_name)
5350                         seq_printf(m, "      event_name: %s\n", field->event_name);
5351         }
5352
5353         seq_printf(m, "      type: %s\n", field->type);
5354         seq_printf(m, "      size: %u\n", field->size);
5355         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5356
5357         return 0;
5358 }
5359
5360 static int field_var_debug_show(struct seq_file *m,
5361                                 struct field_var *field_var, unsigned int i,
5362                                 bool save_vars)
5363 {
5364         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5365         struct hist_field *field;
5366         int ret = 0;
5367
5368         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5369
5370         field = field_var->var;
5371
5372         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5373
5374         hist_field_debug_show_flags(m, field->flags);
5375         seq_printf(m, "      var.name: %s\n", field->var.name);
5376         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5377                    field->var.idx);
5378
5379         field = field_var->val;
5380
5381         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5382         if (field->field)
5383                 seq_printf(m, "      ftrace_event_field name: %s\n",
5384                            field->field->name);
5385         else {
5386                 ret = -EINVAL;
5387                 goto out;
5388         }
5389
5390         seq_printf(m, "      type: %s\n", field->type);
5391         seq_printf(m, "      size: %u\n", field->size);
5392         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5393 out:
5394         return ret;
5395 }
5396
5397 static int hist_action_debug_show(struct seq_file *m,
5398                                   struct action_data *data, int i)
5399 {
5400         int ret = 0;
5401
5402         if (data->handler == HANDLER_ONMAX ||
5403             data->handler == HANDLER_ONCHANGE) {
5404                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5405                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5406                                             HIST_FIELD_FL_VAR_REF);
5407                 if (ret)
5408                         goto out;
5409
5410                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5411                 ret = hist_field_debug_show(m, data->track_data.track_var,
5412                                             HIST_FIELD_FL_VAR);
5413                 if (ret)
5414                         goto out;
5415         }
5416
5417         if (data->handler == HANDLER_ONMATCH) {
5418                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5419                            i, data->match_data.event_system);
5420                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5421                            i, data->match_data.event);
5422         }
5423 out:
5424         return ret;
5425 }
5426
5427 static int hist_actions_debug_show(struct seq_file *m,
5428                                    struct hist_trigger_data *hist_data)
5429 {
5430         int i, ret = 0;
5431
5432         if (hist_data->n_actions)
5433                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5434
5435         for (i = 0; i < hist_data->n_actions; i++) {
5436                 struct action_data *action = hist_data->actions[i];
5437
5438                 ret = hist_action_debug_show(m, action, i);
5439                 if (ret)
5440                         goto out;
5441         }
5442
5443         if (hist_data->n_save_vars)
5444                 seq_puts(m, "\n  save action variables (save() params):\n");
5445
5446         for (i = 0; i < hist_data->n_save_vars; i++) {
5447                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5448                 if (ret)
5449                         goto out;
5450         }
5451 out:
5452         return ret;
5453 }
5454
5455 static void hist_trigger_debug_show(struct seq_file *m,
5456                                     struct event_trigger_data *data, int n)
5457 {
5458         struct hist_trigger_data *hist_data;
5459         int i, ret;
5460
5461         if (n > 0)
5462                 seq_puts(m, "\n\n");
5463
5464         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5465         data->ops->print(m, data->ops, data);
5466         seq_puts(m, "#\n\n");
5467
5468         hist_data = data->private_data;
5469
5470         seq_printf(m, "hist_data: %p\n\n", hist_data);
5471         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5472         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5473         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5474
5475         seq_puts(m, "\n  val fields:\n\n");
5476
5477         seq_puts(m, "    hist_data->fields[0]:\n");
5478         ret = hist_field_debug_show(m, hist_data->fields[0],
5479                                     HIST_FIELD_FL_HITCOUNT);
5480         if (ret)
5481                 return;
5482
5483         for (i = 1; i < hist_data->n_vals; i++) {
5484                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5485                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5486                 if (ret)
5487                         return;
5488         }
5489
5490         seq_puts(m, "\n  key fields:\n");
5491
5492         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5493                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5494                 ret = hist_field_debug_show(m, hist_data->fields[i],
5495                                             HIST_FIELD_FL_KEY);
5496                 if (ret)
5497                         return;
5498         }
5499
5500         if (hist_data->n_var_refs)
5501                 seq_puts(m, "\n  variable reference fields:\n");
5502
5503         for (i = 0; i < hist_data->n_var_refs; i++) {
5504                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5505                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5506                                             HIST_FIELD_FL_VAR_REF);
5507                 if (ret)
5508                         return;
5509         }
5510
5511         if (hist_data->n_field_vars)
5512                 seq_puts(m, "\n  field variables:\n");
5513
5514         for (i = 0; i < hist_data->n_field_vars; i++) {
5515                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5516                 if (ret)
5517                         return;
5518         }
5519
5520         ret = hist_actions_debug_show(m, hist_data);
5521         if (ret)
5522                 return;
5523 }
5524
5525 static int hist_debug_show(struct seq_file *m, void *v)
5526 {
5527         struct event_trigger_data *data;
5528         struct trace_event_file *event_file;
5529         int n = 0, ret = 0;
5530
5531         mutex_lock(&event_mutex);
5532
5533         event_file = event_file_data(m->private);
5534         if (unlikely(!event_file)) {
5535                 ret = -ENODEV;
5536                 goto out_unlock;
5537         }
5538
5539         list_for_each_entry(data, &event_file->triggers, list) {
5540                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5541                         hist_trigger_debug_show(m, data, n++);
5542         }
5543
5544  out_unlock:
5545         mutex_unlock(&event_mutex);
5546
5547         return ret;
5548 }
5549
5550 static int event_hist_debug_open(struct inode *inode, struct file *file)
5551 {
5552         int ret;
5553
5554         ret = security_locked_down(LOCKDOWN_TRACEFS);
5555         if (ret)
5556                 return ret;
5557
5558         return single_open(file, hist_debug_show, file);
5559 }
5560
5561 const struct file_operations event_hist_debug_fops = {
5562         .open = event_hist_debug_open,
5563         .read = seq_read,
5564         .llseek = seq_lseek,
5565         .release = single_release,
5566 };
5567 #endif
5568
5569 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5570 {
5571         const char *field_name = hist_field_name(hist_field, 0);
5572
5573         if (hist_field->var.name)
5574                 seq_printf(m, "%s=", hist_field->var.name);
5575
5576         if (hist_field->flags & HIST_FIELD_FL_CPU)
5577                 seq_puts(m, "common_cpu");
5578         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5579                 seq_printf(m, "%llu", hist_field->constant);
5580         else if (field_name) {
5581                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5582                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5583                         seq_putc(m, '$');
5584                 seq_printf(m, "%s", field_name);
5585         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5586                 seq_puts(m, "common_timestamp");
5587
5588         if (hist_field->flags) {
5589                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5590                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5591                         const char *flags = get_hist_field_flags(hist_field);
5592
5593                         if (flags)
5594                                 seq_printf(m, ".%s", flags);
5595                 }
5596         }
5597         if (hist_field->buckets)
5598                 seq_printf(m, "=%ld", hist_field->buckets);
5599 }
5600
5601 static int event_hist_trigger_print(struct seq_file *m,
5602                                     struct event_trigger_ops *ops,
5603                                     struct event_trigger_data *data)
5604 {
5605         struct hist_trigger_data *hist_data = data->private_data;
5606         struct hist_field *field;
5607         bool have_var = false;
5608         unsigned int i;
5609
5610         seq_puts(m, "hist:");
5611
5612         if (data->name)
5613                 seq_printf(m, "%s:", data->name);
5614
5615         seq_puts(m, "keys=");
5616
5617         for_each_hist_key_field(i, hist_data) {
5618                 field = hist_data->fields[i];
5619
5620                 if (i > hist_data->n_vals)
5621                         seq_puts(m, ",");
5622
5623                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5624                         seq_puts(m, "stacktrace");
5625                 else
5626                         hist_field_print(m, field);
5627         }
5628
5629         seq_puts(m, ":vals=");
5630
5631         for_each_hist_val_field(i, hist_data) {
5632                 field = hist_data->fields[i];
5633                 if (field->flags & HIST_FIELD_FL_VAR) {
5634                         have_var = true;
5635                         continue;
5636                 }
5637
5638                 if (i == HITCOUNT_IDX)
5639                         seq_puts(m, "hitcount");
5640                 else {
5641                         seq_puts(m, ",");
5642                         hist_field_print(m, field);
5643                 }
5644         }
5645
5646         if (have_var) {
5647                 unsigned int n = 0;
5648
5649                 seq_puts(m, ":");
5650
5651                 for_each_hist_val_field(i, hist_data) {
5652                         field = hist_data->fields[i];
5653
5654                         if (field->flags & HIST_FIELD_FL_VAR) {
5655                                 if (n++)
5656                                         seq_puts(m, ",");
5657                                 hist_field_print(m, field);
5658                         }
5659                 }
5660         }
5661
5662         seq_puts(m, ":sort=");
5663
5664         for (i = 0; i < hist_data->n_sort_keys; i++) {
5665                 struct tracing_map_sort_key *sort_key;
5666                 unsigned int idx, first_key_idx;
5667
5668                 /* skip VAR vals */
5669                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5670
5671                 sort_key = &hist_data->sort_keys[i];
5672                 idx = sort_key->field_idx;
5673
5674                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5675                         return -EINVAL;
5676
5677                 if (i > 0)
5678                         seq_puts(m, ",");
5679
5680                 if (idx == HITCOUNT_IDX)
5681                         seq_puts(m, "hitcount");
5682                 else {
5683                         if (idx >= first_key_idx)
5684                                 idx += hist_data->n_vars;
5685                         hist_field_print(m, hist_data->fields[idx]);
5686                 }
5687
5688                 if (sort_key->descending)
5689                         seq_puts(m, ".descending");
5690         }
5691         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5692         if (hist_data->enable_timestamps)
5693                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5694
5695         print_actions_spec(m, hist_data);
5696
5697         if (data->filter_str)
5698                 seq_printf(m, " if %s", data->filter_str);
5699
5700         if (data->paused)
5701                 seq_puts(m, " [paused]");
5702         else
5703                 seq_puts(m, " [active]");
5704
5705         seq_putc(m, '\n');
5706
5707         return 0;
5708 }
5709
5710 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5711                                    struct event_trigger_data *data)
5712 {
5713         struct hist_trigger_data *hist_data = data->private_data;
5714
5715         if (!data->ref && hist_data->attrs->name)
5716                 save_named_trigger(hist_data->attrs->name, data);
5717
5718         data->ref++;
5719
5720         return 0;
5721 }
5722
5723 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5724 {
5725         struct trace_event_file *file;
5726         unsigned int i;
5727         char *cmd;
5728         int ret;
5729
5730         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5731                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5732                 cmd = hist_data->field_var_hists[i]->cmd;
5733                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5734                                                "!hist", "hist", cmd);
5735                 WARN_ON_ONCE(ret < 0);
5736         }
5737 }
5738
5739 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5740                                     struct event_trigger_data *data)
5741 {
5742         struct hist_trigger_data *hist_data = data->private_data;
5743
5744         if (WARN_ON_ONCE(data->ref <= 0))
5745                 return;
5746
5747         data->ref--;
5748         if (!data->ref) {
5749                 if (data->name)
5750                         del_named_trigger(data);
5751
5752                 trigger_data_free(data);
5753
5754                 remove_hist_vars(hist_data);
5755
5756                 unregister_field_var_hists(hist_data);
5757
5758                 destroy_hist_data(hist_data);
5759         }
5760 }
5761
5762 static struct event_trigger_ops event_hist_trigger_ops = {
5763         .trigger                = event_hist_trigger,
5764         .print                  = event_hist_trigger_print,
5765         .init                   = event_hist_trigger_init,
5766         .free                   = event_hist_trigger_free,
5767 };
5768
5769 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5770                                          struct event_trigger_data *data)
5771 {
5772         data->ref++;
5773
5774         save_named_trigger(data->named_data->name, data);
5775
5776         event_hist_trigger_init(ops, data->named_data);
5777
5778         return 0;
5779 }
5780
5781 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5782                                           struct event_trigger_data *data)
5783 {
5784         if (WARN_ON_ONCE(data->ref <= 0))
5785                 return;
5786
5787         event_hist_trigger_free(ops, data->named_data);
5788
5789         data->ref--;
5790         if (!data->ref) {
5791                 del_named_trigger(data);
5792                 trigger_data_free(data);
5793         }
5794 }
5795
5796 static struct event_trigger_ops event_hist_trigger_named_ops = {
5797         .trigger                = event_hist_trigger,
5798         .print                  = event_hist_trigger_print,
5799         .init                   = event_hist_trigger_named_init,
5800         .free                   = event_hist_trigger_named_free,
5801 };
5802
5803 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5804                                                             char *param)
5805 {
5806         return &event_hist_trigger_ops;
5807 }
5808
5809 static void hist_clear(struct event_trigger_data *data)
5810 {
5811         struct hist_trigger_data *hist_data = data->private_data;
5812
5813         if (data->name)
5814                 pause_named_trigger(data);
5815
5816         tracepoint_synchronize_unregister();
5817
5818         tracing_map_clear(hist_data->map);
5819
5820         if (data->name)
5821                 unpause_named_trigger(data);
5822 }
5823
5824 static bool compatible_field(struct ftrace_event_field *field,
5825                              struct ftrace_event_field *test_field)
5826 {
5827         if (field == test_field)
5828                 return true;
5829         if (field == NULL || test_field == NULL)
5830                 return false;
5831         if (strcmp(field->name, test_field->name) != 0)
5832                 return false;
5833         if (strcmp(field->type, test_field->type) != 0)
5834                 return false;
5835         if (field->size != test_field->size)
5836                 return false;
5837         if (field->is_signed != test_field->is_signed)
5838                 return false;
5839
5840         return true;
5841 }
5842
5843 static bool hist_trigger_match(struct event_trigger_data *data,
5844                                struct event_trigger_data *data_test,
5845                                struct event_trigger_data *named_data,
5846                                bool ignore_filter)
5847 {
5848         struct tracing_map_sort_key *sort_key, *sort_key_test;
5849         struct hist_trigger_data *hist_data, *hist_data_test;
5850         struct hist_field *key_field, *key_field_test;
5851         unsigned int i;
5852
5853         if (named_data && (named_data != data_test) &&
5854             (named_data != data_test->named_data))
5855                 return false;
5856
5857         if (!named_data && is_named_trigger(data_test))
5858                 return false;
5859
5860         hist_data = data->private_data;
5861         hist_data_test = data_test->private_data;
5862
5863         if (hist_data->n_vals != hist_data_test->n_vals ||
5864             hist_data->n_fields != hist_data_test->n_fields ||
5865             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5866                 return false;
5867
5868         if (!ignore_filter) {
5869                 if ((data->filter_str && !data_test->filter_str) ||
5870                    (!data->filter_str && data_test->filter_str))
5871                         return false;
5872         }
5873
5874         for_each_hist_field(i, hist_data) {
5875                 key_field = hist_data->fields[i];
5876                 key_field_test = hist_data_test->fields[i];
5877
5878                 if (key_field->flags != key_field_test->flags)
5879                         return false;
5880                 if (!compatible_field(key_field->field, key_field_test->field))
5881                         return false;
5882                 if (key_field->offset != key_field_test->offset)
5883                         return false;
5884                 if (key_field->size != key_field_test->size)
5885                         return false;
5886                 if (key_field->is_signed != key_field_test->is_signed)
5887                         return false;
5888                 if (!!key_field->var.name != !!key_field_test->var.name)
5889                         return false;
5890                 if (key_field->var.name &&
5891                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5892                         return false;
5893         }
5894
5895         for (i = 0; i < hist_data->n_sort_keys; i++) {
5896                 sort_key = &hist_data->sort_keys[i];
5897                 sort_key_test = &hist_data_test->sort_keys[i];
5898
5899                 if (sort_key->field_idx != sort_key_test->field_idx ||
5900                     sort_key->descending != sort_key_test->descending)
5901                         return false;
5902         }
5903
5904         if (!ignore_filter && data->filter_str &&
5905             (strcmp(data->filter_str, data_test->filter_str) != 0))
5906                 return false;
5907
5908         if (!actions_match(hist_data, hist_data_test))
5909                 return false;
5910
5911         return true;
5912 }
5913
5914 static int hist_register_trigger(char *glob,
5915                                  struct event_trigger_data *data,
5916                                  struct trace_event_file *file)
5917 {
5918         struct hist_trigger_data *hist_data = data->private_data;
5919         struct event_trigger_data *test, *named_data = NULL;
5920         struct trace_array *tr = file->tr;
5921         int ret = 0;
5922
5923         if (hist_data->attrs->name) {
5924                 named_data = find_named_trigger(hist_data->attrs->name);
5925                 if (named_data) {
5926                         if (!hist_trigger_match(data, named_data, named_data,
5927                                                 true)) {
5928                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5929                                 ret = -EINVAL;
5930                                 goto out;
5931                         }
5932                 }
5933         }
5934
5935         if (hist_data->attrs->name && !named_data)
5936                 goto new;
5937
5938         lockdep_assert_held(&event_mutex);
5939
5940         list_for_each_entry(test, &file->triggers, list) {
5941                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5942                         if (!hist_trigger_match(data, test, named_data, false))
5943                                 continue;
5944                         if (hist_data->attrs->pause)
5945                                 test->paused = true;
5946                         else if (hist_data->attrs->cont)
5947                                 test->paused = false;
5948                         else if (hist_data->attrs->clear)
5949                                 hist_clear(test);
5950                         else {
5951                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5952                                 ret = -EEXIST;
5953                         }
5954                         goto out;
5955                 }
5956         }
5957  new:
5958         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5959                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5960                 ret = -ENOENT;
5961                 goto out;
5962         }
5963
5964         if (hist_data->attrs->pause)
5965                 data->paused = true;
5966
5967         if (named_data) {
5968                 data->private_data = named_data->private_data;
5969                 set_named_trigger_data(data, named_data);
5970                 data->ops = &event_hist_trigger_named_ops;
5971         }
5972
5973         if (data->ops->init) {
5974                 ret = data->ops->init(data->ops, data);
5975                 if (ret < 0)
5976                         goto out;
5977         }
5978
5979         if (hist_data->enable_timestamps) {
5980                 char *clock = hist_data->attrs->clock;
5981
5982                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5983                 if (ret) {
5984                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5985                         goto out;
5986                 }
5987
5988                 tracing_set_filter_buffering(file->tr, true);
5989         }
5990
5991         if (named_data)
5992                 destroy_hist_data(hist_data);
5993
5994         ret++;
5995  out:
5996         return ret;
5997 }
5998
5999 static int hist_trigger_enable(struct event_trigger_data *data,
6000                                struct trace_event_file *file)
6001 {
6002         int ret = 0;
6003
6004         list_add_tail_rcu(&data->list, &file->triggers);
6005
6006         update_cond_flag(file);
6007
6008         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6009                 list_del_rcu(&data->list);
6010                 update_cond_flag(file);
6011                 ret--;
6012         }
6013
6014         return ret;
6015 }
6016
6017 static bool have_hist_trigger_match(struct event_trigger_data *data,
6018                                     struct trace_event_file *file)
6019 {
6020         struct hist_trigger_data *hist_data = data->private_data;
6021         struct event_trigger_data *test, *named_data = NULL;
6022         bool match = false;
6023
6024         lockdep_assert_held(&event_mutex);
6025
6026         if (hist_data->attrs->name)
6027                 named_data = find_named_trigger(hist_data->attrs->name);
6028
6029         list_for_each_entry(test, &file->triggers, list) {
6030                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6031                         if (hist_trigger_match(data, test, named_data, false)) {
6032                                 match = true;
6033                                 break;
6034                         }
6035                 }
6036         }
6037
6038         return match;
6039 }
6040
6041 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6042                                     struct trace_event_file *file)
6043 {
6044         struct hist_trigger_data *hist_data = data->private_data;
6045         struct event_trigger_data *test, *named_data = NULL;
6046
6047         lockdep_assert_held(&event_mutex);
6048
6049         if (hist_data->attrs->name)
6050                 named_data = find_named_trigger(hist_data->attrs->name);
6051
6052         list_for_each_entry(test, &file->triggers, list) {
6053                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6054                         if (!hist_trigger_match(data, test, named_data, false))
6055                                 continue;
6056                         hist_data = test->private_data;
6057                         if (check_var_refs(hist_data))
6058                                 return true;
6059                         break;
6060                 }
6061         }
6062
6063         return false;
6064 }
6065
6066 static void hist_unregister_trigger(char *glob,
6067                                     struct event_trigger_data *data,
6068                                     struct trace_event_file *file)
6069 {
6070         struct hist_trigger_data *hist_data = data->private_data;
6071         struct event_trigger_data *test, *named_data = NULL;
6072         bool unregistered = false;
6073
6074         lockdep_assert_held(&event_mutex);
6075
6076         if (hist_data->attrs->name)
6077                 named_data = find_named_trigger(hist_data->attrs->name);
6078
6079         list_for_each_entry(test, &file->triggers, list) {
6080                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6081                         if (!hist_trigger_match(data, test, named_data, false))
6082                                 continue;
6083                         unregistered = true;
6084                         list_del_rcu(&test->list);
6085                         trace_event_trigger_enable_disable(file, 0);
6086                         update_cond_flag(file);
6087                         break;
6088                 }
6089         }
6090
6091         if (unregistered && test->ops->free)
6092                 test->ops->free(test->ops, test);
6093
6094         if (hist_data->enable_timestamps) {
6095                 if (!hist_data->remove || unregistered)
6096                         tracing_set_filter_buffering(file->tr, false);
6097         }
6098 }
6099
6100 static bool hist_file_check_refs(struct trace_event_file *file)
6101 {
6102         struct hist_trigger_data *hist_data;
6103         struct event_trigger_data *test;
6104
6105         lockdep_assert_held(&event_mutex);
6106
6107         list_for_each_entry(test, &file->triggers, list) {
6108                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6109                         hist_data = test->private_data;
6110                         if (check_var_refs(hist_data))
6111                                 return true;
6112                 }
6113         }
6114
6115         return false;
6116 }
6117
6118 static void hist_unreg_all(struct trace_event_file *file)
6119 {
6120         struct event_trigger_data *test, *n;
6121         struct hist_trigger_data *hist_data;
6122         struct synth_event *se;
6123         const char *se_name;
6124
6125         lockdep_assert_held(&event_mutex);
6126
6127         if (hist_file_check_refs(file))
6128                 return;
6129
6130         list_for_each_entry_safe(test, n, &file->triggers, list) {
6131                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6132                         hist_data = test->private_data;
6133                         list_del_rcu(&test->list);
6134                         trace_event_trigger_enable_disable(file, 0);
6135
6136                         se_name = trace_event_name(file->event_call);
6137                         se = find_synth_event(se_name);
6138                         if (se)
6139                                 se->ref--;
6140
6141                         update_cond_flag(file);
6142                         if (hist_data->enable_timestamps)
6143                                 tracing_set_filter_buffering(file->tr, false);
6144                         if (test->ops->free)
6145                                 test->ops->free(test->ops, test);
6146                 }
6147         }
6148 }
6149
6150 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6151                                     struct trace_event_file *file,
6152                                     char *glob, char *cmd, char *param)
6153 {
6154         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6155         struct event_trigger_data *trigger_data;
6156         struct hist_trigger_attrs *attrs;
6157         struct event_trigger_ops *trigger_ops;
6158         struct hist_trigger_data *hist_data;
6159         struct synth_event *se;
6160         const char *se_name;
6161         bool remove = false;
6162         char *trigger, *p, *start;
6163         int ret = 0;
6164
6165         lockdep_assert_held(&event_mutex);
6166
6167         WARN_ON(!glob);
6168
6169         if (strlen(glob)) {
6170                 hist_err_clear();
6171                 last_cmd_set(file, param);
6172         }
6173
6174         if (!param)
6175                 return -EINVAL;
6176
6177         if (glob[0] == '!')
6178                 remove = true;
6179
6180         /*
6181          * separate the trigger from the filter (k:v [if filter])
6182          * allowing for whitespace in the trigger
6183          */
6184         p = trigger = param;
6185         do {
6186                 p = strstr(p, "if");
6187                 if (!p)
6188                         break;
6189                 if (p == param)
6190                         return -EINVAL;
6191                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6192                         p++;
6193                         continue;
6194                 }
6195                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6196                         return -EINVAL;
6197                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6198                         p++;
6199                         continue;
6200                 }
6201                 break;
6202         } while (p);
6203
6204         if (!p)
6205                 param = NULL;
6206         else {
6207                 *(p - 1) = '\0';
6208                 param = strstrip(p);
6209                 trigger = strstrip(trigger);
6210         }
6211
6212         /*
6213          * To simplify arithmetic expression parsing, replace occurrences of
6214          * '.sym-offset' modifier with '.symXoffset'
6215          */
6216         start = strstr(trigger, ".sym-offset");
6217         while (start) {
6218                 *(start + 4) = 'X';
6219                 start = strstr(start + 11, ".sym-offset");
6220         }
6221
6222         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6223         if (IS_ERR(attrs))
6224                 return PTR_ERR(attrs);
6225
6226         if (attrs->map_bits)
6227                 hist_trigger_bits = attrs->map_bits;
6228
6229         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6230         if (IS_ERR(hist_data)) {
6231                 destroy_hist_trigger_attrs(attrs);
6232                 return PTR_ERR(hist_data);
6233         }
6234
6235         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6236
6237         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6238         if (!trigger_data) {
6239                 ret = -ENOMEM;
6240                 goto out_free;
6241         }
6242
6243         trigger_data->count = -1;
6244         trigger_data->ops = trigger_ops;
6245         trigger_data->cmd_ops = cmd_ops;
6246
6247         INIT_LIST_HEAD(&trigger_data->list);
6248         RCU_INIT_POINTER(trigger_data->filter, NULL);
6249
6250         trigger_data->private_data = hist_data;
6251
6252         /* if param is non-empty, it's supposed to be a filter */
6253         if (param && cmd_ops->set_filter) {
6254                 ret = cmd_ops->set_filter(param, trigger_data, file);
6255                 if (ret < 0)
6256                         goto out_free;
6257         }
6258
6259         if (remove) {
6260                 if (!have_hist_trigger_match(trigger_data, file))
6261                         goto out_free;
6262
6263                 if (hist_trigger_check_refs(trigger_data, file)) {
6264                         ret = -EBUSY;
6265                         goto out_free;
6266                 }
6267
6268                 cmd_ops->unreg(glob+1, trigger_data, file);
6269                 se_name = trace_event_name(file->event_call);
6270                 se = find_synth_event(se_name);
6271                 if (se)
6272                         se->ref--;
6273                 ret = 0;
6274                 goto out_free;
6275         }
6276
6277         ret = cmd_ops->reg(glob, trigger_data, file);
6278         /*
6279          * The above returns on success the # of triggers registered,
6280          * but if it didn't register any it returns zero.  Consider no
6281          * triggers registered a failure too.
6282          */
6283         if (!ret) {
6284                 if (!(attrs->pause || attrs->cont || attrs->clear))
6285                         ret = -ENOENT;
6286                 goto out_free;
6287         } else if (ret < 0)
6288                 goto out_free;
6289
6290         if (get_named_trigger_data(trigger_data))
6291                 goto enable;
6292
6293         if (has_hist_vars(hist_data))
6294                 save_hist_vars(hist_data);
6295
6296         ret = create_actions(hist_data);
6297         if (ret)
6298                 goto out_unreg;
6299
6300         ret = tracing_map_init(hist_data->map);
6301         if (ret)
6302                 goto out_unreg;
6303 enable:
6304         ret = hist_trigger_enable(trigger_data, file);
6305         if (ret)
6306                 goto out_unreg;
6307
6308         se_name = trace_event_name(file->event_call);
6309         se = find_synth_event(se_name);
6310         if (se)
6311                 se->ref++;
6312         /* Just return zero, not the number of registered triggers */
6313         ret = 0;
6314  out:
6315         if (ret == 0)
6316                 hist_err_clear();
6317
6318         return ret;
6319  out_unreg:
6320         cmd_ops->unreg(glob+1, trigger_data, file);
6321  out_free:
6322         if (cmd_ops->set_filter)
6323                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6324
6325         remove_hist_vars(hist_data);
6326
6327         kfree(trigger_data);
6328
6329         destroy_hist_data(hist_data);
6330         goto out;
6331 }
6332
6333 static struct event_command trigger_hist_cmd = {
6334         .name                   = "hist",
6335         .trigger_type           = ETT_EVENT_HIST,
6336         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6337         .parse                  = event_hist_trigger_parse,
6338         .reg                    = hist_register_trigger,
6339         .unreg                  = hist_unregister_trigger,
6340         .unreg_all              = hist_unreg_all,
6341         .get_trigger_ops        = event_hist_get_trigger_ops,
6342         .set_filter             = set_trigger_filter,
6343 };
6344
6345 __init int register_trigger_hist_cmd(void)
6346 {
6347         int ret;
6348
6349         ret = register_event_command(&trigger_hist_cmd);
6350         WARN_ON(ret < 0);
6351
6352         return ret;
6353 }
6354
6355 static void
6356 hist_enable_trigger(struct event_trigger_data *data,
6357                     struct trace_buffer *buffer,  void *rec,
6358                     struct ring_buffer_event *event)
6359 {
6360         struct enable_trigger_data *enable_data = data->private_data;
6361         struct event_trigger_data *test;
6362
6363         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6364                                 lockdep_is_held(&event_mutex)) {
6365                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6366                         if (enable_data->enable)
6367                                 test->paused = false;
6368                         else
6369                                 test->paused = true;
6370                 }
6371         }
6372 }
6373
6374 static void
6375 hist_enable_count_trigger(struct event_trigger_data *data,
6376                           struct trace_buffer *buffer,  void *rec,
6377                           struct ring_buffer_event *event)
6378 {
6379         if (!data->count)
6380                 return;
6381
6382         if (data->count != -1)
6383                 (data->count)--;
6384
6385         hist_enable_trigger(data, buffer, rec, event);
6386 }
6387
6388 static struct event_trigger_ops hist_enable_trigger_ops = {
6389         .trigger                = hist_enable_trigger,
6390         .print                  = event_enable_trigger_print,
6391         .init                   = event_trigger_init,
6392         .free                   = event_enable_trigger_free,
6393 };
6394
6395 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6396         .trigger                = hist_enable_count_trigger,
6397         .print                  = event_enable_trigger_print,
6398         .init                   = event_trigger_init,
6399         .free                   = event_enable_trigger_free,
6400 };
6401
6402 static struct event_trigger_ops hist_disable_trigger_ops = {
6403         .trigger                = hist_enable_trigger,
6404         .print                  = event_enable_trigger_print,
6405         .init                   = event_trigger_init,
6406         .free                   = event_enable_trigger_free,
6407 };
6408
6409 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6410         .trigger                = hist_enable_count_trigger,
6411         .print                  = event_enable_trigger_print,
6412         .init                   = event_trigger_init,
6413         .free                   = event_enable_trigger_free,
6414 };
6415
6416 static struct event_trigger_ops *
6417 hist_enable_get_trigger_ops(char *cmd, char *param)
6418 {
6419         struct event_trigger_ops *ops;
6420         bool enable;
6421
6422         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6423
6424         if (enable)
6425                 ops = param ? &hist_enable_count_trigger_ops :
6426                         &hist_enable_trigger_ops;
6427         else
6428                 ops = param ? &hist_disable_count_trigger_ops :
6429                         &hist_disable_trigger_ops;
6430
6431         return ops;
6432 }
6433
6434 static void hist_enable_unreg_all(struct trace_event_file *file)
6435 {
6436         struct event_trigger_data *test, *n;
6437
6438         list_for_each_entry_safe(test, n, &file->triggers, list) {
6439                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6440                         list_del_rcu(&test->list);
6441                         update_cond_flag(file);
6442                         trace_event_trigger_enable_disable(file, 0);
6443                         if (test->ops->free)
6444                                 test->ops->free(test->ops, test);
6445                 }
6446         }
6447 }
6448
6449 static struct event_command trigger_hist_enable_cmd = {
6450         .name                   = ENABLE_HIST_STR,
6451         .trigger_type           = ETT_HIST_ENABLE,
6452         .parse                  = event_enable_trigger_parse,
6453         .reg                    = event_enable_register_trigger,
6454         .unreg                  = event_enable_unregister_trigger,
6455         .unreg_all              = hist_enable_unreg_all,
6456         .get_trigger_ops        = hist_enable_get_trigger_ops,
6457         .set_filter             = set_trigger_filter,
6458 };
6459
6460 static struct event_command trigger_hist_disable_cmd = {
6461         .name                   = DISABLE_HIST_STR,
6462         .trigger_type           = ETT_HIST_ENABLE,
6463         .parse                  = event_enable_trigger_parse,
6464         .reg                    = event_enable_register_trigger,
6465         .unreg                  = event_enable_unregister_trigger,
6466         .unreg_all              = hist_enable_unreg_all,
6467         .get_trigger_ops        = hist_enable_get_trigger_ops,
6468         .set_filter             = set_trigger_filter,
6469 };
6470
6471 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6472 {
6473         unregister_event_command(&trigger_hist_enable_cmd);
6474         unregister_event_command(&trigger_hist_disable_cmd);
6475 }
6476
6477 __init int register_trigger_hist_enable_disable_cmds(void)
6478 {
6479         int ret;
6480
6481         ret = register_event_command(&trigger_hist_enable_cmd);
6482         if (WARN_ON(ret < 0))
6483                 return ret;
6484         ret = register_event_command(&trigger_hist_disable_cmd);
6485         if (WARN_ON(ret < 0))
6486                 unregister_trigger_hist_enable_disable_cmds();
6487
6488         return ret;
6489 }