kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),
  73
  74 #undef C
  75 #define C(a, b)         HIST_ERR_##a
  76
  77 enum { ERRORS };
  78
  79 #undef C
  80 #define C(a, b)         b
  81
  82 static const char *err_text[] = { ERRORS };
  83
  84 struct hist_field;
  85
  86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  87                                 struct tracing_map_elt *elt,
  88                                 struct trace_buffer *buffer,
  89                                 struct ring_buffer_event *rbe,
  90                                 void *event);
  91
  92 #define HIST_FIELD_OPERANDS_MAX 2
  93 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  94 #define HIST_ACTIONS_MAX        8
  95 #define HIST_CONST_DIGITS_MAX   21
  96 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  97
  98 enum field_op_id {
  99         FIELD_OP_NONE,
 100         FIELD_OP_PLUS,
 101         FIELD_OP_MINUS,
 102         FIELD_OP_UNARY_MINUS,
 103         FIELD_OP_DIV,
 104         FIELD_OP_MULT,
 105 };
 106
 107 /*
 108  * A hist_var (histogram variable) contains variable information for
 109  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 110  * flag set.  A hist_var has a variable name e.g. ts0, and is
 111  * associated with a given histogram trigger, as specified by
 112  * hist_data.  The hist_var idx is the unique index assigned to the
 113  * variable by the hist trigger's tracing_map.  The idx is what is
 114  * used to set a variable's value and, by a variable reference, to
 115  * retrieve it.
 116  */
 117 struct hist_var {
 118         char                            *name;
 119         struct hist_trigger_data        *hist_data;
 120         unsigned int                    idx;
 121 };
 122
 123 struct hist_field {
 124         struct ftrace_event_field       *field;
 125         unsigned long                   flags;
 126         hist_field_fn_t                 fn;
 127         unsigned int                    ref;
 128         unsigned int                    size;
 129         unsigned int                    offset;
 130         unsigned int                    is_signed;
 131         unsigned long                   buckets;
 132         const char                      *type;
 133         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 134         struct hist_trigger_data        *hist_data;
 135
 136         /*
 137          * Variable fields contain variable-specific info in var.
 138          */
 139         struct hist_var                 var;
 140         enum field_op_id                operator;
 141         char                            *system;
 142         char                            *event_name;
 143
 144         /*
 145          * The name field is used for EXPR and VAR_REF fields.  VAR
 146          * fields contain the variable name in var.name.
 147          */
 148         char                            *name;
 149
 150         /*
 151          * When a histogram trigger is hit, if it has any references
 152          * to variables, the values of those variables are collected
 153          * into a var_ref_vals array by resolve_var_refs().  The
 154          * current value of each variable is read from the tracing_map
 155          * using the hist field's hist_var.idx and entered into the
 156          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 157          */
 158         unsigned int                    var_ref_idx;
 159         bool                            read_once;
 160
 161         unsigned int                    var_str_idx;
 162
 163         /* Numeric literals are represented as u64 */
 164         u64                             constant;
 165         /* Used to optimize division by constants */
 166         u64                             div_multiplier;
 167 };
 168
 169 static u64 hist_field_none(struct hist_field *field,
 170                            struct tracing_map_elt *elt,
 171                            struct trace_buffer *buffer,
 172                            struct ring_buffer_event *rbe,
 173                            void *event)
 174 {
 175         return 0;
 176 }
 177
 178 static u64 hist_field_const(struct hist_field *field,
 179                            struct tracing_map_elt *elt,
 180                            struct trace_buffer *buffer,
 181                            struct ring_buffer_event *rbe,
 182                            void *event)
 183 {
 184         return field->constant;
 185 }
 186
 187 static u64 hist_field_counter(struct hist_field *field,
 188                               struct tracing_map_elt *elt,
 189                               struct trace_buffer *buffer,
 190                               struct ring_buffer_event *rbe,
 191                               void *event)
 192 {
 193         return 1;
 194 }
 195
 196 static u64 hist_field_string(struct hist_field *hist_field,
 197                              struct tracing_map_elt *elt,
 198                              struct trace_buffer *buffer,
 199                              struct ring_buffer_event *rbe,
 200                              void *event)
 201 {
 202         char *addr = (char *)(event + hist_field->field->offset);
 203
 204         return (u64)(unsigned long)addr;
 205 }
 206
 207 static u64 hist_field_dynstring(struct hist_field *hist_field,
 208                                 struct tracing_map_elt *elt,
 209                                 struct trace_buffer *buffer,
 210                                 struct ring_buffer_event *rbe,
 211                                 void *event)
 212 {
 213         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 214         int str_loc = str_item & 0xffff;
 215         char *addr = (char *)(event + str_loc);
 216
 217         return (u64)(unsigned long)addr;
 218 }
 219
 220 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 221                                    struct tracing_map_elt *elt,
 222                                    struct trace_buffer *buffer,
 223                                    struct ring_buffer_event *rbe,
 224                                    void *event)
 225 {
 226         u32 *item = event + hist_field->field->offset;
 227         u32 str_item = *item;
 228         int str_loc = str_item & 0xffff;
 229         char *addr = (char *)&item[1] + str_loc;
 230
 231         return (u64)(unsigned long)addr;
 232 }
 233
 234 static u64 hist_field_pstring(struct hist_field *hist_field,
 235                               struct tracing_map_elt *elt,
 236                               struct trace_buffer *buffer,
 237                               struct ring_buffer_event *rbe,
 238                               void *event)
 239 {
 240         char **addr = (char **)(event + hist_field->field->offset);
 241
 242         return (u64)(unsigned long)*addr;
 243 }
 244
 245 static u64 hist_field_log2(struct hist_field *hist_field,
 246                            struct tracing_map_elt *elt,
 247                            struct trace_buffer *buffer,
 248                            struct ring_buffer_event *rbe,
 249                            void *event)
 250 {
 251         struct hist_field *operand = hist_field->operands[0];
 252
 253         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 254
 255         return (u64) ilog2(roundup_pow_of_two(val));
 256 }
 257
 258 static u64 hist_field_bucket(struct hist_field *hist_field,
 259                              struct tracing_map_elt *elt,
 260                              struct trace_buffer *buffer,
 261                              struct ring_buffer_event *rbe,
 262                              void *event)
 263 {
 264         struct hist_field *operand = hist_field->operands[0];
 265         unsigned long buckets = hist_field->buckets;
 266
 267         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 268
 269         if (WARN_ON_ONCE(!buckets))
 270                 return val;
 271
 272         if (val >= LONG_MAX)
 273                 val = div64_ul(val, buckets);
 274         else
 275                 val = (u64)((unsigned long)val / buckets);
 276         return val * buckets;
 277 }
 278
 279 static u64 hist_field_plus(struct hist_field *hist_field,
 280                            struct tracing_map_elt *elt,
 281                            struct trace_buffer *buffer,
 282                            struct ring_buffer_event *rbe,
 283                            void *event)
 284 {
 285         struct hist_field *operand1 = hist_field->operands[0];
 286         struct hist_field *operand2 = hist_field->operands[1];
 287
 288         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 289         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 290
 291         return val1 + val2;
 292 }
 293
 294 static u64 hist_field_minus(struct hist_field *hist_field,
 295                             struct tracing_map_elt *elt,
 296                             struct trace_buffer *buffer,
 297                             struct ring_buffer_event *rbe,
 298                             void *event)
 299 {
 300         struct hist_field *operand1 = hist_field->operands[0];
 301         struct hist_field *operand2 = hist_field->operands[1];
 302
 303         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 304         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 305
 306         return val1 - val2;
 307 }
 308
 309 static u64 hist_field_div(struct hist_field *hist_field,
 310                            struct tracing_map_elt *elt,
 311                            struct trace_buffer *buffer,
 312                            struct ring_buffer_event *rbe,
 313                            void *event)
 314 {
 315         struct hist_field *operand1 = hist_field->operands[0];
 316         struct hist_field *operand2 = hist_field->operands[1];
 317
 318         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 319         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 320
 321         /* Return -1 for the undefined case */
 322         if (!val2)
 323                 return -1;
 324
 325         /* Use shift if the divisor is a power of 2 */
 326         if (!(val2 & (val2 - 1)))
 327                 return val1 >> __ffs64(val2);
 328
 329         return div64_u64(val1, val2);
 330 }
 331
 332 static u64 div_by_power_of_two(struct hist_field *hist_field,
 333                                 struct tracing_map_elt *elt,
 334                                 struct trace_buffer *buffer,
 335                                 struct ring_buffer_event *rbe,
 336                                 void *event)
 337 {
 338         struct hist_field *operand1 = hist_field->operands[0];
 339         struct hist_field *operand2 = hist_field->operands[1];
 340
 341         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 342
 343         return val1 >> __ffs64(operand2->constant);
 344 }
 345
 346 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 347                                 struct tracing_map_elt *elt,
 348                                 struct trace_buffer *buffer,
 349                                 struct ring_buffer_event *rbe,
 350                                 void *event)
 351 {
 352         struct hist_field *operand1 = hist_field->operands[0];
 353         struct hist_field *operand2 = hist_field->operands[1];
 354
 355         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 356
 357         return div64_u64(val1, operand2->constant);
 358 }
 359
 360 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 361                                 struct tracing_map_elt *elt,
 362                                 struct trace_buffer *buffer,
 363                                 struct ring_buffer_event *rbe,
 364                                 void *event)
 365 {
 366         struct hist_field *operand1 = hist_field->operands[0];
 367         struct hist_field *operand2 = hist_field->operands[1];
 368
 369         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 370
 371         /*
 372          * If the divisor is a constant, do a multiplication and shift instead.
 373          *
 374          * Choose Z = some power of 2. If Y <= Z, then:
 375          *     X / Y = (X * (Z / Y)) / Z
 376          *
 377          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 378          *     X / Y = (X * mult) / Z
 379          *
 380          * The division by Z can be replaced by a shift since Z is a power of 2:
 381          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 382          *
 383          * As long, as X < Z the results will not be off by more than 1.
 384          */
 385         if (val1 < (1 << HIST_DIV_SHIFT)) {
 386                 u64 mult = operand2->div_multiplier;
 387
 388                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 389         }
 390
 391         return div64_u64(val1, operand2->constant);
 392 }
 393
 394 static u64 hist_field_mult(struct hist_field *hist_field,
 395                            struct tracing_map_elt *elt,
 396                            struct trace_buffer *buffer,
 397                            struct ring_buffer_event *rbe,
 398                            void *event)
 399 {
 400         struct hist_field *operand1 = hist_field->operands[0];
 401         struct hist_field *operand2 = hist_field->operands[1];
 402
 403         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 404         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 405
 406         return val1 * val2;
 407 }
 408
 409 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 410                                   struct tracing_map_elt *elt,
 411                                   struct trace_buffer *buffer,
 412                                   struct ring_buffer_event *rbe,
 413                                   void *event)
 414 {
 415         struct hist_field *operand = hist_field->operands[0];
 416
 417         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 418         u64 val = (u64)-sval;
 419
 420         return val;
 421 }
 422
 423 #define DEFINE_HIST_FIELD_FN(type)                                      \
 424         static u64 hist_field_##type(struct hist_field *hist_field,     \
 425                                      struct tracing_map_elt *elt,       \
 426                                      struct trace_buffer *buffer,       \
 427                                      struct ring_buffer_event *rbe,     \
 428                                      void *event)                       \
 429 {                                                                       \
 430         type *addr = (type *)(event + hist_field->field->offset);       \
 431                                                                         \
 432         return (u64)(unsigned long)*addr;                               \
 433 }
 434
 435 DEFINE_HIST_FIELD_FN(s64);
 436 DEFINE_HIST_FIELD_FN(u64);
 437 DEFINE_HIST_FIELD_FN(s32);
 438 DEFINE_HIST_FIELD_FN(u32);
 439 DEFINE_HIST_FIELD_FN(s16);
 440 DEFINE_HIST_FIELD_FN(u16);
 441 DEFINE_HIST_FIELD_FN(s8);
 442 DEFINE_HIST_FIELD_FN(u8);
 443
 444 #define for_each_hist_field(i, hist_data)       \
 445         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 446
 447 #define for_each_hist_val_field(i, hist_data)   \
 448         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 449
 450 #define for_each_hist_key_field(i, hist_data)   \
 451         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 452
 453 #define HIST_STACKTRACE_DEPTH   16
 454 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 455 #define HIST_STACKTRACE_SKIP    5
 456
 457 #define HITCOUNT_IDX            0
 458 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 459
 460 enum hist_field_flags {
 461         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 462         HIST_FIELD_FL_KEY               = 1 << 1,
 463         HIST_FIELD_FL_STRING            = 1 << 2,
 464         HIST_FIELD_FL_HEX               = 1 << 3,
 465         HIST_FIELD_FL_SYM               = 1 << 4,
 466         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 467         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 468         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 469         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 470         HIST_FIELD_FL_LOG2              = 1 << 9,
 471         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 472         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 473         HIST_FIELD_FL_VAR               = 1 << 12,
 474         HIST_FIELD_FL_EXPR              = 1 << 13,
 475         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 476         HIST_FIELD_FL_CPU               = 1 << 15,
 477         HIST_FIELD_FL_ALIAS             = 1 << 16,
 478         HIST_FIELD_FL_BUCKET            = 1 << 17,
 479         HIST_FIELD_FL_CONST             = 1 << 18,
 480 };
 481
 482 struct var_defs {
 483         unsigned int    n_vars;
 484         char            *name[TRACING_MAP_VARS_MAX];
 485         char            *expr[TRACING_MAP_VARS_MAX];
 486 };
 487
 488 struct hist_trigger_attrs {
 489         char            *keys_str;
 490         char            *vals_str;
 491         char            *sort_key_str;
 492         char            *name;
 493         char            *clock;
 494         bool            pause;
 495         bool            cont;
 496         bool            clear;
 497         bool            ts_in_usecs;
 498         unsigned int    map_bits;
 499
 500         char            *assignment_str[TRACING_MAP_VARS_MAX];
 501         unsigned int    n_assignments;
 502
 503         char            *action_str[HIST_ACTIONS_MAX];
 504         unsigned int    n_actions;
 505
 506         struct var_defs var_defs;
 507 };
 508
 509 struct field_var {
 510         struct hist_field       *var;
 511         struct hist_field       *val;
 512 };
 513
 514 struct field_var_hist {
 515         struct hist_trigger_data        *hist_data;
 516         char                            *cmd;
 517 };
 518
 519 struct hist_trigger_data {
 520         struct hist_field               *fields[HIST_FIELDS_MAX];
 521         unsigned int                    n_vals;
 522         unsigned int                    n_keys;
 523         unsigned int                    n_fields;
 524         unsigned int                    n_vars;
 525         unsigned int                    n_var_str;
 526         unsigned int                    key_size;
 527         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 528         unsigned int                    n_sort_keys;
 529         struct trace_event_file         *event_file;
 530         struct hist_trigger_attrs       *attrs;
 531         struct tracing_map              *map;
 532         bool                            enable_timestamps;
 533         bool                            remove;
 534         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 535         unsigned int                    n_var_refs;
 536
 537         struct action_data              *actions[HIST_ACTIONS_MAX];
 538         unsigned int                    n_actions;
 539
 540         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 541         unsigned int                    n_field_vars;
 542         unsigned int                    n_field_var_str;
 543         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 544         unsigned int                    n_field_var_hists;
 545
 546         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 547         unsigned int                    n_save_vars;
 548         unsigned int                    n_save_var_str;
 549 };
 550
 551 struct action_data;
 552
 553 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 554                              struct tracing_map_elt *elt,
 555                              struct trace_buffer *buffer, void *rec,
 556                              struct ring_buffer_event *rbe, void *key,
 557                              struct action_data *data, u64 *var_ref_vals);
 558
 559 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 560
 561 enum handler_id {
 562         HANDLER_ONMATCH = 1,
 563         HANDLER_ONMAX,
 564         HANDLER_ONCHANGE,
 565 };
 566
 567 enum action_id {
 568         ACTION_SAVE = 1,
 569         ACTION_TRACE,
 570         ACTION_SNAPSHOT,
 571 };
 572
 573 struct action_data {
 574         enum handler_id         handler;
 575         enum action_id          action;
 576         char                    *action_name;
 577         action_fn_t             fn;
 578
 579         unsigned int            n_params;
 580         char                    *params[SYNTH_FIELDS_MAX];
 581
 582         /*
 583          * When a histogram trigger is hit, the values of any
 584          * references to variables, including variables being passed
 585          * as parameters to synthetic events, are collected into a
 586          * var_ref_vals array.  This var_ref_idx array is an array of
 587          * indices into the var_ref_vals array, one for each synthetic
 588          * event param, and is passed to the synthetic event
 589          * invocation.
 590          */
 591         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 592         struct synth_event      *synth_event;
 593         bool                    use_trace_keyword;
 594         char                    *synth_event_name;
 595
 596         union {
 597                 struct {
 598                         char                    *event;
 599                         char                    *event_system;
 600                 } match_data;
 601
 602                 struct {
 603                         /*
 604                          * var_str contains the $-unstripped variable
 605                          * name referenced by var_ref, and used when
 606                          * printing the action.  Because var_ref
 607                          * creation is deferred to create_actions(),
 608                          * we need a per-action way to save it until
 609                          * then, thus var_str.
 610                          */
 611                         char                    *var_str;
 612
 613                         /*
 614                          * var_ref refers to the variable being
 615                          * tracked e.g onmax($var).
 616                          */
 617                         struct hist_field       *var_ref;
 618
 619                         /*
 620                          * track_var contains the 'invisible' tracking
 621                          * variable created to keep the current
 622                          * e.g. max value.
 623                          */
 624                         struct hist_field       *track_var;
 625
 626                         check_track_val_fn_t    check_val;
 627                         action_fn_t             save_data;
 628                 } track_data;
 629         };
 630 };
 631
 632 struct track_data {
 633         u64                             track_val;
 634         bool                            updated;
 635
 636         unsigned int                    key_len;
 637         void                            *key;
 638         struct tracing_map_elt          elt;
 639
 640         struct action_data              *action_data;
 641         struct hist_trigger_data        *hist_data;
 642 };
 643
 644 struct hist_elt_data {
 645         char *comm;
 646         u64 *var_ref_vals;
 647         char **field_var_str;
 648         int n_field_var_str;
 649 };
 650
 651 struct snapshot_context {
 652         struct tracing_map_elt  *elt;
 653         void                    *key;
 654 };
 655
 656 /*
 657  * Returns the specific division function to use if the divisor
 658  * is constant. This avoids extra branches when the trigger is hit.
 659  */
 660 static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
 661 {
 662         u64 div = divisor->constant;
 663
 664         if (!(div & (div - 1)))
 665                 return div_by_power_of_two;
 666
 667         /* If the divisor is too large, do a regular division */
 668         if (div > (1 << HIST_DIV_SHIFT))
 669                 return div_by_not_power_of_two;
 670
 671         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 672         return div_by_mult_and_shift;
 673 }
 674
 675 static void track_data_free(struct track_data *track_data)
 676 {
 677         struct hist_elt_data *elt_data;
 678
 679         if (!track_data)
 680                 return;
 681
 682         kfree(track_data->key);
 683
 684         elt_data = track_data->elt.private_data;
 685         if (elt_data) {
 686                 kfree(elt_data->comm);
 687                 kfree(elt_data);
 688         }
 689
 690         kfree(track_data);
 691 }
 692
 693 static struct track_data *track_data_alloc(unsigned int key_len,
 694                                            struct action_data *action_data,
 695                                            struct hist_trigger_data *hist_data)
 696 {
 697         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 698         struct hist_elt_data *elt_data;
 699
 700         if (!data)
 701                 return ERR_PTR(-ENOMEM);
 702
 703         data->key = kzalloc(key_len, GFP_KERNEL);
 704         if (!data->key) {
 705                 track_data_free(data);
 706                 return ERR_PTR(-ENOMEM);
 707         }
 708
 709         data->key_len = key_len;
 710         data->action_data = action_data;
 711         data->hist_data = hist_data;
 712
 713         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 714         if (!elt_data) {
 715                 track_data_free(data);
 716                 return ERR_PTR(-ENOMEM);
 717         }
 718
 719         data->elt.private_data = elt_data;
 720
 721         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 722         if (!elt_data->comm) {
 723                 track_data_free(data);
 724                 return ERR_PTR(-ENOMEM);
 725         }
 726
 727         return data;
 728 }
 729
 730 static char last_cmd[MAX_FILTER_STR_VAL];
 731 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 732
 733 static int errpos(char *str)
 734 {
 735         return err_pos(last_cmd, str);
 736 }
 737
 738 static void last_cmd_set(struct trace_event_file *file, char *str)
 739 {
 740         const char *system = NULL, *name = NULL;
 741         struct trace_event_call *call;
 742
 743         if (!str)
 744                 return;
 745
 746         strcpy(last_cmd, "hist:");
 747         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 748
 749         if (file) {
 750                 call = file->event_call;
 751                 system = call->class->system;
 752                 if (system) {
 753                         name = trace_event_name(call);
 754                         if (!name)
 755                                 system = NULL;
 756                 }
 757         }
 758
 759         if (system)
 760                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 761 }
 762
 763 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 764 {
 765         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 766                         err_type, err_pos);
 767 }
 768
 769 static void hist_err_clear(void)
 770 {
 771         last_cmd[0] = '\0';
 772         last_cmd_loc[0] = '\0';
 773 }
 774
 775 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 776                                     unsigned int *var_ref_idx);
 777
 778 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 779                                unsigned int *var_ref_idx)
 780 {
 781         struct tracepoint *tp = event->tp;
 782
 783         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 784                 struct tracepoint_func *probe_func_ptr;
 785                 synth_probe_func_t probe_func;
 786                 void *__data;
 787
 788                 if (!(cpu_online(raw_smp_processor_id())))
 789                         return;
 790
 791                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 792                 if (probe_func_ptr) {
 793                         do {
 794                                 probe_func = probe_func_ptr->func;
 795                                 __data = probe_func_ptr->data;
 796                                 probe_func(__data, var_ref_vals, var_ref_idx);
 797                         } while ((++probe_func_ptr)->func);
 798                 }
 799         }
 800 }
 801
 802 static void action_trace(struct hist_trigger_data *hist_data,
 803                          struct tracing_map_elt *elt,
 804                          struct trace_buffer *buffer, void *rec,
 805                          struct ring_buffer_event *rbe, void *key,
 806                          struct action_data *data, u64 *var_ref_vals)
 807 {
 808         struct synth_event *event = data->synth_event;
 809
 810         trace_synth(event, var_ref_vals, data->var_ref_idx);
 811 }
 812
 813 struct hist_var_data {
 814         struct list_head list;
 815         struct hist_trigger_data *hist_data;
 816 };
 817
 818 static u64 hist_field_timestamp(struct hist_field *hist_field,
 819                                 struct tracing_map_elt *elt,
 820                                 struct trace_buffer *buffer,
 821                                 struct ring_buffer_event *rbe,
 822                                 void *event)
 823 {
 824         struct hist_trigger_data *hist_data = hist_field->hist_data;
 825         struct trace_array *tr = hist_data->event_file->tr;
 826
 827         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 828
 829         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 830                 ts = ns2usecs(ts);
 831
 832         return ts;
 833 }
 834
 835 static u64 hist_field_cpu(struct hist_field *hist_field,
 836                           struct tracing_map_elt *elt,
 837                           struct trace_buffer *buffer,
 838                           struct ring_buffer_event *rbe,
 839                           void *event)
 840 {
 841         int cpu = smp_processor_id();
 842
 843         return cpu;
 844 }
 845
 846 /**
 847  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 848  * @hist_field: The VAR_REF field to check
 849  * @var_data: The hist trigger that owns the variable
 850  * @var_idx: The trigger variable identifier
 851  *
 852  * Check the given VAR_REF field to see whether or not it references
 853  * the given variable associated with the given trigger.
 854  *
 855  * Return: The VAR_REF field if it does reference the variable, NULL if not
 856  */
 857 static struct hist_field *
 858 check_field_for_var_ref(struct hist_field *hist_field,
 859                         struct hist_trigger_data *var_data,
 860                         unsigned int var_idx)
 861 {
 862         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 863
 864         if (hist_field && hist_field->var.idx == var_idx &&
 865             hist_field->var.hist_data == var_data)
 866                 return hist_field;
 867
 868         return NULL;
 869 }
 870
 871 /**
 872  * find_var_ref - Check if a trigger has a reference to a trigger variable
 873  * @hist_data: The hist trigger that might have a reference to the variable
 874  * @var_data: The hist trigger that owns the variable
 875  * @var_idx: The trigger variable identifier
 876  *
 877  * Check the list of var_refs[] on the first hist trigger to see
 878  * whether any of them are references to the variable on the second
 879  * trigger.
 880  *
 881  * Return: The VAR_REF field referencing the variable if so, NULL if not
 882  */
 883 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 884                                        struct hist_trigger_data *var_data,
 885                                        unsigned int var_idx)
 886 {
 887         struct hist_field *hist_field;
 888         unsigned int i;
 889
 890         for (i = 0; i < hist_data->n_var_refs; i++) {
 891                 hist_field = hist_data->var_refs[i];
 892                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 893                         return hist_field;
 894         }
 895
 896         return NULL;
 897 }
 898
 899 /**
 900  * find_any_var_ref - Check if there is a reference to a given trigger variable
 901  * @hist_data: The hist trigger
 902  * @var_idx: The trigger variable identifier
 903  *
 904  * Check to see whether the given variable is currently referenced by
 905  * any other trigger.
 906  *
 907  * The trigger the variable is defined on is explicitly excluded - the
 908  * assumption being that a self-reference doesn't prevent a trigger
 909  * from being removed.
 910  *
 911  * Return: The VAR_REF field referencing the variable if so, NULL if not
 912  */
 913 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 914                                            unsigned int var_idx)
 915 {
 916         struct trace_array *tr = hist_data->event_file->tr;
 917         struct hist_field *found = NULL;
 918         struct hist_var_data *var_data;
 919
 920         list_for_each_entry(var_data, &tr->hist_vars, list) {
 921                 if (var_data->hist_data == hist_data)
 922                         continue;
 923                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 924                 if (found)
 925                         break;
 926         }
 927
 928         return found;
 929 }
 930
 931 /**
 932  * check_var_refs - Check if there is a reference to any of trigger's variables
 933  * @hist_data: The hist trigger
 934  *
 935  * A trigger can define one or more variables.  If any one of them is
 936  * currently referenced by any other trigger, this function will
 937  * determine that.
 938
 939  * Typically used to determine whether or not a trigger can be removed
 940  * - if there are any references to a trigger's variables, it cannot.
 941  *
 942  * Return: True if there is a reference to any of trigger's variables
 943  */
 944 static bool check_var_refs(struct hist_trigger_data *hist_data)
 945 {
 946         struct hist_field *field;
 947         bool found = false;
 948         int i;
 949
 950         for_each_hist_field(i, hist_data) {
 951                 field = hist_data->fields[i];
 952                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 953                         if (find_any_var_ref(hist_data, field->var.idx)) {
 954                                 found = true;
 955                                 break;
 956                         }
 957                 }
 958         }
 959
 960         return found;
 961 }
 962
 963 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 964 {
 965         struct trace_array *tr = hist_data->event_file->tr;
 966         struct hist_var_data *var_data, *found = NULL;
 967
 968         list_for_each_entry(var_data, &tr->hist_vars, list) {
 969                 if (var_data->hist_data == hist_data) {
 970                         found = var_data;
 971                         break;
 972                 }
 973         }
 974
 975         return found;
 976 }
 977
 978 static bool field_has_hist_vars(struct hist_field *hist_field,
 979                                 unsigned int level)
 980 {
 981         int i;
 982
 983         if (level > 3)
 984                 return false;
 985
 986         if (!hist_field)
 987                 return false;
 988
 989         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 990             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 991                 return true;
 992
 993         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 994                 struct hist_field *operand;
 995
 996                 operand = hist_field->operands[i];
 997                 if (field_has_hist_vars(operand, level + 1))
 998                         return true;
 999         }
1000
1001         return false;
1002 }
1003
1004 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1005 {
1006         struct hist_field *hist_field;
1007         int i;
1008
1009         for_each_hist_field(i, hist_data) {
1010                 hist_field = hist_data->fields[i];
1011                 if (field_has_hist_vars(hist_field, 0))
1012                         return true;
1013         }
1014
1015         return false;
1016 }
1017
1018 static int save_hist_vars(struct hist_trigger_data *hist_data)
1019 {
1020         struct trace_array *tr = hist_data->event_file->tr;
1021         struct hist_var_data *var_data;
1022
1023         var_data = find_hist_vars(hist_data);
1024         if (var_data)
1025                 return 0;
1026
1027         if (tracing_check_open_get_tr(tr))
1028                 return -ENODEV;
1029
1030         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1031         if (!var_data) {
1032                 trace_array_put(tr);
1033                 return -ENOMEM;
1034         }
1035
1036         var_data->hist_data = hist_data;
1037         list_add(&var_data->list, &tr->hist_vars);
1038
1039         return 0;
1040 }
1041
1042 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1043 {
1044         struct trace_array *tr = hist_data->event_file->tr;
1045         struct hist_var_data *var_data;
1046
1047         var_data = find_hist_vars(hist_data);
1048         if (!var_data)
1049                 return;
1050
1051         if (WARN_ON(check_var_refs(hist_data)))
1052                 return;
1053
1054         list_del(&var_data->list);
1055
1056         kfree(var_data);
1057
1058         trace_array_put(tr);
1059 }
1060
1061 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1062                                          const char *var_name)
1063 {
1064         struct hist_field *hist_field, *found = NULL;
1065         int i;
1066
1067         for_each_hist_field(i, hist_data) {
1068                 hist_field = hist_data->fields[i];
1069                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1070                     strcmp(hist_field->var.name, var_name) == 0) {
1071                         found = hist_field;
1072                         break;
1073                 }
1074         }
1075
1076         return found;
1077 }
1078
1079 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1080                                    struct trace_event_file *file,
1081                                    const char *var_name)
1082 {
1083         struct hist_trigger_data *test_data;
1084         struct event_trigger_data *test;
1085         struct hist_field *hist_field;
1086
1087         lockdep_assert_held(&event_mutex);
1088
1089         hist_field = find_var_field(hist_data, var_name);
1090         if (hist_field)
1091                 return hist_field;
1092
1093         list_for_each_entry(test, &file->triggers, list) {
1094                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1095                         test_data = test->private_data;
1096                         hist_field = find_var_field(test_data, var_name);
1097                         if (hist_field)
1098                                 return hist_field;
1099                 }
1100         }
1101
1102         return NULL;
1103 }
1104
1105 static struct trace_event_file *find_var_file(struct trace_array *tr,
1106                                               char *system,
1107                                               char *event_name,
1108                                               char *var_name)
1109 {
1110         struct hist_trigger_data *var_hist_data;
1111         struct hist_var_data *var_data;
1112         struct trace_event_file *file, *found = NULL;
1113
1114         if (system)
1115                 return find_event_file(tr, system, event_name);
1116
1117         list_for_each_entry(var_data, &tr->hist_vars, list) {
1118                 var_hist_data = var_data->hist_data;
1119                 file = var_hist_data->event_file;
1120                 if (file == found)
1121                         continue;
1122
1123                 if (find_var_field(var_hist_data, var_name)) {
1124                         if (found) {
1125                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1126                                 return NULL;
1127                         }
1128
1129                         found = file;
1130                 }
1131         }
1132
1133         return found;
1134 }
1135
1136 static struct hist_field *find_file_var(struct trace_event_file *file,
1137                                         const char *var_name)
1138 {
1139         struct hist_trigger_data *test_data;
1140         struct event_trigger_data *test;
1141         struct hist_field *hist_field;
1142
1143         lockdep_assert_held(&event_mutex);
1144
1145         list_for_each_entry(test, &file->triggers, list) {
1146                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1147                         test_data = test->private_data;
1148                         hist_field = find_var_field(test_data, var_name);
1149                         if (hist_field)
1150                                 return hist_field;
1151                 }
1152         }
1153
1154         return NULL;
1155 }
1156
1157 static struct hist_field *
1158 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1159 {
1160         struct trace_array *tr = hist_data->event_file->tr;
1161         struct hist_field *hist_field, *found = NULL;
1162         struct trace_event_file *file;
1163         unsigned int i;
1164
1165         for (i = 0; i < hist_data->n_actions; i++) {
1166                 struct action_data *data = hist_data->actions[i];
1167
1168                 if (data->handler == HANDLER_ONMATCH) {
1169                         char *system = data->match_data.event_system;
1170                         char *event_name = data->match_data.event;
1171
1172                         file = find_var_file(tr, system, event_name, var_name);
1173                         if (!file)
1174                                 continue;
1175                         hist_field = find_file_var(file, var_name);
1176                         if (hist_field) {
1177                                 if (found) {
1178                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1179                                                  errpos(var_name));
1180                                         return ERR_PTR(-EINVAL);
1181                                 }
1182
1183                                 found = hist_field;
1184                         }
1185                 }
1186         }
1187         return found;
1188 }
1189
1190 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1191                                          char *system,
1192                                          char *event_name,
1193                                          char *var_name)
1194 {
1195         struct trace_array *tr = hist_data->event_file->tr;
1196         struct hist_field *hist_field = NULL;
1197         struct trace_event_file *file;
1198
1199         if (!system || !event_name) {
1200                 hist_field = find_match_var(hist_data, var_name);
1201                 if (IS_ERR(hist_field))
1202                         return NULL;
1203                 if (hist_field)
1204                         return hist_field;
1205         }
1206
1207         file = find_var_file(tr, system, event_name, var_name);
1208         if (!file)
1209                 return NULL;
1210
1211         hist_field = find_file_var(file, var_name);
1212
1213         return hist_field;
1214 }
1215
1216 static u64 hist_field_var_ref(struct hist_field *hist_field,
1217                               struct tracing_map_elt *elt,
1218                               struct trace_buffer *buffer,
1219                               struct ring_buffer_event *rbe,
1220                               void *event)
1221 {
1222         struct hist_elt_data *elt_data;
1223         u64 var_val = 0;
1224
1225         if (WARN_ON_ONCE(!elt))
1226                 return var_val;
1227
1228         elt_data = elt->private_data;
1229         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1230
1231         return var_val;
1232 }
1233
1234 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1235                              u64 *var_ref_vals, bool self)
1236 {
1237         struct hist_trigger_data *var_data;
1238         struct tracing_map_elt *var_elt;
1239         struct hist_field *hist_field;
1240         unsigned int i, var_idx;
1241         bool resolved = true;
1242         u64 var_val = 0;
1243
1244         for (i = 0; i < hist_data->n_var_refs; i++) {
1245                 hist_field = hist_data->var_refs[i];
1246                 var_idx = hist_field->var.idx;
1247                 var_data = hist_field->var.hist_data;
1248
1249                 if (var_data == NULL) {
1250                         resolved = false;
1251                         break;
1252                 }
1253
1254                 if ((self && var_data != hist_data) ||
1255                     (!self && var_data == hist_data))
1256                         continue;
1257
1258                 var_elt = tracing_map_lookup(var_data->map, key);
1259                 if (!var_elt) {
1260                         resolved = false;
1261                         break;
1262                 }
1263
1264                 if (!tracing_map_var_set(var_elt, var_idx)) {
1265                         resolved = false;
1266                         break;
1267                 }
1268
1269                 if (self || !hist_field->read_once)
1270                         var_val = tracing_map_read_var(var_elt, var_idx);
1271                 else
1272                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1273
1274                 var_ref_vals[i] = var_val;
1275         }
1276
1277         return resolved;
1278 }
1279
1280 static const char *hist_field_name(struct hist_field *field,
1281                                    unsigned int level)
1282 {
1283         const char *field_name = "";
1284
1285         if (level > 1)
1286                 return field_name;
1287
1288         if (field->field)
1289                 field_name = field->field->name;
1290         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1291                  field->flags & HIST_FIELD_FL_ALIAS ||
1292                  field->flags & HIST_FIELD_FL_BUCKET)
1293                 field_name = hist_field_name(field->operands[0], ++level);
1294         else if (field->flags & HIST_FIELD_FL_CPU)
1295                 field_name = "common_cpu";
1296         else if (field->flags & HIST_FIELD_FL_EXPR ||
1297                  field->flags & HIST_FIELD_FL_VAR_REF) {
1298                 if (field->system) {
1299                         static char full_name[MAX_FILTER_STR_VAL];
1300
1301                         strcat(full_name, field->system);
1302                         strcat(full_name, ".");
1303                         strcat(full_name, field->event_name);
1304                         strcat(full_name, ".");
1305                         strcat(full_name, field->name);
1306                         field_name = full_name;
1307                 } else
1308                         field_name = field->name;
1309         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1310                 field_name = "common_timestamp";
1311
1312         if (field_name == NULL)
1313                 field_name = "";
1314
1315         return field_name;
1316 }
1317
1318 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1319 {
1320         hist_field_fn_t fn = NULL;
1321
1322         switch (field_size) {
1323         case 8:
1324                 if (field_is_signed)
1325                         fn = hist_field_s64;
1326                 else
1327                         fn = hist_field_u64;
1328                 break;
1329         case 4:
1330                 if (field_is_signed)
1331                         fn = hist_field_s32;
1332                 else
1333                         fn = hist_field_u32;
1334                 break;
1335         case 2:
1336                 if (field_is_signed)
1337                         fn = hist_field_s16;
1338                 else
1339                         fn = hist_field_u16;
1340                 break;
1341         case 1:
1342                 if (field_is_signed)
1343                         fn = hist_field_s8;
1344                 else
1345                         fn = hist_field_u8;
1346                 break;
1347         }
1348
1349         return fn;
1350 }
1351
1352 static int parse_map_size(char *str)
1353 {
1354         unsigned long size, map_bits;
1355         int ret;
1356
1357         ret = kstrtoul(str, 0, &size);
1358         if (ret)
1359                 goto out;
1360
1361         map_bits = ilog2(roundup_pow_of_two(size));
1362         if (map_bits < TRACING_MAP_BITS_MIN ||
1363             map_bits > TRACING_MAP_BITS_MAX)
1364                 ret = -EINVAL;
1365         else
1366                 ret = map_bits;
1367  out:
1368         return ret;
1369 }
1370
1371 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1372 {
1373         unsigned int i;
1374
1375         if (!attrs)
1376                 return;
1377
1378         for (i = 0; i < attrs->n_assignments; i++)
1379                 kfree(attrs->assignment_str[i]);
1380
1381         for (i = 0; i < attrs->n_actions; i++)
1382                 kfree(attrs->action_str[i]);
1383
1384         kfree(attrs->name);
1385         kfree(attrs->sort_key_str);
1386         kfree(attrs->keys_str);
1387         kfree(attrs->vals_str);
1388         kfree(attrs->clock);
1389         kfree(attrs);
1390 }
1391
1392 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1393 {
1394         int ret = -EINVAL;
1395
1396         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1397                 return ret;
1398
1399         if ((str_has_prefix(str, "onmatch(")) ||
1400             (str_has_prefix(str, "onmax(")) ||
1401             (str_has_prefix(str, "onchange("))) {
1402                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1403                 if (!attrs->action_str[attrs->n_actions]) {
1404                         ret = -ENOMEM;
1405                         return ret;
1406                 }
1407                 attrs->n_actions++;
1408                 ret = 0;
1409         }
1410         return ret;
1411 }
1412
1413 static int parse_assignment(struct trace_array *tr,
1414                             char *str, struct hist_trigger_attrs *attrs)
1415 {
1416         int len, ret = 0;
1417
1418         if ((len = str_has_prefix(str, "key=")) ||
1419             (len = str_has_prefix(str, "keys="))) {
1420                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1421                 if (!attrs->keys_str) {
1422                         ret = -ENOMEM;
1423                         goto out;
1424                 }
1425         } else if ((len = str_has_prefix(str, "val=")) ||
1426                    (len = str_has_prefix(str, "vals=")) ||
1427                    (len = str_has_prefix(str, "values="))) {
1428                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1429                 if (!attrs->vals_str) {
1430                         ret = -ENOMEM;
1431                         goto out;
1432                 }
1433         } else if ((len = str_has_prefix(str, "sort="))) {
1434                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1435                 if (!attrs->sort_key_str) {
1436                         ret = -ENOMEM;
1437                         goto out;
1438                 }
1439         } else if (str_has_prefix(str, "name=")) {
1440                 attrs->name = kstrdup(str, GFP_KERNEL);
1441                 if (!attrs->name) {
1442                         ret = -ENOMEM;
1443                         goto out;
1444                 }
1445         } else if ((len = str_has_prefix(str, "clock="))) {
1446                 str += len;
1447
1448                 str = strstrip(str);
1449                 attrs->clock = kstrdup(str, GFP_KERNEL);
1450                 if (!attrs->clock) {
1451                         ret = -ENOMEM;
1452                         goto out;
1453                 }
1454         } else if ((len = str_has_prefix(str, "size="))) {
1455                 int map_bits = parse_map_size(str + len);
1456
1457                 if (map_bits < 0) {
1458                         ret = map_bits;
1459                         goto out;
1460                 }
1461                 attrs->map_bits = map_bits;
1462         } else {
1463                 char *assignment;
1464
1465                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1466                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1467                         ret = -EINVAL;
1468                         goto out;
1469                 }
1470
1471                 assignment = kstrdup(str, GFP_KERNEL);
1472                 if (!assignment) {
1473                         ret = -ENOMEM;
1474                         goto out;
1475                 }
1476
1477                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1478         }
1479  out:
1480         return ret;
1481 }
1482
1483 static struct hist_trigger_attrs *
1484 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1485 {
1486         struct hist_trigger_attrs *attrs;
1487         int ret = 0;
1488
1489         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1490         if (!attrs)
1491                 return ERR_PTR(-ENOMEM);
1492
1493         while (trigger_str) {
1494                 char *str = strsep(&trigger_str, ":");
1495                 char *rhs;
1496
1497                 rhs = strchr(str, '=');
1498                 if (rhs) {
1499                         if (!strlen(++rhs)) {
1500                                 ret = -EINVAL;
1501                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1502                                 goto free;
1503                         }
1504                         ret = parse_assignment(tr, str, attrs);
1505                         if (ret)
1506                                 goto free;
1507                 } else if (strcmp(str, "pause") == 0)
1508                         attrs->pause = true;
1509                 else if ((strcmp(str, "cont") == 0) ||
1510                          (strcmp(str, "continue") == 0))
1511                         attrs->cont = true;
1512                 else if (strcmp(str, "clear") == 0)
1513                         attrs->clear = true;
1514                 else {
1515                         ret = parse_action(str, attrs);
1516                         if (ret)
1517                                 goto free;
1518                 }
1519         }
1520
1521         if (!attrs->keys_str) {
1522                 ret = -EINVAL;
1523                 goto free;
1524         }
1525
1526         if (!attrs->clock) {
1527                 attrs->clock = kstrdup("global", GFP_KERNEL);
1528                 if (!attrs->clock) {
1529                         ret = -ENOMEM;
1530                         goto free;
1531                 }
1532         }
1533
1534         return attrs;
1535  free:
1536         destroy_hist_trigger_attrs(attrs);
1537
1538         return ERR_PTR(ret);
1539 }
1540
1541 static inline void save_comm(char *comm, struct task_struct *task)
1542 {
1543         if (!task->pid) {
1544                 strcpy(comm, "<idle>");
1545                 return;
1546         }
1547
1548         if (WARN_ON_ONCE(task->pid < 0)) {
1549                 strcpy(comm, "<XXX>");
1550                 return;
1551         }
1552
1553         strncpy(comm, task->comm, TASK_COMM_LEN);
1554 }
1555
1556 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1557 {
1558         unsigned int i;
1559
1560         for (i = 0; i < elt_data->n_field_var_str; i++)
1561                 kfree(elt_data->field_var_str[i]);
1562
1563         kfree(elt_data->field_var_str);
1564
1565         kfree(elt_data->comm);
1566         kfree(elt_data);
1567 }
1568
1569 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1570 {
1571         struct hist_elt_data *elt_data = elt->private_data;
1572
1573         hist_elt_data_free(elt_data);
1574 }
1575
1576 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1577 {
1578         struct hist_trigger_data *hist_data = elt->map->private_data;
1579         unsigned int size = TASK_COMM_LEN;
1580         struct hist_elt_data *elt_data;
1581         struct hist_field *hist_field;
1582         unsigned int i, n_str;
1583
1584         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1585         if (!elt_data)
1586                 return -ENOMEM;
1587
1588         for_each_hist_field(i, hist_data) {
1589                 hist_field = hist_data->fields[i];
1590
1591                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1592                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1593                         if (!elt_data->comm) {
1594                                 kfree(elt_data);
1595                                 return -ENOMEM;
1596                         }
1597                         break;
1598                 }
1599         }
1600
1601         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1602                 hist_data->n_var_str;
1603         if (n_str > SYNTH_FIELDS_MAX) {
1604                 hist_elt_data_free(elt_data);
1605                 return -EINVAL;
1606         }
1607
1608         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1609
1610         size = STR_VAR_LEN_MAX;
1611
1612         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1613         if (!elt_data->field_var_str) {
1614                 hist_elt_data_free(elt_data);
1615                 return -EINVAL;
1616         }
1617         elt_data->n_field_var_str = n_str;
1618
1619         for (i = 0; i < n_str; i++) {
1620                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1621                 if (!elt_data->field_var_str[i]) {
1622                         hist_elt_data_free(elt_data);
1623                         return -ENOMEM;
1624                 }
1625         }
1626
1627         elt->private_data = elt_data;
1628
1629         return 0;
1630 }
1631
1632 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1633 {
1634         struct hist_elt_data *elt_data = elt->private_data;
1635
1636         if (elt_data->comm)
1637                 save_comm(elt_data->comm, current);
1638 }
1639
1640 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1641         .elt_alloc      = hist_trigger_elt_data_alloc,
1642         .elt_free       = hist_trigger_elt_data_free,
1643         .elt_init       = hist_trigger_elt_data_init,
1644 };
1645
1646 static const char *get_hist_field_flags(struct hist_field *hist_field)
1647 {
1648         const char *flags_str = NULL;
1649
1650         if (hist_field->flags & HIST_FIELD_FL_HEX)
1651                 flags_str = "hex";
1652         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1653                 flags_str = "sym";
1654         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1655                 flags_str = "sym-offset";
1656         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1657                 flags_str = "execname";
1658         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1659                 flags_str = "syscall";
1660         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1661                 flags_str = "log2";
1662         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1663                 flags_str = "buckets";
1664         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1665                 flags_str = "usecs";
1666
1667         return flags_str;
1668 }
1669
1670 static void expr_field_str(struct hist_field *field, char *expr)
1671 {
1672         if (field->flags & HIST_FIELD_FL_VAR_REF)
1673                 strcat(expr, "$");
1674         else if (field->flags & HIST_FIELD_FL_CONST) {
1675                 char str[HIST_CONST_DIGITS_MAX];
1676
1677                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1678                 strcat(expr, str);
1679         }
1680
1681         strcat(expr, hist_field_name(field, 0));
1682
1683         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1684                 const char *flags_str = get_hist_field_flags(field);
1685
1686                 if (flags_str) {
1687                         strcat(expr, ".");
1688                         strcat(expr, flags_str);
1689                 }
1690         }
1691 }
1692
1693 static char *expr_str(struct hist_field *field, unsigned int level)
1694 {
1695         char *expr;
1696
1697         if (level > 1)
1698                 return NULL;
1699
1700         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1701         if (!expr)
1702                 return NULL;
1703
1704         if (!field->operands[0]) {
1705                 expr_field_str(field, expr);
1706                 return expr;
1707         }
1708
1709         if (field->operator == FIELD_OP_UNARY_MINUS) {
1710                 char *subexpr;
1711
1712                 strcat(expr, "-(");
1713                 subexpr = expr_str(field->operands[0], ++level);
1714                 if (!subexpr) {
1715                         kfree(expr);
1716                         return NULL;
1717                 }
1718                 strcat(expr, subexpr);
1719                 strcat(expr, ")");
1720
1721                 kfree(subexpr);
1722
1723                 return expr;
1724         }
1725
1726         expr_field_str(field->operands[0], expr);
1727
1728         switch (field->operator) {
1729         case FIELD_OP_MINUS:
1730                 strcat(expr, "-");
1731                 break;
1732         case FIELD_OP_PLUS:
1733                 strcat(expr, "+");
1734                 break;
1735         case FIELD_OP_DIV:
1736                 strcat(expr, "/");
1737                 break;
1738         case FIELD_OP_MULT:
1739                 strcat(expr, "*");
1740                 break;
1741         default:
1742                 kfree(expr);
1743                 return NULL;
1744         }
1745
1746         expr_field_str(field->operands[1], expr);
1747
1748         return expr;
1749 }
1750
1751 /*
1752  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1753  * of the expression tree to be evaluated.
1754  */
1755 static int contains_operator(char *str, char **sep)
1756 {
1757         enum field_op_id field_op = FIELD_OP_NONE;
1758         char *minus_op, *plus_op, *div_op, *mult_op;
1759
1760
1761         /*
1762          * Report the last occurrence of the operators first, so that the
1763          * expression is evaluated left to right. This is important since
1764          * subtraction and division are not associative.
1765          *
1766          *      e.g
1767          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1768          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1769          */
1770
1771         /*
1772          * First, find lower precedence addition and subtraction
1773          * since the expression will be evaluated recursively.
1774          */
1775         minus_op = strrchr(str, '-');
1776         if (minus_op) {
1777                 /*
1778                  * Unary minus is not supported in sub-expressions. If
1779                  * present, it is always the next root operator.
1780                  */
1781                 if (minus_op == str) {
1782                         field_op = FIELD_OP_UNARY_MINUS;
1783                         goto out;
1784                 }
1785
1786                 field_op = FIELD_OP_MINUS;
1787         }
1788
1789         plus_op = strrchr(str, '+');
1790         if (plus_op || minus_op) {
1791                 /*
1792                  * For operators of the same precedence use to rightmost as the
1793                  * root, so that the expression is evaluated left to right.
1794                  */
1795                 if (plus_op > minus_op)
1796                         field_op = FIELD_OP_PLUS;
1797                 goto out;
1798         }
1799
1800         /*
1801          * Multiplication and division have higher precedence than addition and
1802          * subtraction.
1803          */
1804         div_op = strrchr(str, '/');
1805         if (div_op)
1806                 field_op = FIELD_OP_DIV;
1807
1808         mult_op = strrchr(str, '*');
1809         /*
1810          * For operators of the same precedence use to rightmost as the
1811          * root, so that the expression is evaluated left to right.
1812          */
1813         if (mult_op > div_op)
1814                 field_op = FIELD_OP_MULT;
1815
1816 out:
1817         if (sep) {
1818                 switch (field_op) {
1819                 case FIELD_OP_UNARY_MINUS:
1820                 case FIELD_OP_MINUS:
1821                         *sep = minus_op;
1822                         break;
1823                 case FIELD_OP_PLUS:
1824                         *sep = plus_op;
1825                         break;
1826                 case FIELD_OP_DIV:
1827                         *sep = div_op;
1828                         break;
1829                 case FIELD_OP_MULT:
1830                         *sep = mult_op;
1831                         break;
1832                 case FIELD_OP_NONE:
1833                 default:
1834                         *sep = NULL;
1835                         break;
1836                 }
1837         }
1838
1839         return field_op;
1840 }
1841
1842 static void get_hist_field(struct hist_field *hist_field)
1843 {
1844         hist_field->ref++;
1845 }
1846
1847 static void __destroy_hist_field(struct hist_field *hist_field)
1848 {
1849         if (--hist_field->ref > 1)
1850                 return;
1851
1852         kfree(hist_field->var.name);
1853         kfree(hist_field->name);
1854
1855         /* Can likely be a const */
1856         kfree_const(hist_field->type);
1857
1858         kfree(hist_field->system);
1859         kfree(hist_field->event_name);
1860
1861         kfree(hist_field);
1862 }
1863
1864 static void destroy_hist_field(struct hist_field *hist_field,
1865                                unsigned int level)
1866 {
1867         unsigned int i;
1868
1869         if (level > 3)
1870                 return;
1871
1872         if (!hist_field)
1873                 return;
1874
1875         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1876                 return; /* var refs will be destroyed separately */
1877
1878         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1879                 destroy_hist_field(hist_field->operands[i], level + 1);
1880
1881         __destroy_hist_field(hist_field);
1882 }
1883
1884 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1885                                             struct ftrace_event_field *field,
1886                                             unsigned long flags,
1887                                             char *var_name)
1888 {
1889         struct hist_field *hist_field;
1890
1891         if (field && is_function_field(field))
1892                 return NULL;
1893
1894         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1895         if (!hist_field)
1896                 return NULL;
1897
1898         hist_field->ref = 1;
1899
1900         hist_field->hist_data = hist_data;
1901
1902         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1903                 goto out; /* caller will populate */
1904
1905         if (flags & HIST_FIELD_FL_VAR_REF) {
1906                 hist_field->fn = hist_field_var_ref;
1907                 goto out;
1908         }
1909
1910         if (flags & HIST_FIELD_FL_HITCOUNT) {
1911                 hist_field->fn = hist_field_counter;
1912                 hist_field->size = sizeof(u64);
1913                 hist_field->type = "u64";
1914                 goto out;
1915         }
1916
1917         if (flags & HIST_FIELD_FL_CONST) {
1918                 hist_field->fn = hist_field_const;
1919                 hist_field->size = sizeof(u64);
1920                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1921                 if (!hist_field->type)
1922                         goto free;
1923                 goto out;
1924         }
1925
1926         if (flags & HIST_FIELD_FL_STACKTRACE) {
1927                 hist_field->fn = hist_field_none;
1928                 goto out;
1929         }
1930
1931         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1932                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1933                 hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1934                         hist_field_bucket;
1935                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1936                 hist_field->size = hist_field->operands[0]->size;
1937                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1938                 if (!hist_field->type)
1939                         goto free;
1940                 goto out;
1941         }
1942
1943         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1944                 hist_field->fn = hist_field_timestamp;
1945                 hist_field->size = sizeof(u64);
1946                 hist_field->type = "u64";
1947                 goto out;
1948         }
1949
1950         if (flags & HIST_FIELD_FL_CPU) {
1951                 hist_field->fn = hist_field_cpu;
1952                 hist_field->size = sizeof(int);
1953                 hist_field->type = "unsigned int";
1954                 goto out;
1955         }
1956
1957         if (WARN_ON_ONCE(!field))
1958                 goto out;
1959
1960         /* Pointers to strings are just pointers and dangerous to dereference */
1961         if (is_string_field(field) &&
1962             (field->filter_type != FILTER_PTR_STRING)) {
1963                 flags |= HIST_FIELD_FL_STRING;
1964
1965                 hist_field->size = MAX_FILTER_STR_VAL;
1966                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1967                 if (!hist_field->type)
1968                         goto free;
1969
1970                 if (field->filter_type == FILTER_STATIC_STRING) {
1971                         hist_field->fn = hist_field_string;
1972                         hist_field->size = field->size;
1973                 } else if (field->filter_type == FILTER_DYN_STRING) {
1974                         hist_field->fn = hist_field_dynstring;
1975                 } else if (field->filter_type == FILTER_RDYN_STRING)
1976                         hist_field->fn = hist_field_reldynstring;
1977                 else
1978                         hist_field->fn = hist_field_pstring;
1979         } else {
1980                 hist_field->size = field->size;
1981                 hist_field->is_signed = field->is_signed;
1982                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1983                 if (!hist_field->type)
1984                         goto free;
1985
1986                 hist_field->fn = select_value_fn(field->size,
1987                                                  field->is_signed);
1988                 if (!hist_field->fn) {
1989                         destroy_hist_field(hist_field, 0);
1990                         return NULL;
1991                 }
1992         }
1993  out:
1994         hist_field->field = field;
1995         hist_field->flags = flags;
1996
1997         if (var_name) {
1998                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1999                 if (!hist_field->var.name)
2000                         goto free;
2001         }
2002
2003         return hist_field;
2004  free:
2005         destroy_hist_field(hist_field, 0);
2006         return NULL;
2007 }
2008
2009 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2010 {
2011         unsigned int i;
2012
2013         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2014                 if (hist_data->fields[i]) {
2015                         destroy_hist_field(hist_data->fields[i], 0);
2016                         hist_data->fields[i] = NULL;
2017                 }
2018         }
2019
2020         for (i = 0; i < hist_data->n_var_refs; i++) {
2021                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2022                 __destroy_hist_field(hist_data->var_refs[i]);
2023                 hist_data->var_refs[i] = NULL;
2024         }
2025 }
2026
2027 static int init_var_ref(struct hist_field *ref_field,
2028                         struct hist_field *var_field,
2029                         char *system, char *event_name)
2030 {
2031         int err = 0;
2032
2033         ref_field->var.idx = var_field->var.idx;
2034         ref_field->var.hist_data = var_field->hist_data;
2035         ref_field->size = var_field->size;
2036         ref_field->is_signed = var_field->is_signed;
2037         ref_field->flags |= var_field->flags &
2038                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2039
2040         if (system) {
2041                 ref_field->system = kstrdup(system, GFP_KERNEL);
2042                 if (!ref_field->system)
2043                         return -ENOMEM;
2044         }
2045
2046         if (event_name) {
2047                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2048                 if (!ref_field->event_name) {
2049                         err = -ENOMEM;
2050                         goto free;
2051                 }
2052         }
2053
2054         if (var_field->var.name) {
2055                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2056                 if (!ref_field->name) {
2057                         err = -ENOMEM;
2058                         goto free;
2059                 }
2060         } else if (var_field->name) {
2061                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2062                 if (!ref_field->name) {
2063                         err = -ENOMEM;
2064                         goto free;
2065                 }
2066         }
2067
2068         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2069         if (!ref_field->type) {
2070                 err = -ENOMEM;
2071                 goto free;
2072         }
2073  out:
2074         return err;
2075  free:
2076         kfree(ref_field->system);
2077         kfree(ref_field->event_name);
2078         kfree(ref_field->name);
2079
2080         goto out;
2081 }
2082
2083 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2084                             struct hist_field *var_field)
2085 {
2086         struct hist_field *ref_field;
2087         int i;
2088
2089         for (i = 0; i < hist_data->n_var_refs; i++) {
2090                 ref_field = hist_data->var_refs[i];
2091                 if (ref_field->var.idx == var_field->var.idx &&
2092                     ref_field->var.hist_data == var_field->hist_data)
2093                         return i;
2094         }
2095
2096         return -ENOENT;
2097 }
2098
2099 /**
2100  * create_var_ref - Create a variable reference and attach it to trigger
2101  * @hist_data: The trigger that will be referencing the variable
2102  * @var_field: The VAR field to create a reference to
2103  * @system: The optional system string
2104  * @event_name: The optional event_name string
2105  *
2106  * Given a variable hist_field, create a VAR_REF hist_field that
2107  * represents a reference to it.
2108  *
2109  * This function also adds the reference to the trigger that
2110  * now references the variable.
2111  *
2112  * Return: The VAR_REF field if successful, NULL if not
2113  */
2114 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2115                                          struct hist_field *var_field,
2116                                          char *system, char *event_name)
2117 {
2118         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2119         struct hist_field *ref_field;
2120         int i;
2121
2122         /* Check if the variable already exists */
2123         for (i = 0; i < hist_data->n_var_refs; i++) {
2124                 ref_field = hist_data->var_refs[i];
2125                 if (ref_field->var.idx == var_field->var.idx &&
2126                     ref_field->var.hist_data == var_field->hist_data) {
2127                         get_hist_field(ref_field);
2128                         return ref_field;
2129                 }
2130         }
2131
2132         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2133         if (ref_field) {
2134                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2135                         destroy_hist_field(ref_field, 0);
2136                         return NULL;
2137                 }
2138
2139                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2140                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2141         }
2142
2143         return ref_field;
2144 }
2145
2146 static bool is_var_ref(char *var_name)
2147 {
2148         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2149                 return false;
2150
2151         return true;
2152 }
2153
2154 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2155                                  char *var_name)
2156 {
2157         char *name, *field;
2158         unsigned int i;
2159
2160         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2161                 name = hist_data->attrs->var_defs.name[i];
2162
2163                 if (strcmp(var_name, name) == 0) {
2164                         field = hist_data->attrs->var_defs.expr[i];
2165                         if (contains_operator(field, NULL) || is_var_ref(field))
2166                                 continue;
2167                         return field;
2168                 }
2169         }
2170
2171         return NULL;
2172 }
2173
2174 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2175                                  char *system, char *event_name,
2176                                  char *var_name)
2177 {
2178         struct trace_event_call *call;
2179
2180         if (system && event_name) {
2181                 call = hist_data->event_file->event_call;
2182
2183                 if (strcmp(system, call->class->system) != 0)
2184                         return NULL;
2185
2186                 if (strcmp(event_name, trace_event_name(call)) != 0)
2187                         return NULL;
2188         }
2189
2190         if (!!system != !!event_name)
2191                 return NULL;
2192
2193         if (!is_var_ref(var_name))
2194                 return NULL;
2195
2196         var_name++;
2197
2198         return field_name_from_var(hist_data, var_name);
2199 }
2200
2201 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2202                                         char *system, char *event_name,
2203                                         char *var_name)
2204 {
2205         struct hist_field *var_field = NULL, *ref_field = NULL;
2206         struct trace_array *tr = hist_data->event_file->tr;
2207
2208         if (!is_var_ref(var_name))
2209                 return NULL;
2210
2211         var_name++;
2212
2213         var_field = find_event_var(hist_data, system, event_name, var_name);
2214         if (var_field)
2215                 ref_field = create_var_ref(hist_data, var_field,
2216                                            system, event_name);
2217
2218         if (!ref_field)
2219                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2220
2221         return ref_field;
2222 }
2223
2224 static struct ftrace_event_field *
2225 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2226             char *field_str, unsigned long *flags, unsigned long *buckets)
2227 {
2228         struct ftrace_event_field *field = NULL;
2229         char *field_name, *modifier, *str;
2230         struct trace_array *tr = file->tr;
2231
2232         modifier = str = kstrdup(field_str, GFP_KERNEL);
2233         if (!modifier)
2234                 return ERR_PTR(-ENOMEM);
2235
2236         field_name = strsep(&modifier, ".");
2237         if (modifier) {
2238                 if (strcmp(modifier, "hex") == 0)
2239                         *flags |= HIST_FIELD_FL_HEX;
2240                 else if (strcmp(modifier, "sym") == 0)
2241                         *flags |= HIST_FIELD_FL_SYM;
2242                 /*
2243                  * 'sym-offset' occurrences in the trigger string are modified
2244                  * to 'symXoffset' to simplify arithmetic expression parsing.
2245                  */
2246                 else if (strcmp(modifier, "symXoffset") == 0)
2247                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2248                 else if ((strcmp(modifier, "execname") == 0) &&
2249                          (strcmp(field_name, "common_pid") == 0))
2250                         *flags |= HIST_FIELD_FL_EXECNAME;
2251                 else if (strcmp(modifier, "syscall") == 0)
2252                         *flags |= HIST_FIELD_FL_SYSCALL;
2253                 else if (strcmp(modifier, "log2") == 0)
2254                         *flags |= HIST_FIELD_FL_LOG2;
2255                 else if (strcmp(modifier, "usecs") == 0)
2256                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2257                 else if (strncmp(modifier, "bucket", 6) == 0) {
2258                         int ret;
2259
2260                         modifier += 6;
2261
2262                         if (*modifier == 's')
2263                                 modifier++;
2264                         if (*modifier != '=')
2265                                 goto error;
2266                         modifier++;
2267                         ret = kstrtoul(modifier, 0, buckets);
2268                         if (ret || !(*buckets))
2269                                 goto error;
2270                         *flags |= HIST_FIELD_FL_BUCKET;
2271                 } else {
2272  error:
2273                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2274                         field = ERR_PTR(-EINVAL);
2275                         goto out;
2276                 }
2277         }
2278
2279         if (strcmp(field_name, "common_timestamp") == 0) {
2280                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2281                 hist_data->enable_timestamps = true;
2282                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2283                         hist_data->attrs->ts_in_usecs = true;
2284         } else if (strcmp(field_name, "common_cpu") == 0)
2285                 *flags |= HIST_FIELD_FL_CPU;
2286         else {
2287                 field = trace_find_event_field(file->event_call, field_name);
2288                 if (!field || !field->size) {
2289                         /*
2290                          * For backward compatibility, if field_name
2291                          * was "cpu", then we treat this the same as
2292                          * common_cpu.
2293                          */
2294                         if (strcmp(field_name, "cpu") == 0) {
2295                                 *flags |= HIST_FIELD_FL_CPU;
2296                         } else {
2297                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2298                                          errpos(field_name));
2299                                 field = ERR_PTR(-EINVAL);
2300                                 goto out;
2301                         }
2302                 }
2303         }
2304  out:
2305         kfree(str);
2306
2307         return field;
2308 }
2309
2310 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2311                                        struct hist_field *var_ref,
2312                                        char *var_name)
2313 {
2314         struct hist_field *alias = NULL;
2315         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2316
2317         alias = create_hist_field(hist_data, NULL, flags, var_name);
2318         if (!alias)
2319                 return NULL;
2320
2321         alias->fn = var_ref->fn;
2322         alias->operands[0] = var_ref;
2323
2324         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2325                 destroy_hist_field(alias, 0);
2326                 return NULL;
2327         }
2328
2329         alias->var_ref_idx = var_ref->var_ref_idx;
2330
2331         return alias;
2332 }
2333
2334 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2335                                       char *str, char *var_name,
2336                                       unsigned long *flags)
2337 {
2338         struct trace_array *tr = hist_data->event_file->tr;
2339         struct hist_field *field = NULL;
2340         u64 constant;
2341
2342         if (kstrtoull(str, 0, &constant)) {
2343                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2344                 return NULL;
2345         }
2346
2347         *flags |= HIST_FIELD_FL_CONST;
2348         field = create_hist_field(hist_data, NULL, *flags, var_name);
2349         if (!field)
2350                 return NULL;
2351
2352         field->constant = constant;
2353
2354         return field;
2355 }
2356
2357 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2358                                      struct trace_event_file *file, char *str,
2359                                      unsigned long *flags, char *var_name)
2360 {
2361         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2362         struct ftrace_event_field *field = NULL;
2363         struct hist_field *hist_field = NULL;
2364         unsigned long buckets = 0;
2365         int ret = 0;
2366
2367         if (isdigit(str[0])) {
2368                 hist_field = parse_const(hist_data, str, var_name, flags);
2369                 if (!hist_field) {
2370                         ret = -EINVAL;
2371                         goto out;
2372                 }
2373                 return hist_field;
2374         }
2375
2376         s = strchr(str, '.');
2377         if (s) {
2378                 s = strchr(++s, '.');
2379                 if (s) {
2380                         ref_system = strsep(&str, ".");
2381                         if (!str) {
2382                                 ret = -EINVAL;
2383                                 goto out;
2384                         }
2385                         ref_event = strsep(&str, ".");
2386                         if (!str) {
2387                                 ret = -EINVAL;
2388                                 goto out;
2389                         }
2390                         ref_var = str;
2391                 }
2392         }
2393
2394         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2395         if (!s) {
2396                 hist_field = parse_var_ref(hist_data, ref_system,
2397                                            ref_event, ref_var);
2398                 if (hist_field) {
2399                         if (var_name) {
2400                                 hist_field = create_alias(hist_data, hist_field, var_name);
2401                                 if (!hist_field) {
2402                                         ret = -ENOMEM;
2403                                         goto out;
2404                                 }
2405                         }
2406                         return hist_field;
2407                 }
2408         } else
2409                 str = s;
2410
2411         field = parse_field(hist_data, file, str, flags, &buckets);
2412         if (IS_ERR(field)) {
2413                 ret = PTR_ERR(field);
2414                 goto out;
2415         }
2416
2417         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2418         if (!hist_field) {
2419                 ret = -ENOMEM;
2420                 goto out;
2421         }
2422         hist_field->buckets = buckets;
2423
2424         return hist_field;
2425  out:
2426         return ERR_PTR(ret);
2427 }
2428
2429 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2430                                      struct trace_event_file *file,
2431                                      char *str, unsigned long flags,
2432                                      char *var_name, unsigned int *n_subexprs);
2433
2434 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2435                                       struct trace_event_file *file,
2436                                       char *str, unsigned long flags,
2437                                       char *var_name, unsigned int *n_subexprs)
2438 {
2439         struct hist_field *operand1, *expr = NULL;
2440         unsigned long operand_flags;
2441         int ret = 0;
2442         char *s;
2443
2444         /* Unary minus operator, increment n_subexprs */
2445         ++*n_subexprs;
2446
2447         /* we support only -(xxx) i.e. explicit parens required */
2448
2449         if (*n_subexprs > 3) {
2450                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2451                 ret = -EINVAL;
2452                 goto free;
2453         }
2454
2455         str++; /* skip leading '-' */
2456
2457         s = strchr(str, '(');
2458         if (s)
2459                 str++;
2460         else {
2461                 ret = -EINVAL;
2462                 goto free;
2463         }
2464
2465         s = strrchr(str, ')');
2466         if (s) {
2467                  /* unary minus not supported in sub-expressions */
2468                 if (*(s+1) != '\0') {
2469                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2470                                  errpos(str));
2471                         ret = -EINVAL;
2472                         goto free;
2473                 }
2474                 *s = '\0';
2475         }
2476         else {
2477                 ret = -EINVAL; /* no closing ')' */
2478                 goto free;
2479         }
2480
2481         flags |= HIST_FIELD_FL_EXPR;
2482         expr = create_hist_field(hist_data, NULL, flags, var_name);
2483         if (!expr) {
2484                 ret = -ENOMEM;
2485                 goto free;
2486         }
2487
2488         operand_flags = 0;
2489         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2490         if (IS_ERR(operand1)) {
2491                 ret = PTR_ERR(operand1);
2492                 goto free;
2493         }
2494         if (operand1->flags & HIST_FIELD_FL_STRING) {
2495                 /* String type can not be the operand of unary operator. */
2496                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2497                 destroy_hist_field(operand1, 0);
2498                 ret = -EINVAL;
2499                 goto free;
2500         }
2501
2502         expr->flags |= operand1->flags &
2503                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2504         expr->fn = hist_field_unary_minus;
2505         expr->operands[0] = operand1;
2506         expr->operator = FIELD_OP_UNARY_MINUS;
2507         expr->name = expr_str(expr, 0);
2508         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2509         if (!expr->type) {
2510                 ret = -ENOMEM;
2511                 goto free;
2512         }
2513
2514         return expr;
2515  free:
2516         destroy_hist_field(expr, 0);
2517         return ERR_PTR(ret);
2518 }
2519
2520 /*
2521  * If the operands are var refs, return pointers the
2522  * variable(s) referenced in var1 and var2, else NULL.
2523  */
2524 static int check_expr_operands(struct trace_array *tr,
2525                                struct hist_field *operand1,
2526                                struct hist_field *operand2,
2527                                struct hist_field **var1,
2528                                struct hist_field **var2)
2529 {
2530         unsigned long operand1_flags = operand1->flags;
2531         unsigned long operand2_flags = operand2->flags;
2532
2533         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2534             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2535                 struct hist_field *var;
2536
2537                 var = find_var_field(operand1->var.hist_data, operand1->name);
2538                 if (!var)
2539                         return -EINVAL;
2540                 operand1_flags = var->flags;
2541                 *var1 = var;
2542         }
2543
2544         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2545             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2546                 struct hist_field *var;
2547
2548                 var = find_var_field(operand2->var.hist_data, operand2->name);
2549                 if (!var)
2550                         return -EINVAL;
2551                 operand2_flags = var->flags;
2552                 *var2 = var;
2553         }
2554
2555         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2556             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2557                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2558                 return -EINVAL;
2559         }
2560
2561         return 0;
2562 }
2563
2564 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2565                                      struct trace_event_file *file,
2566                                      char *str, unsigned long flags,
2567                                      char *var_name, unsigned int *n_subexprs)
2568 {
2569         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2570         struct hist_field *var1 = NULL, *var2 = NULL;
2571         unsigned long operand_flags, operand2_flags;
2572         int field_op, ret = -EINVAL;
2573         char *sep, *operand1_str;
2574         hist_field_fn_t op_fn;
2575         bool combine_consts;
2576
2577         if (*n_subexprs > 3) {
2578                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2579                 return ERR_PTR(-EINVAL);
2580         }
2581
2582         field_op = contains_operator(str, &sep);
2583
2584         if (field_op == FIELD_OP_NONE)
2585                 return parse_atom(hist_data, file, str, &flags, var_name);
2586
2587         if (field_op == FIELD_OP_UNARY_MINUS)
2588                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2589
2590         /* Binary operator found, increment n_subexprs */
2591         ++*n_subexprs;
2592
2593         /* Split the expression string at the root operator */
2594         if (!sep)
2595                 return ERR_PTR(-EINVAL);
2596
2597         *sep = '\0';
2598         operand1_str = str;
2599         str = sep+1;
2600
2601         /* Binary operator requires both operands */
2602         if (*operand1_str == '\0' || *str == '\0')
2603                 return ERR_PTR(-EINVAL);
2604
2605         operand_flags = 0;
2606
2607         /* LHS of string is an expression e.g. a+b in a+b+c */
2608         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2609         if (IS_ERR(operand1))
2610                 return ERR_CAST(operand1);
2611
2612         if (operand1->flags & HIST_FIELD_FL_STRING) {
2613                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2614                 ret = -EINVAL;
2615                 goto free_op1;
2616         }
2617
2618         /* RHS of string is another expression e.g. c in a+b+c */
2619         operand_flags = 0;
2620         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2621         if (IS_ERR(operand2)) {
2622                 ret = PTR_ERR(operand2);
2623                 goto free_op1;
2624         }
2625         if (operand2->flags & HIST_FIELD_FL_STRING) {
2626                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2627                 ret = -EINVAL;
2628                 goto free_operands;
2629         }
2630
2631         switch (field_op) {
2632         case FIELD_OP_MINUS:
2633                 op_fn = hist_field_minus;
2634                 break;
2635         case FIELD_OP_PLUS:
2636                 op_fn = hist_field_plus;
2637                 break;
2638         case FIELD_OP_DIV:
2639                 op_fn = hist_field_div;
2640                 break;
2641         case FIELD_OP_MULT:
2642                 op_fn = hist_field_mult;
2643                 break;
2644         default:
2645                 ret = -EINVAL;
2646                 goto free_operands;
2647         }
2648
2649         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2650         if (ret)
2651                 goto free_operands;
2652
2653         operand_flags = var1 ? var1->flags : operand1->flags;
2654         operand2_flags = var2 ? var2->flags : operand2->flags;
2655
2656         /*
2657          * If both operands are constant, the expression can be
2658          * collapsed to a single constant.
2659          */
2660         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2661
2662         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2663
2664         flags |= operand1->flags &
2665                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2666
2667         expr = create_hist_field(hist_data, NULL, flags, var_name);
2668         if (!expr) {
2669                 ret = -ENOMEM;
2670                 goto free_operands;
2671         }
2672
2673         operand1->read_once = true;
2674         operand2->read_once = true;
2675
2676         /* The operands are now owned and free'd by 'expr' */
2677         expr->operands[0] = operand1;
2678         expr->operands[1] = operand2;
2679
2680         if (field_op == FIELD_OP_DIV &&
2681                         operand2_flags & HIST_FIELD_FL_CONST) {
2682                 u64 divisor = var2 ? var2->constant : operand2->constant;
2683
2684                 if (!divisor) {
2685                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2686                         ret = -EDOM;
2687                         goto free_expr;
2688                 }
2689
2690                 /*
2691                  * Copy the divisor here so we don't have to look it up
2692                  * later if this is a var ref
2693                  */
2694                 operand2->constant = divisor;
2695                 op_fn = hist_field_get_div_fn(operand2);
2696         }
2697
2698         if (combine_consts) {
2699                 if (var1)
2700                         expr->operands[0] = var1;
2701                 if (var2)
2702                         expr->operands[1] = var2;
2703
2704                 expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2705
2706                 expr->operands[0] = NULL;
2707                 expr->operands[1] = NULL;
2708
2709                 /*
2710                  * var refs won't be destroyed immediately
2711                  * See: destroy_hist_field()
2712                  */
2713                 destroy_hist_field(operand2, 0);
2714                 destroy_hist_field(operand1, 0);
2715
2716                 expr->name = expr_str(expr, 0);
2717         } else {
2718                 expr->fn = op_fn;
2719
2720                 /* The operand sizes should be the same, so just pick one */
2721                 expr->size = operand1->size;
2722
2723                 expr->operator = field_op;
2724                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2725                 if (!expr->type) {
2726                         ret = -ENOMEM;
2727                         goto free_expr;
2728                 }
2729
2730                 expr->name = expr_str(expr, 0);
2731         }
2732
2733         return expr;
2734
2735 free_operands:
2736         destroy_hist_field(operand2, 0);
2737 free_op1:
2738         destroy_hist_field(operand1, 0);
2739         return ERR_PTR(ret);
2740
2741 free_expr:
2742         destroy_hist_field(expr, 0);
2743         return ERR_PTR(ret);
2744 }
2745
2746 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2747                                  struct trace_event_file *file)
2748 {
2749         struct event_trigger_data *test;
2750
2751         lockdep_assert_held(&event_mutex);
2752
2753         list_for_each_entry(test, &file->triggers, list) {
2754                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2755                         if (test->private_data == hist_data)
2756                                 return test->filter_str;
2757                 }
2758         }
2759
2760         return NULL;
2761 }
2762
2763 static struct event_command trigger_hist_cmd;
2764 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2765                                     struct trace_event_file *file,
2766                                     char *glob, char *cmd, char *param);
2767
2768 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2769                             struct hist_trigger_data *hist_data,
2770                             unsigned int n_keys)
2771 {
2772         struct hist_field *target_hist_field, *hist_field;
2773         unsigned int n, i, j;
2774
2775         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2776                 return false;
2777
2778         i = hist_data->n_vals;
2779         j = target_hist_data->n_vals;
2780
2781         for (n = 0; n < n_keys; n++) {
2782                 hist_field = hist_data->fields[i + n];
2783                 target_hist_field = target_hist_data->fields[j + n];
2784
2785                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2786                         return false;
2787                 if (hist_field->size != target_hist_field->size)
2788                         return false;
2789                 if (hist_field->is_signed != target_hist_field->is_signed)
2790                         return false;
2791         }
2792
2793         return true;
2794 }
2795
2796 static struct hist_trigger_data *
2797 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2798                      struct trace_event_file *file)
2799 {
2800         struct hist_trigger_data *hist_data;
2801         struct event_trigger_data *test;
2802         unsigned int n_keys;
2803
2804         lockdep_assert_held(&event_mutex);
2805
2806         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2807
2808         list_for_each_entry(test, &file->triggers, list) {
2809                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2810                         hist_data = test->private_data;
2811
2812                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2813                                 return hist_data;
2814                 }
2815         }
2816
2817         return NULL;
2818 }
2819
2820 static struct trace_event_file *event_file(struct trace_array *tr,
2821                                            char *system, char *event_name)
2822 {
2823         struct trace_event_file *file;
2824
2825         file = __find_event_file(tr, system, event_name);
2826         if (!file)
2827                 return ERR_PTR(-EINVAL);
2828
2829         return file;
2830 }
2831
2832 static struct hist_field *
2833 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2834                          char *system, char *event_name, char *field_name)
2835 {
2836         struct hist_field *event_var;
2837         char *synthetic_name;
2838
2839         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2840         if (!synthetic_name)
2841                 return ERR_PTR(-ENOMEM);
2842
2843         strcpy(synthetic_name, "synthetic_");
2844         strcat(synthetic_name, field_name);
2845
2846         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2847
2848         kfree(synthetic_name);
2849
2850         return event_var;
2851 }
2852
2853 /**
2854  * create_field_var_hist - Automatically create a histogram and var for a field
2855  * @target_hist_data: The target hist trigger
2856  * @subsys_name: Optional subsystem name
2857  * @event_name: Optional event name
2858  * @field_name: The name of the field (and the resulting variable)
2859  *
2860  * Hist trigger actions fetch data from variables, not directly from
2861  * events.  However, for convenience, users are allowed to directly
2862  * specify an event field in an action, which will be automatically
2863  * converted into a variable on their behalf.
2864  *
2865  * If a user specifies a field on an event that isn't the event the
2866  * histogram currently being defined (the target event histogram), the
2867  * only way that can be accomplished is if a new hist trigger is
2868  * created and the field variable defined on that.
2869  *
2870  * This function creates a new histogram compatible with the target
2871  * event (meaning a histogram with the same key as the target
2872  * histogram), and creates a variable for the specified field, but
2873  * with 'synthetic_' prepended to the variable name in order to avoid
2874  * collision with normal field variables.
2875  *
2876  * Return: The variable created for the field.
2877  */
2878 static struct hist_field *
2879 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2880                       char *subsys_name, char *event_name, char *field_name)
2881 {
2882         struct trace_array *tr = target_hist_data->event_file->tr;
2883         struct hist_trigger_data *hist_data;
2884         unsigned int i, n, first = true;
2885         struct field_var_hist *var_hist;
2886         struct trace_event_file *file;
2887         struct hist_field *key_field;
2888         struct hist_field *event_var;
2889         char *saved_filter;
2890         char *cmd;
2891         int ret;
2892
2893         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2894                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2895                 return ERR_PTR(-EINVAL);
2896         }
2897
2898         file = event_file(tr, subsys_name, event_name);
2899
2900         if (IS_ERR(file)) {
2901                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2902                 ret = PTR_ERR(file);
2903                 return ERR_PTR(ret);
2904         }
2905
2906         /*
2907          * Look for a histogram compatible with target.  We'll use the
2908          * found histogram specification to create a new matching
2909          * histogram with our variable on it.  target_hist_data is not
2910          * yet a registered histogram so we can't use that.
2911          */
2912         hist_data = find_compatible_hist(target_hist_data, file);
2913         if (!hist_data) {
2914                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2915                 return ERR_PTR(-EINVAL);
2916         }
2917
2918         /* See if a synthetic field variable has already been created */
2919         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2920                                              event_name, field_name);
2921         if (!IS_ERR_OR_NULL(event_var))
2922                 return event_var;
2923
2924         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2925         if (!var_hist)
2926                 return ERR_PTR(-ENOMEM);
2927
2928         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2929         if (!cmd) {
2930                 kfree(var_hist);
2931                 return ERR_PTR(-ENOMEM);
2932         }
2933
2934         /* Use the same keys as the compatible histogram */
2935         strcat(cmd, "keys=");
2936
2937         for_each_hist_key_field(i, hist_data) {
2938                 key_field = hist_data->fields[i];
2939                 if (!first)
2940                         strcat(cmd, ",");
2941                 strcat(cmd, key_field->field->name);
2942                 first = false;
2943         }
2944
2945         /* Create the synthetic field variable specification */
2946         strcat(cmd, ":synthetic_");
2947         strcat(cmd, field_name);
2948         strcat(cmd, "=");
2949         strcat(cmd, field_name);
2950
2951         /* Use the same filter as the compatible histogram */
2952         saved_filter = find_trigger_filter(hist_data, file);
2953         if (saved_filter) {
2954                 strcat(cmd, " if ");
2955                 strcat(cmd, saved_filter);
2956         }
2957
2958         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2959         if (!var_hist->cmd) {
2960                 kfree(cmd);
2961                 kfree(var_hist);
2962                 return ERR_PTR(-ENOMEM);
2963         }
2964
2965         /* Save the compatible histogram information */
2966         var_hist->hist_data = hist_data;
2967
2968         /* Create the new histogram with our variable */
2969         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
2970                                        "", "hist", cmd);
2971         if (ret) {
2972                 kfree(cmd);
2973                 kfree(var_hist->cmd);
2974                 kfree(var_hist);
2975                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2976                 return ERR_PTR(ret);
2977         }
2978
2979         kfree(cmd);
2980
2981         /* If we can't find the variable, something went wrong */
2982         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2983                                              event_name, field_name);
2984         if (IS_ERR_OR_NULL(event_var)) {
2985                 kfree(var_hist->cmd);
2986                 kfree(var_hist);
2987                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2988                 return ERR_PTR(-EINVAL);
2989         }
2990
2991         n = target_hist_data->n_field_var_hists;
2992         target_hist_data->field_var_hists[n] = var_hist;
2993         target_hist_data->n_field_var_hists++;
2994
2995         return event_var;
2996 }
2997
2998 static struct hist_field *
2999 find_target_event_var(struct hist_trigger_data *hist_data,
3000                       char *subsys_name, char *event_name, char *var_name)
3001 {
3002         struct trace_event_file *file = hist_data->event_file;
3003         struct hist_field *hist_field = NULL;
3004
3005         if (subsys_name) {
3006                 struct trace_event_call *call;
3007
3008                 if (!event_name)
3009                         return NULL;
3010
3011                 call = file->event_call;
3012
3013                 if (strcmp(subsys_name, call->class->system) != 0)
3014                         return NULL;
3015
3016                 if (strcmp(event_name, trace_event_name(call)) != 0)
3017                         return NULL;
3018         }
3019
3020         hist_field = find_var_field(hist_data, var_name);
3021
3022         return hist_field;
3023 }
3024
3025 static inline void __update_field_vars(struct tracing_map_elt *elt,
3026                                        struct trace_buffer *buffer,
3027                                        struct ring_buffer_event *rbe,
3028                                        void *rec,
3029                                        struct field_var **field_vars,
3030                                        unsigned int n_field_vars,
3031                                        unsigned int field_var_str_start)
3032 {
3033         struct hist_elt_data *elt_data = elt->private_data;
3034         unsigned int i, j, var_idx;
3035         u64 var_val;
3036
3037         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3038                 struct field_var *field_var = field_vars[i];
3039                 struct hist_field *var = field_var->var;
3040                 struct hist_field *val = field_var->val;
3041
3042                 var_val = val->fn(val, elt, buffer, rbe, rec);
3043                 var_idx = var->var.idx;
3044
3045                 if (val->flags & HIST_FIELD_FL_STRING) {
3046                         char *str = elt_data->field_var_str[j++];
3047                         char *val_str = (char *)(uintptr_t)var_val;
3048                         unsigned int size;
3049
3050                         size = min(val->size, STR_VAR_LEN_MAX);
3051                         strscpy(str, val_str, size);
3052                         var_val = (u64)(uintptr_t)str;
3053                 }
3054                 tracing_map_set_var(elt, var_idx, var_val);
3055         }
3056 }
3057
3058 static void update_field_vars(struct hist_trigger_data *hist_data,
3059                               struct tracing_map_elt *elt,
3060                               struct trace_buffer *buffer,
3061                               struct ring_buffer_event *rbe,
3062                               void *rec)
3063 {
3064         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3065                             hist_data->n_field_vars, 0);
3066 }
3067
3068 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3069                                  struct tracing_map_elt *elt,
3070                                  struct trace_buffer *buffer,  void *rec,
3071                                  struct ring_buffer_event *rbe, void *key,
3072                                  struct action_data *data, u64 *var_ref_vals)
3073 {
3074         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3075                             hist_data->n_save_vars, hist_data->n_field_var_str);
3076 }
3077
3078 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3079                                      struct trace_event_file *file,
3080                                      char *name, int size, const char *type)
3081 {
3082         struct hist_field *var;
3083         int idx;
3084
3085         if (find_var(hist_data, file, name) && !hist_data->remove) {
3086                 var = ERR_PTR(-EINVAL);
3087                 goto out;
3088         }
3089
3090         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3091         if (!var) {
3092                 var = ERR_PTR(-ENOMEM);
3093                 goto out;
3094         }
3095
3096         idx = tracing_map_add_var(hist_data->map);
3097         if (idx < 0) {
3098                 kfree(var);
3099                 var = ERR_PTR(-EINVAL);
3100                 goto out;
3101         }
3102
3103         var->ref = 1;
3104         var->flags = HIST_FIELD_FL_VAR;
3105         var->var.idx = idx;
3106         var->var.hist_data = var->hist_data = hist_data;
3107         var->size = size;
3108         var->var.name = kstrdup(name, GFP_KERNEL);
3109         var->type = kstrdup_const(type, GFP_KERNEL);
3110         if (!var->var.name || !var->type) {
3111                 kfree_const(var->type);
3112                 kfree(var->var.name);
3113                 kfree(var);
3114                 var = ERR_PTR(-ENOMEM);
3115         }
3116  out:
3117         return var;
3118 }
3119
3120 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3121                                           struct trace_event_file *file,
3122                                           char *field_name)
3123 {
3124         struct hist_field *val = NULL, *var = NULL;
3125         unsigned long flags = HIST_FIELD_FL_VAR;
3126         struct trace_array *tr = file->tr;
3127         struct field_var *field_var;
3128         int ret = 0;
3129
3130         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3131                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3132                 ret = -EINVAL;
3133                 goto err;
3134         }
3135
3136         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3137         if (IS_ERR(val)) {
3138                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3139                 ret = PTR_ERR(val);
3140                 goto err;
3141         }
3142
3143         var = create_var(hist_data, file, field_name, val->size, val->type);
3144         if (IS_ERR(var)) {
3145                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3146                 kfree(val);
3147                 ret = PTR_ERR(var);
3148                 goto err;
3149         }
3150
3151         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3152         if (!field_var) {
3153                 kfree(val);
3154                 kfree(var);
3155                 ret =  -ENOMEM;
3156                 goto err;
3157         }
3158
3159         field_var->var = var;
3160         field_var->val = val;
3161  out:
3162         return field_var;
3163  err:
3164         field_var = ERR_PTR(ret);
3165         goto out;
3166 }
3167
3168 /**
3169  * create_target_field_var - Automatically create a variable for a field
3170  * @target_hist_data: The target hist trigger
3171  * @subsys_name: Optional subsystem name
3172  * @event_name: Optional event name
3173  * @var_name: The name of the field (and the resulting variable)
3174  *
3175  * Hist trigger actions fetch data from variables, not directly from
3176  * events.  However, for convenience, users are allowed to directly
3177  * specify an event field in an action, which will be automatically
3178  * converted into a variable on their behalf.
3179
3180  * This function creates a field variable with the name var_name on
3181  * the hist trigger currently being defined on the target event.  If
3182  * subsys_name and event_name are specified, this function simply
3183  * verifies that they do in fact match the target event subsystem and
3184  * event name.
3185  *
3186  * Return: The variable created for the field.
3187  */
3188 static struct field_var *
3189 create_target_field_var(struct hist_trigger_data *target_hist_data,
3190                         char *subsys_name, char *event_name, char *var_name)
3191 {
3192         struct trace_event_file *file = target_hist_data->event_file;
3193
3194         if (subsys_name) {
3195                 struct trace_event_call *call;
3196
3197                 if (!event_name)
3198                         return NULL;
3199
3200                 call = file->event_call;
3201
3202                 if (strcmp(subsys_name, call->class->system) != 0)
3203                         return NULL;
3204
3205                 if (strcmp(event_name, trace_event_name(call)) != 0)
3206                         return NULL;
3207         }
3208
3209         return create_field_var(target_hist_data, file, var_name);
3210 }
3211
3212 static bool check_track_val_max(u64 track_val, u64 var_val)
3213 {
3214         if (var_val <= track_val)
3215                 return false;
3216
3217         return true;
3218 }
3219
3220 static bool check_track_val_changed(u64 track_val, u64 var_val)
3221 {
3222         if (var_val == track_val)
3223                 return false;
3224
3225         return true;
3226 }
3227
3228 static u64 get_track_val(struct hist_trigger_data *hist_data,
3229                          struct tracing_map_elt *elt,
3230                          struct action_data *data)
3231 {
3232         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3233         u64 track_val;
3234
3235         track_val = tracing_map_read_var(elt, track_var_idx);
3236
3237         return track_val;
3238 }
3239
3240 static void save_track_val(struct hist_trigger_data *hist_data,
3241                            struct tracing_map_elt *elt,
3242                            struct action_data *data, u64 var_val)
3243 {
3244         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3245
3246         tracing_map_set_var(elt, track_var_idx, var_val);
3247 }
3248
3249 static void save_track_data(struct hist_trigger_data *hist_data,
3250                             struct tracing_map_elt *elt,
3251                             struct trace_buffer *buffer, void *rec,
3252                             struct ring_buffer_event *rbe, void *key,
3253                             struct action_data *data, u64 *var_ref_vals)
3254 {
3255         if (data->track_data.save_data)
3256                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3257                                            key, data, var_ref_vals);
3258 }
3259
3260 static bool check_track_val(struct tracing_map_elt *elt,
3261                             struct action_data *data,
3262                             u64 var_val)
3263 {
3264         struct hist_trigger_data *hist_data;
3265         u64 track_val;
3266
3267         hist_data = data->track_data.track_var->hist_data;
3268         track_val = get_track_val(hist_data, elt, data);
3269
3270         return data->track_data.check_val(track_val, var_val);
3271 }
3272
3273 #ifdef CONFIG_TRACER_SNAPSHOT
3274 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3275 {
3276         /* called with tr->max_lock held */
3277         struct track_data *track_data = tr->cond_snapshot->cond_data;
3278         struct hist_elt_data *elt_data, *track_elt_data;
3279         struct snapshot_context *context = cond_data;
3280         struct action_data *action;
3281         u64 track_val;
3282
3283         if (!track_data)
3284                 return false;
3285
3286         action = track_data->action_data;
3287
3288         track_val = get_track_val(track_data->hist_data, context->elt,
3289                                   track_data->action_data);
3290
3291         if (!action->track_data.check_val(track_data->track_val, track_val))
3292                 return false;
3293
3294         track_data->track_val = track_val;
3295         memcpy(track_data->key, context->key, track_data->key_len);
3296
3297         elt_data = context->elt->private_data;
3298         track_elt_data = track_data->elt.private_data;
3299         if (elt_data->comm)
3300                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3301
3302         track_data->updated = true;
3303
3304         return true;
3305 }
3306
3307 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3308                                      struct tracing_map_elt *elt,
3309                                      struct trace_buffer *buffer, void *rec,
3310                                      struct ring_buffer_event *rbe, void *key,
3311                                      struct action_data *data,
3312                                      u64 *var_ref_vals)
3313 {
3314         struct trace_event_file *file = hist_data->event_file;
3315         struct snapshot_context context;
3316
3317         context.elt = elt;
3318         context.key = key;
3319
3320         tracing_snapshot_cond(file->tr, &context);
3321 }
3322
3323 static void hist_trigger_print_key(struct seq_file *m,
3324                                    struct hist_trigger_data *hist_data,
3325                                    void *key,
3326                                    struct tracing_map_elt *elt);
3327
3328 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3329 {
3330         unsigned int i;
3331
3332         if (!hist_data->n_actions)
3333                 return NULL;
3334
3335         for (i = 0; i < hist_data->n_actions; i++) {
3336                 struct action_data *data = hist_data->actions[i];
3337
3338                 if (data->action == ACTION_SNAPSHOT)
3339                         return data;
3340         }
3341
3342         return NULL;
3343 }
3344
3345 static void track_data_snapshot_print(struct seq_file *m,
3346                                       struct hist_trigger_data *hist_data)
3347 {
3348         struct trace_event_file *file = hist_data->event_file;
3349         struct track_data *track_data;
3350         struct action_data *action;
3351
3352         track_data = tracing_cond_snapshot_data(file->tr);
3353         if (!track_data)
3354                 return;
3355
3356         if (!track_data->updated)
3357                 return;
3358
3359         action = snapshot_action(hist_data);
3360         if (!action)
3361                 return;
3362
3363         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3364         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3365                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3366                    action->track_data.var_str, track_data->track_val);
3367
3368         seq_puts(m, "\ttriggered by event with key: ");
3369         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3370         seq_putc(m, '\n');
3371 }
3372 #else
3373 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3374 {
3375         return false;
3376 }
3377 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3378                                      struct tracing_map_elt *elt,
3379                                      struct trace_buffer *buffer, void *rec,
3380                                      struct ring_buffer_event *rbe, void *key,
3381                                      struct action_data *data,
3382                                      u64 *var_ref_vals) {}
3383 static void track_data_snapshot_print(struct seq_file *m,
3384                                       struct hist_trigger_data *hist_data) {}
3385 #endif /* CONFIG_TRACER_SNAPSHOT */
3386
3387 static void track_data_print(struct seq_file *m,
3388                              struct hist_trigger_data *hist_data,
3389                              struct tracing_map_elt *elt,
3390                              struct action_data *data)
3391 {
3392         u64 track_val = get_track_val(hist_data, elt, data);
3393         unsigned int i, save_var_idx;
3394
3395         if (data->handler == HANDLER_ONMAX)
3396                 seq_printf(m, "\n\tmax: %10llu", track_val);
3397         else if (data->handler == HANDLER_ONCHANGE)
3398                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3399
3400         if (data->action == ACTION_SNAPSHOT)
3401                 return;
3402
3403         for (i = 0; i < hist_data->n_save_vars; i++) {
3404                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3405                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3406                 u64 val;
3407
3408                 save_var_idx = save_var->var.idx;
3409
3410                 val = tracing_map_read_var(elt, save_var_idx);
3411
3412                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3413                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3414                                    (char *)(uintptr_t)(val));
3415                 } else
3416                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3417         }
3418 }
3419
3420 static void ontrack_action(struct hist_trigger_data *hist_data,
3421                            struct tracing_map_elt *elt,
3422                            struct trace_buffer *buffer, void *rec,
3423                            struct ring_buffer_event *rbe, void *key,
3424                            struct action_data *data, u64 *var_ref_vals)
3425 {
3426         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3427
3428         if (check_track_val(elt, data, var_val)) {
3429                 save_track_val(hist_data, elt, data, var_val);
3430                 save_track_data(hist_data, elt, buffer, rec, rbe,
3431                                 key, data, var_ref_vals);
3432         }
3433 }
3434
3435 static void action_data_destroy(struct action_data *data)
3436 {
3437         unsigned int i;
3438
3439         lockdep_assert_held(&event_mutex);
3440
3441         kfree(data->action_name);
3442
3443         for (i = 0; i < data->n_params; i++)
3444                 kfree(data->params[i]);
3445
3446         if (data->synth_event)
3447                 data->synth_event->ref--;
3448
3449         kfree(data->synth_event_name);
3450
3451         kfree(data);
3452 }
3453
3454 static void track_data_destroy(struct hist_trigger_data *hist_data,
3455                                struct action_data *data)
3456 {
3457         struct trace_event_file *file = hist_data->event_file;
3458
3459         destroy_hist_field(data->track_data.track_var, 0);
3460
3461         if (data->action == ACTION_SNAPSHOT) {
3462                 struct track_data *track_data;
3463
3464                 track_data = tracing_cond_snapshot_data(file->tr);
3465                 if (track_data && track_data->hist_data == hist_data) {
3466                         tracing_snapshot_cond_disable(file->tr);
3467                         track_data_free(track_data);
3468                 }
3469         }
3470
3471         kfree(data->track_data.var_str);
3472
3473         action_data_destroy(data);
3474 }
3475
3476 static int action_create(struct hist_trigger_data *hist_data,
3477                          struct action_data *data);
3478
3479 static int track_data_create(struct hist_trigger_data *hist_data,
3480                              struct action_data *data)
3481 {
3482         struct hist_field *var_field, *ref_field, *track_var = NULL;
3483         struct trace_event_file *file = hist_data->event_file;
3484         struct trace_array *tr = file->tr;
3485         char *track_data_var_str;
3486         int ret = 0;
3487
3488         track_data_var_str = data->track_data.var_str;
3489         if (track_data_var_str[0] != '$') {
3490                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3491                 return -EINVAL;
3492         }
3493         track_data_var_str++;
3494
3495         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3496         if (!var_field) {
3497                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3498                 return -EINVAL;
3499         }
3500
3501         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3502         if (!ref_field)
3503                 return -ENOMEM;
3504
3505         data->track_data.var_ref = ref_field;
3506
3507         if (data->handler == HANDLER_ONMAX)
3508                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3509         if (IS_ERR(track_var)) {
3510                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3511                 ret = PTR_ERR(track_var);
3512                 goto out;
3513         }
3514
3515         if (data->handler == HANDLER_ONCHANGE)
3516                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3517         if (IS_ERR(track_var)) {
3518                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3519                 ret = PTR_ERR(track_var);
3520                 goto out;
3521         }
3522         data->track_data.track_var = track_var;
3523
3524         ret = action_create(hist_data, data);
3525  out:
3526         return ret;
3527 }
3528
3529 static int parse_action_params(struct trace_array *tr, char *params,
3530                                struct action_data *data)
3531 {
3532         char *param, *saved_param;
3533         bool first_param = true;
3534         int ret = 0;
3535
3536         while (params) {
3537                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3538                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3539                         goto out;
3540                 }
3541
3542                 param = strsep(&params, ",");
3543                 if (!param) {
3544                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3545                         ret = -EINVAL;
3546                         goto out;
3547                 }
3548
3549                 param = strstrip(param);
3550                 if (strlen(param) < 2) {
3551                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3552                         ret = -EINVAL;
3553                         goto out;
3554                 }
3555
3556                 saved_param = kstrdup(param, GFP_KERNEL);
3557                 if (!saved_param) {
3558                         ret = -ENOMEM;
3559                         goto out;
3560                 }
3561
3562                 if (first_param && data->use_trace_keyword) {
3563                         data->synth_event_name = saved_param;
3564                         first_param = false;
3565                         continue;
3566                 }
3567                 first_param = false;
3568
3569                 data->params[data->n_params++] = saved_param;
3570         }
3571  out:
3572         return ret;
3573 }
3574
3575 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3576                         enum handler_id handler)
3577 {
3578         char *action_name;
3579         int ret = 0;
3580
3581         strsep(&str, ".");
3582         if (!str) {
3583                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3584                 ret = -EINVAL;
3585                 goto out;
3586         }
3587
3588         action_name = strsep(&str, "(");
3589         if (!action_name || !str) {
3590                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3591                 ret = -EINVAL;
3592                 goto out;
3593         }
3594
3595         if (str_has_prefix(action_name, "save")) {
3596                 char *params = strsep(&str, ")");
3597
3598                 if (!params) {
3599                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3600                         ret = -EINVAL;
3601                         goto out;
3602                 }
3603
3604                 ret = parse_action_params(tr, params, data);
3605                 if (ret)
3606                         goto out;
3607
3608                 if (handler == HANDLER_ONMAX)
3609                         data->track_data.check_val = check_track_val_max;
3610                 else if (handler == HANDLER_ONCHANGE)
3611                         data->track_data.check_val = check_track_val_changed;
3612                 else {
3613                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3614                         ret = -EINVAL;
3615                         goto out;
3616                 }
3617
3618                 data->track_data.save_data = save_track_data_vars;
3619                 data->fn = ontrack_action;
3620                 data->action = ACTION_SAVE;
3621         } else if (str_has_prefix(action_name, "snapshot")) {
3622                 char *params = strsep(&str, ")");
3623
3624                 if (!str) {
3625                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3626                         ret = -EINVAL;
3627                         goto out;
3628                 }
3629
3630                 if (handler == HANDLER_ONMAX)
3631                         data->track_data.check_val = check_track_val_max;
3632                 else if (handler == HANDLER_ONCHANGE)
3633                         data->track_data.check_val = check_track_val_changed;
3634                 else {
3635                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3636                         ret = -EINVAL;
3637                         goto out;
3638                 }
3639
3640                 data->track_data.save_data = save_track_data_snapshot;
3641                 data->fn = ontrack_action;
3642                 data->action = ACTION_SNAPSHOT;
3643         } else {
3644                 char *params = strsep(&str, ")");
3645
3646                 if (str_has_prefix(action_name, "trace"))
3647                         data->use_trace_keyword = true;
3648
3649                 if (params) {
3650                         ret = parse_action_params(tr, params, data);
3651                         if (ret)
3652                                 goto out;
3653                 }
3654
3655                 if (handler == HANDLER_ONMAX)
3656                         data->track_data.check_val = check_track_val_max;
3657                 else if (handler == HANDLER_ONCHANGE)
3658                         data->track_data.check_val = check_track_val_changed;
3659
3660                 if (handler != HANDLER_ONMATCH) {
3661                         data->track_data.save_data = action_trace;
3662                         data->fn = ontrack_action;
3663                 } else
3664                         data->fn = action_trace;
3665
3666                 data->action = ACTION_TRACE;
3667         }
3668
3669         data->action_name = kstrdup(action_name, GFP_KERNEL);
3670         if (!data->action_name) {
3671                 ret = -ENOMEM;
3672                 goto out;
3673         }
3674
3675         data->handler = handler;
3676  out:
3677         return ret;
3678 }
3679
3680 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3681                                             char *str, enum handler_id handler)
3682 {
3683         struct action_data *data;
3684         int ret = -EINVAL;
3685         char *var_str;
3686
3687         data = kzalloc(sizeof(*data), GFP_KERNEL);
3688         if (!data)
3689                 return ERR_PTR(-ENOMEM);
3690
3691         var_str = strsep(&str, ")");
3692         if (!var_str || !str) {
3693                 ret = -EINVAL;
3694                 goto free;
3695         }
3696
3697         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3698         if (!data->track_data.var_str) {
3699                 ret = -ENOMEM;
3700                 goto free;
3701         }
3702
3703         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3704         if (ret)
3705                 goto free;
3706  out:
3707         return data;
3708  free:
3709         track_data_destroy(hist_data, data);
3710         data = ERR_PTR(ret);
3711         goto out;
3712 }
3713
3714 static void onmatch_destroy(struct action_data *data)
3715 {
3716         kfree(data->match_data.event);
3717         kfree(data->match_data.event_system);
3718
3719         action_data_destroy(data);
3720 }
3721
3722 static void destroy_field_var(struct field_var *field_var)
3723 {
3724         if (!field_var)
3725                 return;
3726
3727         destroy_hist_field(field_var->var, 0);
3728         destroy_hist_field(field_var->val, 0);
3729
3730         kfree(field_var);
3731 }
3732
3733 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3734 {
3735         unsigned int i;
3736
3737         for (i = 0; i < hist_data->n_field_vars; i++)
3738                 destroy_field_var(hist_data->field_vars[i]);
3739
3740         for (i = 0; i < hist_data->n_save_vars; i++)
3741                 destroy_field_var(hist_data->save_vars[i]);
3742 }
3743
3744 static void save_field_var(struct hist_trigger_data *hist_data,
3745                            struct field_var *field_var)
3746 {
3747         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3748
3749         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3750                 hist_data->n_field_var_str++;
3751 }
3752
3753
3754 static int check_synth_field(struct synth_event *event,
3755                              struct hist_field *hist_field,
3756                              unsigned int field_pos)
3757 {
3758         struct synth_field *field;
3759
3760         if (field_pos >= event->n_fields)
3761                 return -EINVAL;
3762
3763         field = event->fields[field_pos];
3764
3765         /*
3766          * A dynamic string synth field can accept static or
3767          * dynamic. A static string synth field can only accept a
3768          * same-sized static string, which is checked for later.
3769          */
3770         if (strstr(hist_field->type, "char[") && field->is_string
3771             && field->is_dynamic)
3772                 return 0;
3773
3774         if (strcmp(field->type, hist_field->type) != 0) {
3775                 if (field->size != hist_field->size ||
3776                     (!field->is_string && field->is_signed != hist_field->is_signed))
3777                         return -EINVAL;
3778         }
3779
3780         return 0;
3781 }
3782
3783 static struct hist_field *
3784 trace_action_find_var(struct hist_trigger_data *hist_data,
3785                       struct action_data *data,
3786                       char *system, char *event, char *var)
3787 {
3788         struct trace_array *tr = hist_data->event_file->tr;
3789         struct hist_field *hist_field;
3790
3791         var++; /* skip '$' */
3792
3793         hist_field = find_target_event_var(hist_data, system, event, var);
3794         if (!hist_field) {
3795                 if (!system && data->handler == HANDLER_ONMATCH) {
3796                         system = data->match_data.event_system;
3797                         event = data->match_data.event;
3798                 }
3799
3800                 hist_field = find_event_var(hist_data, system, event, var);
3801         }
3802
3803         if (!hist_field)
3804                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3805
3806         return hist_field;
3807 }
3808
3809 static struct hist_field *
3810 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3811                               struct action_data *data, char *system,
3812                               char *event, char *var)
3813 {
3814         struct hist_field *hist_field = NULL;
3815         struct field_var *field_var;
3816
3817         /*
3818          * First try to create a field var on the target event (the
3819          * currently being defined).  This will create a variable for
3820          * unqualified fields on the target event, or if qualified,
3821          * target fields that have qualified names matching the target.
3822          */
3823         field_var = create_target_field_var(hist_data, system, event, var);
3824
3825         if (field_var && !IS_ERR(field_var)) {
3826                 save_field_var(hist_data, field_var);
3827                 hist_field = field_var->var;
3828         } else {
3829                 field_var = NULL;
3830                 /*
3831                  * If no explicit system.event is specified, default to
3832                  * looking for fields on the onmatch(system.event.xxx)
3833                  * event.
3834                  */
3835                 if (!system && data->handler == HANDLER_ONMATCH) {
3836                         system = data->match_data.event_system;
3837                         event = data->match_data.event;
3838                 }
3839
3840                 if (!event)
3841                         goto free;
3842                 /*
3843                  * At this point, we're looking at a field on another
3844                  * event.  Because we can't modify a hist trigger on
3845                  * another event to add a variable for a field, we need
3846                  * to create a new trigger on that event and create the
3847                  * variable at the same time.
3848                  */
3849                 hist_field = create_field_var_hist(hist_data, system, event, var);
3850                 if (IS_ERR(hist_field))
3851                         goto free;
3852         }
3853  out:
3854         return hist_field;
3855  free:
3856         destroy_field_var(field_var);
3857         hist_field = NULL;
3858         goto out;
3859 }
3860
3861 static int trace_action_create(struct hist_trigger_data *hist_data,
3862                                struct action_data *data)
3863 {
3864         struct trace_array *tr = hist_data->event_file->tr;
3865         char *event_name, *param, *system = NULL;
3866         struct hist_field *hist_field, *var_ref;
3867         unsigned int i;
3868         unsigned int field_pos = 0;
3869         struct synth_event *event;
3870         char *synth_event_name;
3871         int var_ref_idx, ret = 0;
3872
3873         lockdep_assert_held(&event_mutex);
3874
3875         if (data->use_trace_keyword)
3876                 synth_event_name = data->synth_event_name;
3877         else
3878                 synth_event_name = data->action_name;
3879
3880         event = find_synth_event(synth_event_name);
3881         if (!event) {
3882                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3883                 return -EINVAL;
3884         }
3885
3886         event->ref++;
3887
3888         for (i = 0; i < data->n_params; i++) {
3889                 char *p;
3890
3891                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3892                 if (!param) {
3893                         ret = -ENOMEM;
3894                         goto err;
3895                 }
3896
3897                 system = strsep(&param, ".");
3898                 if (!param) {
3899                         param = (char *)system;
3900                         system = event_name = NULL;
3901                 } else {
3902                         event_name = strsep(&param, ".");
3903                         if (!param) {
3904                                 kfree(p);
3905                                 ret = -EINVAL;
3906                                 goto err;
3907                         }
3908                 }
3909
3910                 if (param[0] == '$')
3911                         hist_field = trace_action_find_var(hist_data, data,
3912                                                            system, event_name,
3913                                                            param);
3914                 else
3915                         hist_field = trace_action_create_field_var(hist_data,
3916                                                                    data,
3917                                                                    system,
3918                                                                    event_name,
3919                                                                    param);
3920
3921                 if (!hist_field) {
3922                         kfree(p);
3923                         ret = -EINVAL;
3924                         goto err;
3925                 }
3926
3927                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3928                         var_ref = create_var_ref(hist_data, hist_field,
3929                                                  system, event_name);
3930                         if (!var_ref) {
3931                                 kfree(p);
3932                                 ret = -ENOMEM;
3933                                 goto err;
3934                         }
3935
3936                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3937                         if (WARN_ON(var_ref_idx < 0)) {
3938                                 ret = var_ref_idx;
3939                                 goto err;
3940                         }
3941
3942                         data->var_ref_idx[i] = var_ref_idx;
3943
3944                         field_pos++;
3945                         kfree(p);
3946                         continue;
3947                 }
3948
3949                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3950                 kfree(p);
3951                 ret = -EINVAL;
3952                 goto err;
3953         }
3954
3955         if (field_pos != event->n_fields) {
3956                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3957                 ret = -EINVAL;
3958                 goto err;
3959         }
3960
3961         data->synth_event = event;
3962  out:
3963         return ret;
3964  err:
3965         event->ref--;
3966
3967         goto out;
3968 }
3969
3970 static int action_create(struct hist_trigger_data *hist_data,
3971                          struct action_data *data)
3972 {
3973         struct trace_event_file *file = hist_data->event_file;
3974         struct trace_array *tr = file->tr;
3975         struct track_data *track_data;
3976         struct field_var *field_var;
3977         unsigned int i;
3978         char *param;
3979         int ret = 0;
3980
3981         if (data->action == ACTION_TRACE)
3982                 return trace_action_create(hist_data, data);
3983
3984         if (data->action == ACTION_SNAPSHOT) {
3985                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3986                 if (IS_ERR(track_data)) {
3987                         ret = PTR_ERR(track_data);
3988                         goto out;
3989                 }
3990
3991                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3992                                                    cond_snapshot_update);
3993                 if (ret)
3994                         track_data_free(track_data);
3995
3996                 goto out;
3997         }
3998
3999         if (data->action == ACTION_SAVE) {
4000                 if (hist_data->n_save_vars) {
4001                         ret = -EEXIST;
4002                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4003                         goto out;
4004                 }
4005
4006                 for (i = 0; i < data->n_params; i++) {
4007                         param = kstrdup(data->params[i], GFP_KERNEL);
4008                         if (!param) {
4009                                 ret = -ENOMEM;
4010                                 goto out;
4011                         }
4012
4013                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4014                         if (IS_ERR(field_var)) {
4015                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4016                                          errpos(param));
4017                                 ret = PTR_ERR(field_var);
4018                                 kfree(param);
4019                                 goto out;
4020                         }
4021
4022                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4023                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4024                                 hist_data->n_save_var_str++;
4025                         kfree(param);
4026                 }
4027         }
4028  out:
4029         return ret;
4030 }
4031
4032 static int onmatch_create(struct hist_trigger_data *hist_data,
4033                           struct action_data *data)
4034 {
4035         return action_create(hist_data, data);
4036 }
4037
4038 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4039 {
4040         char *match_event, *match_event_system;
4041         struct action_data *data;
4042         int ret = -EINVAL;
4043
4044         data = kzalloc(sizeof(*data), GFP_KERNEL);
4045         if (!data)
4046                 return ERR_PTR(-ENOMEM);
4047
4048         match_event = strsep(&str, ")");
4049         if (!match_event || !str) {
4050                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4051                 goto free;
4052         }
4053
4054         match_event_system = strsep(&match_event, ".");
4055         if (!match_event) {
4056                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4057                 goto free;
4058         }
4059
4060         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4061                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4062                 goto free;
4063         }
4064
4065         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4066         if (!data->match_data.event) {
4067                 ret = -ENOMEM;
4068                 goto free;
4069         }
4070
4071         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4072         if (!data->match_data.event_system) {
4073                 ret = -ENOMEM;
4074                 goto free;
4075         }
4076
4077         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4078         if (ret)
4079                 goto free;
4080  out:
4081         return data;
4082  free:
4083         onmatch_destroy(data);
4084         data = ERR_PTR(ret);
4085         goto out;
4086 }
4087
4088 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4089 {
4090         hist_data->fields[HITCOUNT_IDX] =
4091                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4092         if (!hist_data->fields[HITCOUNT_IDX])
4093                 return -ENOMEM;
4094
4095         hist_data->n_vals++;
4096         hist_data->n_fields++;
4097
4098         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4099                 return -EINVAL;
4100
4101         return 0;
4102 }
4103
4104 static int __create_val_field(struct hist_trigger_data *hist_data,
4105                               unsigned int val_idx,
4106                               struct trace_event_file *file,
4107                               char *var_name, char *field_str,
4108                               unsigned long flags)
4109 {
4110         struct hist_field *hist_field;
4111         int ret = 0, n_subexprs = 0;
4112
4113         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4114         if (IS_ERR(hist_field)) {
4115                 ret = PTR_ERR(hist_field);
4116                 goto out;
4117         }
4118
4119         hist_data->fields[val_idx] = hist_field;
4120
4121         ++hist_data->n_vals;
4122         ++hist_data->n_fields;
4123
4124         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4125                 ret = -EINVAL;
4126  out:
4127         return ret;
4128 }
4129
4130 static int create_val_field(struct hist_trigger_data *hist_data,
4131                             unsigned int val_idx,
4132                             struct trace_event_file *file,
4133                             char *field_str)
4134 {
4135         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4136                 return -EINVAL;
4137
4138         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4139 }
4140
4141 static const char *no_comm = "(no comm)";
4142
4143 static u64 hist_field_execname(struct hist_field *hist_field,
4144                                struct tracing_map_elt *elt,
4145                                struct trace_buffer *buffer,
4146                                struct ring_buffer_event *rbe,
4147                                void *event)
4148 {
4149         struct hist_elt_data *elt_data;
4150
4151         if (WARN_ON_ONCE(!elt))
4152                 return (u64)(unsigned long)no_comm;
4153
4154         elt_data = elt->private_data;
4155
4156         if (WARN_ON_ONCE(!elt_data->comm))
4157                 return (u64)(unsigned long)no_comm;
4158
4159         return (u64)(unsigned long)(elt_data->comm);
4160 }
4161
4162 /* Convert a var that points to common_pid.execname to a string */
4163 static void update_var_execname(struct hist_field *hist_field)
4164 {
4165         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4166                 HIST_FIELD_FL_EXECNAME;
4167         hist_field->size = MAX_FILTER_STR_VAL;
4168         hist_field->is_signed = 0;
4169
4170         kfree_const(hist_field->type);
4171         hist_field->type = "char[]";
4172
4173         hist_field->fn = hist_field_execname;
4174 }
4175
4176 static int create_var_field(struct hist_trigger_data *hist_data,
4177                             unsigned int val_idx,
4178                             struct trace_event_file *file,
4179                             char *var_name, char *expr_str)
4180 {
4181         struct trace_array *tr = hist_data->event_file->tr;
4182         unsigned long flags = 0;
4183         int ret;
4184
4185         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4186                 return -EINVAL;
4187
4188         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4189                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4190                 return -EINVAL;
4191         }
4192
4193         flags |= HIST_FIELD_FL_VAR;
4194         hist_data->n_vars++;
4195         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4196                 return -EINVAL;
4197
4198         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4199
4200         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4201                 update_var_execname(hist_data->fields[val_idx]);
4202
4203         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4204                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4205
4206         return ret;
4207 }
4208
4209 static int create_val_fields(struct hist_trigger_data *hist_data,
4210                              struct trace_event_file *file)
4211 {
4212         char *fields_str, *field_str;
4213         unsigned int i, j = 1;
4214         int ret;
4215
4216         ret = create_hitcount_val(hist_data);
4217         if (ret)
4218                 goto out;
4219
4220         fields_str = hist_data->attrs->vals_str;
4221         if (!fields_str)
4222                 goto out;
4223
4224         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4225                      j < TRACING_MAP_VALS_MAX; i++) {
4226                 field_str = strsep(&fields_str, ",");
4227                 if (!field_str)
4228                         break;
4229
4230                 if (strcmp(field_str, "hitcount") == 0)
4231                         continue;
4232
4233                 ret = create_val_field(hist_data, j++, file, field_str);
4234                 if (ret)
4235                         goto out;
4236         }
4237
4238         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4239                 ret = -EINVAL;
4240  out:
4241         return ret;
4242 }
4243
4244 static int create_key_field(struct hist_trigger_data *hist_data,
4245                             unsigned int key_idx,
4246                             unsigned int key_offset,
4247                             struct trace_event_file *file,
4248                             char *field_str)
4249 {
4250         struct trace_array *tr = hist_data->event_file->tr;
4251         struct hist_field *hist_field = NULL;
4252         unsigned long flags = 0;
4253         unsigned int key_size;
4254         int ret = 0, n_subexprs = 0;
4255
4256         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4257                 return -EINVAL;
4258
4259         flags |= HIST_FIELD_FL_KEY;
4260
4261         if (strcmp(field_str, "stacktrace") == 0) {
4262                 flags |= HIST_FIELD_FL_STACKTRACE;
4263                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4264                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4265         } else {
4266                 hist_field = parse_expr(hist_data, file, field_str, flags,
4267                                         NULL, &n_subexprs);
4268                 if (IS_ERR(hist_field)) {
4269                         ret = PTR_ERR(hist_field);
4270                         goto out;
4271                 }
4272
4273                 if (field_has_hist_vars(hist_field, 0)) {
4274                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4275                         destroy_hist_field(hist_field, 0);
4276                         ret = -EINVAL;
4277                         goto out;
4278                 }
4279
4280                 key_size = hist_field->size;
4281         }
4282
4283         hist_data->fields[key_idx] = hist_field;
4284
4285         key_size = ALIGN(key_size, sizeof(u64));
4286         hist_data->fields[key_idx]->size = key_size;
4287         hist_data->fields[key_idx]->offset = key_offset;
4288
4289         hist_data->key_size += key_size;
4290
4291         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4292                 ret = -EINVAL;
4293                 goto out;
4294         }
4295
4296         hist_data->n_keys++;
4297         hist_data->n_fields++;
4298
4299         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4300                 return -EINVAL;
4301
4302         ret = key_size;
4303  out:
4304         return ret;
4305 }
4306
4307 static int create_key_fields(struct hist_trigger_data *hist_data,
4308                              struct trace_event_file *file)
4309 {
4310         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4311         char *fields_str, *field_str;
4312         int ret = -EINVAL;
4313
4314         fields_str = hist_data->attrs->keys_str;
4315         if (!fields_str)
4316                 goto out;
4317
4318         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4319                 field_str = strsep(&fields_str, ",");
4320                 if (!field_str)
4321                         break;
4322                 ret = create_key_field(hist_data, i, key_offset,
4323                                        file, field_str);
4324                 if (ret < 0)
4325                         goto out;
4326                 key_offset += ret;
4327         }
4328         if (fields_str) {
4329                 ret = -EINVAL;
4330                 goto out;
4331         }
4332         ret = 0;
4333  out:
4334         return ret;
4335 }
4336
4337 static int create_var_fields(struct hist_trigger_data *hist_data,
4338                              struct trace_event_file *file)
4339 {
4340         unsigned int i, j = hist_data->n_vals;
4341         int ret = 0;
4342
4343         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4344
4345         for (i = 0; i < n_vars; i++) {
4346                 char *var_name = hist_data->attrs->var_defs.name[i];
4347                 char *expr = hist_data->attrs->var_defs.expr[i];
4348
4349                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4350                 if (ret)
4351                         goto out;
4352         }
4353  out:
4354         return ret;
4355 }
4356
4357 static void free_var_defs(struct hist_trigger_data *hist_data)
4358 {
4359         unsigned int i;
4360
4361         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4362                 kfree(hist_data->attrs->var_defs.name[i]);
4363                 kfree(hist_data->attrs->var_defs.expr[i]);
4364         }
4365
4366         hist_data->attrs->var_defs.n_vars = 0;
4367 }
4368
4369 static int parse_var_defs(struct hist_trigger_data *hist_data)
4370 {
4371         struct trace_array *tr = hist_data->event_file->tr;
4372         char *s, *str, *var_name, *field_str;
4373         unsigned int i, j, n_vars = 0;
4374         int ret = 0;
4375
4376         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4377                 str = hist_data->attrs->assignment_str[i];
4378                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4379                         field_str = strsep(&str, ",");
4380                         if (!field_str)
4381                                 break;
4382
4383                         var_name = strsep(&field_str, "=");
4384                         if (!var_name || !field_str) {
4385                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4386                                          errpos(var_name));
4387                                 ret = -EINVAL;
4388                                 goto free;
4389                         }
4390
4391                         if (n_vars == TRACING_MAP_VARS_MAX) {
4392                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4393                                 ret = -EINVAL;
4394                                 goto free;
4395                         }
4396
4397                         s = kstrdup(var_name, GFP_KERNEL);
4398                         if (!s) {
4399                                 ret = -ENOMEM;
4400                                 goto free;
4401                         }
4402                         hist_data->attrs->var_defs.name[n_vars] = s;
4403
4404                         s = kstrdup(field_str, GFP_KERNEL);
4405                         if (!s) {
4406                                 ret = -ENOMEM;
4407                                 goto free;
4408                         }
4409                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4410
4411                         hist_data->attrs->var_defs.n_vars = n_vars;
4412                 }
4413         }
4414
4415         return ret;
4416  free:
4417         free_var_defs(hist_data);
4418
4419         return ret;
4420 }
4421
4422 static int create_hist_fields(struct hist_trigger_data *hist_data,
4423                               struct trace_event_file *file)
4424 {
4425         int ret;
4426
4427         ret = parse_var_defs(hist_data);
4428         if (ret)
4429                 goto out;
4430
4431         ret = create_val_fields(hist_data, file);
4432         if (ret)
4433                 goto out;
4434
4435         ret = create_var_fields(hist_data, file);
4436         if (ret)
4437                 goto out;
4438
4439         ret = create_key_fields(hist_data, file);
4440         if (ret)
4441                 goto out;
4442  out:
4443         free_var_defs(hist_data);
4444
4445         return ret;
4446 }
4447
4448 static int is_descending(struct trace_array *tr, const char *str)
4449 {
4450         if (!str)
4451                 return 0;
4452
4453         if (strcmp(str, "descending") == 0)
4454                 return 1;
4455
4456         if (strcmp(str, "ascending") == 0)
4457                 return 0;
4458
4459         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4460
4461         return -EINVAL;
4462 }
4463
4464 static int create_sort_keys(struct hist_trigger_data *hist_data)
4465 {
4466         struct trace_array *tr = hist_data->event_file->tr;
4467         char *fields_str = hist_data->attrs->sort_key_str;
4468         struct tracing_map_sort_key *sort_key;
4469         int descending, ret = 0;
4470         unsigned int i, j, k;
4471
4472         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4473
4474         if (!fields_str)
4475                 goto out;
4476
4477         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4478                 struct hist_field *hist_field;
4479                 char *field_str, *field_name;
4480                 const char *test_name;
4481
4482                 sort_key = &hist_data->sort_keys[i];
4483
4484                 field_str = strsep(&fields_str, ",");
4485                 if (!field_str)
4486                         break;
4487
4488                 if (!*field_str) {
4489                         ret = -EINVAL;
4490                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4491                         break;
4492                 }
4493
4494                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4495                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4496                         ret = -EINVAL;
4497                         break;
4498                 }
4499
4500                 field_name = strsep(&field_str, ".");
4501                 if (!field_name || !*field_name) {
4502                         ret = -EINVAL;
4503                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4504                         break;
4505                 }
4506
4507                 if (strcmp(field_name, "hitcount") == 0) {
4508                         descending = is_descending(tr, field_str);
4509                         if (descending < 0) {
4510                                 ret = descending;
4511                                 break;
4512                         }
4513                         sort_key->descending = descending;
4514                         continue;
4515                 }
4516
4517                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4518                         unsigned int idx;
4519
4520                         hist_field = hist_data->fields[j];
4521                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4522                                 continue;
4523
4524                         idx = k++;
4525
4526                         test_name = hist_field_name(hist_field, 0);
4527
4528                         if (strcmp(field_name, test_name) == 0) {
4529                                 sort_key->field_idx = idx;
4530                                 descending = is_descending(tr, field_str);
4531                                 if (descending < 0) {
4532                                         ret = descending;
4533                                         goto out;
4534                                 }
4535                                 sort_key->descending = descending;
4536                                 break;
4537                         }
4538                 }
4539                 if (j == hist_data->n_fields) {
4540                         ret = -EINVAL;
4541                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4542                         break;
4543                 }
4544         }
4545
4546         hist_data->n_sort_keys = i;
4547  out:
4548         return ret;
4549 }
4550
4551 static void destroy_actions(struct hist_trigger_data *hist_data)
4552 {
4553         unsigned int i;
4554
4555         for (i = 0; i < hist_data->n_actions; i++) {
4556                 struct action_data *data = hist_data->actions[i];
4557
4558                 if (data->handler == HANDLER_ONMATCH)
4559                         onmatch_destroy(data);
4560                 else if (data->handler == HANDLER_ONMAX ||
4561                          data->handler == HANDLER_ONCHANGE)
4562                         track_data_destroy(hist_data, data);
4563                 else
4564                         kfree(data);
4565         }
4566 }
4567
4568 static int parse_actions(struct hist_trigger_data *hist_data)
4569 {
4570         struct trace_array *tr = hist_data->event_file->tr;
4571         struct action_data *data;
4572         unsigned int i;
4573         int ret = 0;
4574         char *str;
4575         int len;
4576
4577         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4578                 str = hist_data->attrs->action_str[i];
4579
4580                 if ((len = str_has_prefix(str, "onmatch("))) {
4581                         char *action_str = str + len;
4582
4583                         data = onmatch_parse(tr, action_str);
4584                         if (IS_ERR(data)) {
4585                                 ret = PTR_ERR(data);
4586                                 break;
4587                         }
4588                 } else if ((len = str_has_prefix(str, "onmax("))) {
4589                         char *action_str = str + len;
4590
4591                         data = track_data_parse(hist_data, action_str,
4592                                                 HANDLER_ONMAX);
4593                         if (IS_ERR(data)) {
4594                                 ret = PTR_ERR(data);
4595                                 break;
4596                         }
4597                 } else if ((len = str_has_prefix(str, "onchange("))) {
4598                         char *action_str = str + len;
4599
4600                         data = track_data_parse(hist_data, action_str,
4601                                                 HANDLER_ONCHANGE);
4602                         if (IS_ERR(data)) {
4603                                 ret = PTR_ERR(data);
4604                                 break;
4605                         }
4606                 } else {
4607                         ret = -EINVAL;
4608                         break;
4609                 }
4610
4611                 hist_data->actions[hist_data->n_actions++] = data;
4612         }
4613
4614         return ret;
4615 }
4616
4617 static int create_actions(struct hist_trigger_data *hist_data)
4618 {
4619         struct action_data *data;
4620         unsigned int i;
4621         int ret = 0;
4622
4623         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4624                 data = hist_data->actions[i];
4625
4626                 if (data->handler == HANDLER_ONMATCH) {
4627                         ret = onmatch_create(hist_data, data);
4628                         if (ret)
4629                                 break;
4630                 } else if (data->handler == HANDLER_ONMAX ||
4631                            data->handler == HANDLER_ONCHANGE) {
4632                         ret = track_data_create(hist_data, data);
4633                         if (ret)
4634                                 break;
4635                 } else {
4636                         ret = -EINVAL;
4637                         break;
4638                 }
4639         }
4640
4641         return ret;
4642 }
4643
4644 static void print_actions(struct seq_file *m,
4645                           struct hist_trigger_data *hist_data,
4646                           struct tracing_map_elt *elt)
4647 {
4648         unsigned int i;
4649
4650         for (i = 0; i < hist_data->n_actions; i++) {
4651                 struct action_data *data = hist_data->actions[i];
4652
4653                 if (data->action == ACTION_SNAPSHOT)
4654                         continue;
4655
4656                 if (data->handler == HANDLER_ONMAX ||
4657                     data->handler == HANDLER_ONCHANGE)
4658                         track_data_print(m, hist_data, elt, data);
4659         }
4660 }
4661
4662 static void print_action_spec(struct seq_file *m,
4663                               struct hist_trigger_data *hist_data,
4664                               struct action_data *data)
4665 {
4666         unsigned int i;
4667
4668         if (data->action == ACTION_SAVE) {
4669                 for (i = 0; i < hist_data->n_save_vars; i++) {
4670                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4671                         if (i < hist_data->n_save_vars - 1)
4672                                 seq_puts(m, ",");
4673                 }
4674         } else if (data->action == ACTION_TRACE) {
4675                 if (data->use_trace_keyword)
4676                         seq_printf(m, "%s", data->synth_event_name);
4677                 for (i = 0; i < data->n_params; i++) {
4678                         if (i || data->use_trace_keyword)
4679                                 seq_puts(m, ",");
4680                         seq_printf(m, "%s", data->params[i]);
4681                 }
4682         }
4683 }
4684
4685 static void print_track_data_spec(struct seq_file *m,
4686                                   struct hist_trigger_data *hist_data,
4687                                   struct action_data *data)
4688 {
4689         if (data->handler == HANDLER_ONMAX)
4690                 seq_puts(m, ":onmax(");
4691         else if (data->handler == HANDLER_ONCHANGE)
4692                 seq_puts(m, ":onchange(");
4693         seq_printf(m, "%s", data->track_data.var_str);
4694         seq_printf(m, ").%s(", data->action_name);
4695
4696         print_action_spec(m, hist_data, data);
4697
4698         seq_puts(m, ")");
4699 }
4700
4701 static void print_onmatch_spec(struct seq_file *m,
4702                                struct hist_trigger_data *hist_data,
4703                                struct action_data *data)
4704 {
4705         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4706                    data->match_data.event);
4707
4708         seq_printf(m, "%s(", data->action_name);
4709
4710         print_action_spec(m, hist_data, data);
4711
4712         seq_puts(m, ")");
4713 }
4714
4715 static bool actions_match(struct hist_trigger_data *hist_data,
4716                           struct hist_trigger_data *hist_data_test)
4717 {
4718         unsigned int i, j;
4719
4720         if (hist_data->n_actions != hist_data_test->n_actions)
4721                 return false;
4722
4723         for (i = 0; i < hist_data->n_actions; i++) {
4724                 struct action_data *data = hist_data->actions[i];
4725                 struct action_data *data_test = hist_data_test->actions[i];
4726                 char *action_name, *action_name_test;
4727
4728                 if (data->handler != data_test->handler)
4729                         return false;
4730                 if (data->action != data_test->action)
4731                         return false;
4732
4733                 if (data->n_params != data_test->n_params)
4734                         return false;
4735
4736                 for (j = 0; j < data->n_params; j++) {
4737                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4738                                 return false;
4739                 }
4740
4741                 if (data->use_trace_keyword)
4742                         action_name = data->synth_event_name;
4743                 else
4744                         action_name = data->action_name;
4745
4746                 if (data_test->use_trace_keyword)
4747                         action_name_test = data_test->synth_event_name;
4748                 else
4749                         action_name_test = data_test->action_name;
4750
4751                 if (strcmp(action_name, action_name_test) != 0)
4752                         return false;
4753
4754                 if (data->handler == HANDLER_ONMATCH) {
4755                         if (strcmp(data->match_data.event_system,
4756                                    data_test->match_data.event_system) != 0)
4757                                 return false;
4758                         if (strcmp(data->match_data.event,
4759                                    data_test->match_data.event) != 0)
4760                                 return false;
4761                 } else if (data->handler == HANDLER_ONMAX ||
4762                            data->handler == HANDLER_ONCHANGE) {
4763                         if (strcmp(data->track_data.var_str,
4764                                    data_test->track_data.var_str) != 0)
4765                                 return false;
4766                 }
4767         }
4768
4769         return true;
4770 }
4771
4772
4773 static void print_actions_spec(struct seq_file *m,
4774                                struct hist_trigger_data *hist_data)
4775 {
4776         unsigned int i;
4777
4778         for (i = 0; i < hist_data->n_actions; i++) {
4779                 struct action_data *data = hist_data->actions[i];
4780
4781                 if (data->handler == HANDLER_ONMATCH)
4782                         print_onmatch_spec(m, hist_data, data);
4783                 else if (data->handler == HANDLER_ONMAX ||
4784                          data->handler == HANDLER_ONCHANGE)
4785                         print_track_data_spec(m, hist_data, data);
4786         }
4787 }
4788
4789 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4790 {
4791         unsigned int i;
4792
4793         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4794                 kfree(hist_data->field_var_hists[i]->cmd);
4795                 kfree(hist_data->field_var_hists[i]);
4796         }
4797 }
4798
4799 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4800 {
4801         if (!hist_data)
4802                 return;
4803
4804         destroy_hist_trigger_attrs(hist_data->attrs);
4805         destroy_hist_fields(hist_data);
4806         tracing_map_destroy(hist_data->map);
4807
4808         destroy_actions(hist_data);
4809         destroy_field_vars(hist_data);
4810         destroy_field_var_hists(hist_data);
4811
4812         kfree(hist_data);
4813 }
4814
4815 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4816 {
4817         struct tracing_map *map = hist_data->map;
4818         struct ftrace_event_field *field;
4819         struct hist_field *hist_field;
4820         int i, idx = 0;
4821
4822         for_each_hist_field(i, hist_data) {
4823                 hist_field = hist_data->fields[i];
4824                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4825                         tracing_map_cmp_fn_t cmp_fn;
4826
4827                         field = hist_field->field;
4828
4829                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4830                                 cmp_fn = tracing_map_cmp_none;
4831                         else if (!field)
4832                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4833                                                              hist_field->is_signed);
4834                         else if (is_string_field(field))
4835                                 cmp_fn = tracing_map_cmp_string;
4836                         else
4837                                 cmp_fn = tracing_map_cmp_num(field->size,
4838                                                              field->is_signed);
4839                         idx = tracing_map_add_key_field(map,
4840                                                         hist_field->offset,
4841                                                         cmp_fn);
4842                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4843                         idx = tracing_map_add_sum_field(map);
4844
4845                 if (idx < 0)
4846                         return idx;
4847
4848                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4849                         idx = tracing_map_add_var(map);
4850                         if (idx < 0)
4851                                 return idx;
4852                         hist_field->var.idx = idx;
4853                         hist_field->var.hist_data = hist_data;
4854                 }
4855         }
4856
4857         return 0;
4858 }
4859
4860 static struct hist_trigger_data *
4861 create_hist_data(unsigned int map_bits,
4862                  struct hist_trigger_attrs *attrs,
4863                  struct trace_event_file *file,
4864                  bool remove)
4865 {
4866         const struct tracing_map_ops *map_ops = NULL;
4867         struct hist_trigger_data *hist_data;
4868         int ret = 0;
4869
4870         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4871         if (!hist_data)
4872                 return ERR_PTR(-ENOMEM);
4873
4874         hist_data->attrs = attrs;
4875         hist_data->remove = remove;
4876         hist_data->event_file = file;
4877
4878         ret = parse_actions(hist_data);
4879         if (ret)
4880                 goto free;
4881
4882         ret = create_hist_fields(hist_data, file);
4883         if (ret)
4884                 goto free;
4885
4886         ret = create_sort_keys(hist_data);
4887         if (ret)
4888                 goto free;
4889
4890         map_ops = &hist_trigger_elt_data_ops;
4891
4892         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4893                                             map_ops, hist_data);
4894         if (IS_ERR(hist_data->map)) {
4895                 ret = PTR_ERR(hist_data->map);
4896                 hist_data->map = NULL;
4897                 goto free;
4898         }
4899
4900         ret = create_tracing_map_fields(hist_data);
4901         if (ret)
4902                 goto free;
4903  out:
4904         return hist_data;
4905  free:
4906         hist_data->attrs = NULL;
4907
4908         destroy_hist_data(hist_data);
4909
4910         hist_data = ERR_PTR(ret);
4911
4912         goto out;
4913 }
4914
4915 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4916                                     struct tracing_map_elt *elt,
4917                                     struct trace_buffer *buffer, void *rec,
4918                                     struct ring_buffer_event *rbe,
4919                                     u64 *var_ref_vals)
4920 {
4921         struct hist_elt_data *elt_data;
4922         struct hist_field *hist_field;
4923         unsigned int i, var_idx;
4924         u64 hist_val;
4925
4926         elt_data = elt->private_data;
4927         elt_data->var_ref_vals = var_ref_vals;
4928
4929         for_each_hist_val_field(i, hist_data) {
4930                 hist_field = hist_data->fields[i];
4931                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4932                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4933                         var_idx = hist_field->var.idx;
4934
4935                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4936                                 unsigned int str_start, var_str_idx, idx;
4937                                 char *str, *val_str;
4938                                 unsigned int size;
4939
4940                                 str_start = hist_data->n_field_var_str +
4941                                         hist_data->n_save_var_str;
4942                                 var_str_idx = hist_field->var_str_idx;
4943                                 idx = str_start + var_str_idx;
4944
4945                                 str = elt_data->field_var_str[idx];
4946                                 val_str = (char *)(uintptr_t)hist_val;
4947
4948                                 size = min(hist_field->size, STR_VAR_LEN_MAX);
4949                                 strscpy(str, val_str, size);
4950
4951                                 hist_val = (u64)(uintptr_t)str;
4952                         }
4953                         tracing_map_set_var(elt, var_idx, hist_val);
4954                         continue;
4955                 }
4956                 tracing_map_update_sum(elt, i, hist_val);
4957         }
4958
4959         for_each_hist_key_field(i, hist_data) {
4960                 hist_field = hist_data->fields[i];
4961                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4962                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4963                         var_idx = hist_field->var.idx;
4964                         tracing_map_set_var(elt, var_idx, hist_val);
4965                 }
4966         }
4967
4968         update_field_vars(hist_data, elt, buffer, rbe, rec);
4969 }
4970
4971 static inline void add_to_key(char *compound_key, void *key,
4972                               struct hist_field *key_field, void *rec)
4973 {
4974         size_t size = key_field->size;
4975
4976         if (key_field->flags & HIST_FIELD_FL_STRING) {
4977                 struct ftrace_event_field *field;
4978
4979                 field = key_field->field;
4980                 if (field->filter_type == FILTER_DYN_STRING ||
4981                     field->filter_type == FILTER_RDYN_STRING)
4982                         size = *(u32 *)(rec + field->offset) >> 16;
4983                 else if (field->filter_type == FILTER_STATIC_STRING)
4984                         size = field->size;
4985
4986                 /* ensure NULL-termination */
4987                 if (size > key_field->size - 1)
4988                         size = key_field->size - 1;
4989
4990                 strncpy(compound_key + key_field->offset, (char *)key, size);
4991         } else
4992                 memcpy(compound_key + key_field->offset, key, size);
4993 }
4994
4995 static void
4996 hist_trigger_actions(struct hist_trigger_data *hist_data,
4997                      struct tracing_map_elt *elt,
4998                      struct trace_buffer *buffer, void *rec,
4999                      struct ring_buffer_event *rbe, void *key,
5000                      u64 *var_ref_vals)
5001 {
5002         struct action_data *data;
5003         unsigned int i;
5004
5005         for (i = 0; i < hist_data->n_actions; i++) {
5006                 data = hist_data->actions[i];
5007                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5008         }
5009 }
5010
5011 static void event_hist_trigger(struct event_trigger_data *data,
5012                                struct trace_buffer *buffer, void *rec,
5013                                struct ring_buffer_event *rbe)
5014 {
5015         struct hist_trigger_data *hist_data = data->private_data;
5016         bool use_compound_key = (hist_data->n_keys > 1);
5017         unsigned long entries[HIST_STACKTRACE_DEPTH];
5018         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5019         char compound_key[HIST_KEY_SIZE_MAX];
5020         struct tracing_map_elt *elt = NULL;
5021         struct hist_field *key_field;
5022         u64 field_contents;
5023         void *key = NULL;
5024         unsigned int i;
5025
5026         memset(compound_key, 0, hist_data->key_size);
5027
5028         for_each_hist_key_field(i, hist_data) {
5029                 key_field = hist_data->fields[i];
5030
5031                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5032                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5033                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5034                                          HIST_STACKTRACE_SKIP);
5035                         key = entries;
5036                 } else {
5037                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5038                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5039                                 key = (void *)(unsigned long)field_contents;
5040                                 use_compound_key = true;
5041                         } else
5042                                 key = (void *)&field_contents;
5043                 }
5044
5045                 if (use_compound_key)
5046                         add_to_key(compound_key, key, key_field, rec);
5047         }
5048
5049         if (use_compound_key)
5050                 key = compound_key;
5051
5052         if (hist_data->n_var_refs &&
5053             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5054                 return;
5055
5056         elt = tracing_map_insert(hist_data->map, key);
5057         if (!elt)
5058                 return;
5059
5060         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5061
5062         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5063                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5064 }
5065
5066 static void hist_trigger_stacktrace_print(struct seq_file *m,
5067                                           unsigned long *stacktrace_entries,
5068                                           unsigned int max_entries)
5069 {
5070         unsigned int spaces = 8;
5071         unsigned int i;
5072
5073         for (i = 0; i < max_entries; i++) {
5074                 if (!stacktrace_entries[i])
5075                         return;
5076
5077                 seq_printf(m, "%*c", 1 + spaces, ' ');
5078                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5079         }
5080 }
5081
5082 static void hist_trigger_print_key(struct seq_file *m,
5083                                    struct hist_trigger_data *hist_data,
5084                                    void *key,
5085                                    struct tracing_map_elt *elt)
5086 {
5087         struct hist_field *key_field;
5088         bool multiline = false;
5089         const char *field_name;
5090         unsigned int i;
5091         u64 uval;
5092
5093         seq_puts(m, "{ ");
5094
5095         for_each_hist_key_field(i, hist_data) {
5096                 key_field = hist_data->fields[i];
5097
5098                 if (i > hist_data->n_vals)
5099                         seq_puts(m, ", ");
5100
5101                 field_name = hist_field_name(key_field, 0);
5102
5103                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5104                         uval = *(u64 *)(key + key_field->offset);
5105                         seq_printf(m, "%s: %llx", field_name, uval);
5106                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5107                         uval = *(u64 *)(key + key_field->offset);
5108                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5109                                    uval, (void *)(uintptr_t)uval);
5110                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5111                         uval = *(u64 *)(key + key_field->offset);
5112                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5113                                    uval, (void *)(uintptr_t)uval);
5114                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5115                         struct hist_elt_data *elt_data = elt->private_data;
5116                         char *comm;
5117
5118                         if (WARN_ON_ONCE(!elt_data))
5119                                 return;
5120
5121                         comm = elt_data->comm;
5122
5123                         uval = *(u64 *)(key + key_field->offset);
5124                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5125                                    comm, uval);
5126                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5127                         const char *syscall_name;
5128
5129                         uval = *(u64 *)(key + key_field->offset);
5130                         syscall_name = get_syscall_name(uval);
5131                         if (!syscall_name)
5132                                 syscall_name = "unknown_syscall";
5133
5134                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5135                                    syscall_name, uval);
5136                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5137                         seq_puts(m, "stacktrace:\n");
5138                         hist_trigger_stacktrace_print(m,
5139                                                       key + key_field->offset,
5140                                                       HIST_STACKTRACE_DEPTH);
5141                         multiline = true;
5142                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5143                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5144                                    *(u64 *)(key + key_field->offset));
5145                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5146                         unsigned long buckets = key_field->buckets;
5147                         uval = *(u64 *)(key + key_field->offset);
5148                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5149                                    uval, uval + buckets -1);
5150                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5151                         seq_printf(m, "%s: %-50s", field_name,
5152                                    (char *)(key + key_field->offset));
5153                 } else {
5154                         uval = *(u64 *)(key + key_field->offset);
5155                         seq_printf(m, "%s: %10llu", field_name, uval);
5156                 }
5157         }
5158
5159         if (!multiline)
5160                 seq_puts(m, " ");
5161
5162         seq_puts(m, "}");
5163 }
5164
5165 static void hist_trigger_entry_print(struct seq_file *m,
5166                                      struct hist_trigger_data *hist_data,
5167                                      void *key,
5168                                      struct tracing_map_elt *elt)
5169 {
5170         const char *field_name;
5171         unsigned int i;
5172
5173         hist_trigger_print_key(m, hist_data, key, elt);
5174
5175         seq_printf(m, " hitcount: %10llu",
5176                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5177
5178         for (i = 1; i < hist_data->n_vals; i++) {
5179                 field_name = hist_field_name(hist_data->fields[i], 0);
5180
5181                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5182                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5183                         continue;
5184
5185                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5186                         seq_printf(m, "  %s: %10llx", field_name,
5187                                    tracing_map_read_sum(elt, i));
5188                 } else {
5189                         seq_printf(m, "  %s: %10llu", field_name,
5190                                    tracing_map_read_sum(elt, i));
5191                 }
5192         }
5193
5194         print_actions(m, hist_data, elt);
5195
5196         seq_puts(m, "\n");
5197 }
5198
5199 static int print_entries(struct seq_file *m,
5200                          struct hist_trigger_data *hist_data)
5201 {
5202         struct tracing_map_sort_entry **sort_entries = NULL;
5203         struct tracing_map *map = hist_data->map;
5204         int i, n_entries;
5205
5206         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5207                                              hist_data->n_sort_keys,
5208                                              &sort_entries);
5209         if (n_entries < 0)
5210                 return n_entries;
5211
5212         for (i = 0; i < n_entries; i++)
5213                 hist_trigger_entry_print(m, hist_data,
5214                                          sort_entries[i]->key,
5215                                          sort_entries[i]->elt);
5216
5217         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5218
5219         return n_entries;
5220 }
5221
5222 static void hist_trigger_show(struct seq_file *m,
5223                               struct event_trigger_data *data, int n)
5224 {
5225         struct hist_trigger_data *hist_data;
5226         int n_entries;
5227
5228         if (n > 0)
5229                 seq_puts(m, "\n\n");
5230
5231         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5232         data->ops->print(m, data->ops, data);
5233         seq_puts(m, "#\n\n");
5234
5235         hist_data = data->private_data;
5236         n_entries = print_entries(m, hist_data);
5237         if (n_entries < 0)
5238                 n_entries = 0;
5239
5240         track_data_snapshot_print(m, hist_data);
5241
5242         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5243                    (u64)atomic64_read(&hist_data->map->hits),
5244                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5245 }
5246
5247 static int hist_show(struct seq_file *m, void *v)
5248 {
5249         struct event_trigger_data *data;
5250         struct trace_event_file *event_file;
5251         int n = 0, ret = 0;
5252
5253         mutex_lock(&event_mutex);
5254
5255         event_file = event_file_data(m->private);
5256         if (unlikely(!event_file)) {
5257                 ret = -ENODEV;
5258                 goto out_unlock;
5259         }
5260
5261         list_for_each_entry(data, &event_file->triggers, list) {
5262                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5263                         hist_trigger_show(m, data, n++);
5264         }
5265
5266  out_unlock:
5267         mutex_unlock(&event_mutex);
5268
5269         return ret;
5270 }
5271
5272 static int event_hist_open(struct inode *inode, struct file *file)
5273 {
5274         int ret;
5275
5276         ret = security_locked_down(LOCKDOWN_TRACEFS);
5277         if (ret)
5278                 return ret;
5279
5280         return single_open(file, hist_show, file);
5281 }
5282
5283 const struct file_operations event_hist_fops = {
5284         .open = event_hist_open,
5285         .read = seq_read,
5286         .llseek = seq_lseek,
5287         .release = single_release,
5288 };
5289
5290 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5291 static void hist_field_debug_show_flags(struct seq_file *m,
5292                                         unsigned long flags)
5293 {
5294         seq_puts(m, "      flags:\n");
5295
5296         if (flags & HIST_FIELD_FL_KEY)
5297                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5298         else if (flags & HIST_FIELD_FL_HITCOUNT)
5299                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5300         else if (flags & HIST_FIELD_FL_VAR)
5301                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5302         else if (flags & HIST_FIELD_FL_VAR_REF)
5303                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5304         else
5305                 seq_puts(m, "        VAL: normal u64 value\n");
5306
5307         if (flags & HIST_FIELD_FL_ALIAS)
5308                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5309         else if (flags & HIST_FIELD_FL_CONST)
5310                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5311 }
5312
5313 static int hist_field_debug_show(struct seq_file *m,
5314                                  struct hist_field *field, unsigned long flags)
5315 {
5316         if ((field->flags & flags) != flags) {
5317                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5318                 return -EINVAL;
5319         }
5320
5321         hist_field_debug_show_flags(m, field->flags);
5322         if (field->field)
5323                 seq_printf(m, "      ftrace_event_field name: %s\n",
5324                            field->field->name);
5325
5326         if (field->flags & HIST_FIELD_FL_VAR) {
5327                 seq_printf(m, "      var.name: %s\n", field->var.name);
5328                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5329                            field->var.idx);
5330         }
5331
5332         if (field->flags & HIST_FIELD_FL_CONST)
5333                 seq_printf(m, "      constant: %llu\n", field->constant);
5334
5335         if (field->flags & HIST_FIELD_FL_ALIAS)
5336                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5337                            field->var_ref_idx);
5338
5339         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5340                 seq_printf(m, "      name: %s\n", field->name);
5341                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5342                            field->var.idx);
5343                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5344                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5345                            field->var_ref_idx);
5346                 if (field->system)
5347                         seq_printf(m, "      system: %s\n", field->system);
5348                 if (field->event_name)
5349                         seq_printf(m, "      event_name: %s\n", field->event_name);
5350         }
5351
5352         seq_printf(m, "      type: %s\n", field->type);
5353         seq_printf(m, "      size: %u\n", field->size);
5354         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5355
5356         return 0;
5357 }
5358
5359 static int field_var_debug_show(struct seq_file *m,
5360                                 struct field_var *field_var, unsigned int i,
5361                                 bool save_vars)
5362 {
5363         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5364         struct hist_field *field;
5365         int ret = 0;
5366
5367         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5368
5369         field = field_var->var;
5370
5371         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5372
5373         hist_field_debug_show_flags(m, field->flags);
5374         seq_printf(m, "      var.name: %s\n", field->var.name);
5375         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5376                    field->var.idx);
5377
5378         field = field_var->val;
5379
5380         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5381         if (field->field)
5382                 seq_printf(m, "      ftrace_event_field name: %s\n",
5383                            field->field->name);
5384         else {
5385                 ret = -EINVAL;
5386                 goto out;
5387         }
5388
5389         seq_printf(m, "      type: %s\n", field->type);
5390         seq_printf(m, "      size: %u\n", field->size);
5391         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5392 out:
5393         return ret;
5394 }
5395
5396 static int hist_action_debug_show(struct seq_file *m,
5397                                   struct action_data *data, int i)
5398 {
5399         int ret = 0;
5400
5401         if (data->handler == HANDLER_ONMAX ||
5402             data->handler == HANDLER_ONCHANGE) {
5403                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5404                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5405                                             HIST_FIELD_FL_VAR_REF);
5406                 if (ret)
5407                         goto out;
5408
5409                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5410                 ret = hist_field_debug_show(m, data->track_data.track_var,
5411                                             HIST_FIELD_FL_VAR);
5412                 if (ret)
5413                         goto out;
5414         }
5415
5416         if (data->handler == HANDLER_ONMATCH) {
5417                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5418                            i, data->match_data.event_system);
5419                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5420                            i, data->match_data.event);
5421         }
5422 out:
5423         return ret;
5424 }
5425
5426 static int hist_actions_debug_show(struct seq_file *m,
5427                                    struct hist_trigger_data *hist_data)
5428 {
5429         int i, ret = 0;
5430
5431         if (hist_data->n_actions)
5432                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5433
5434         for (i = 0; i < hist_data->n_actions; i++) {
5435                 struct action_data *action = hist_data->actions[i];
5436
5437                 ret = hist_action_debug_show(m, action, i);
5438                 if (ret)
5439                         goto out;
5440         }
5441
5442         if (hist_data->n_save_vars)
5443                 seq_puts(m, "\n  save action variables (save() params):\n");
5444
5445         for (i = 0; i < hist_data->n_save_vars; i++) {
5446                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5447                 if (ret)
5448                         goto out;
5449         }
5450 out:
5451         return ret;
5452 }
5453
5454 static void hist_trigger_debug_show(struct seq_file *m,
5455                                     struct event_trigger_data *data, int n)
5456 {
5457         struct hist_trigger_data *hist_data;
5458         int i, ret;
5459
5460         if (n > 0)
5461                 seq_puts(m, "\n\n");
5462
5463         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5464         data->ops->print(m, data->ops, data);
5465         seq_puts(m, "#\n\n");
5466
5467         hist_data = data->private_data;
5468
5469         seq_printf(m, "hist_data: %p\n\n", hist_data);
5470         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5471         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5472         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5473
5474         seq_puts(m, "\n  val fields:\n\n");
5475
5476         seq_puts(m, "    hist_data->fields[0]:\n");
5477         ret = hist_field_debug_show(m, hist_data->fields[0],
5478                                     HIST_FIELD_FL_HITCOUNT);
5479         if (ret)
5480                 return;
5481
5482         for (i = 1; i < hist_data->n_vals; i++) {
5483                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5484                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5485                 if (ret)
5486                         return;
5487         }
5488
5489         seq_puts(m, "\n  key fields:\n");
5490
5491         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5492                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5493                 ret = hist_field_debug_show(m, hist_data->fields[i],
5494                                             HIST_FIELD_FL_KEY);
5495                 if (ret)
5496                         return;
5497         }
5498
5499         if (hist_data->n_var_refs)
5500                 seq_puts(m, "\n  variable reference fields:\n");
5501
5502         for (i = 0; i < hist_data->n_var_refs; i++) {
5503                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5504                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5505                                             HIST_FIELD_FL_VAR_REF);
5506                 if (ret)
5507                         return;
5508         }
5509
5510         if (hist_data->n_field_vars)
5511                 seq_puts(m, "\n  field variables:\n");
5512
5513         for (i = 0; i < hist_data->n_field_vars; i++) {
5514                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5515                 if (ret)
5516                         return;
5517         }
5518
5519         ret = hist_actions_debug_show(m, hist_data);
5520         if (ret)
5521                 return;
5522 }
5523
5524 static int hist_debug_show(struct seq_file *m, void *v)
5525 {
5526         struct event_trigger_data *data;
5527         struct trace_event_file *event_file;
5528         int n = 0, ret = 0;
5529
5530         mutex_lock(&event_mutex);
5531
5532         event_file = event_file_data(m->private);
5533         if (unlikely(!event_file)) {
5534                 ret = -ENODEV;
5535                 goto out_unlock;
5536         }
5537
5538         list_for_each_entry(data, &event_file->triggers, list) {
5539                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5540                         hist_trigger_debug_show(m, data, n++);
5541         }
5542
5543  out_unlock:
5544         mutex_unlock(&event_mutex);
5545
5546         return ret;
5547 }
5548
5549 static int event_hist_debug_open(struct inode *inode, struct file *file)
5550 {
5551         int ret;
5552
5553         ret = security_locked_down(LOCKDOWN_TRACEFS);
5554         if (ret)
5555                 return ret;
5556
5557         return single_open(file, hist_debug_show, file);
5558 }
5559
5560 const struct file_operations event_hist_debug_fops = {
5561         .open = event_hist_debug_open,
5562         .read = seq_read,
5563         .llseek = seq_lseek,
5564         .release = single_release,
5565 };
5566 #endif
5567
5568 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5569 {
5570         const char *field_name = hist_field_name(hist_field, 0);
5571
5572         if (hist_field->var.name)
5573                 seq_printf(m, "%s=", hist_field->var.name);
5574
5575         if (hist_field->flags & HIST_FIELD_FL_CPU)
5576                 seq_puts(m, "common_cpu");
5577         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5578                 seq_printf(m, "%llu", hist_field->constant);
5579         else if (field_name) {
5580                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5581                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5582                         seq_putc(m, '$');
5583                 seq_printf(m, "%s", field_name);
5584         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5585                 seq_puts(m, "common_timestamp");
5586
5587         if (hist_field->flags) {
5588                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5589                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5590                         const char *flags = get_hist_field_flags(hist_field);
5591
5592                         if (flags)
5593                                 seq_printf(m, ".%s", flags);
5594                 }
5595         }
5596         if (hist_field->buckets)
5597                 seq_printf(m, "=%ld", hist_field->buckets);
5598 }
5599
5600 static int event_hist_trigger_print(struct seq_file *m,
5601                                     struct event_trigger_ops *ops,
5602                                     struct event_trigger_data *data)
5603 {
5604         struct hist_trigger_data *hist_data = data->private_data;
5605         struct hist_field *field;
5606         bool have_var = false;
5607         unsigned int i;
5608
5609         seq_puts(m, "hist:");
5610
5611         if (data->name)
5612                 seq_printf(m, "%s:", data->name);
5613
5614         seq_puts(m, "keys=");
5615
5616         for_each_hist_key_field(i, hist_data) {
5617                 field = hist_data->fields[i];
5618
5619                 if (i > hist_data->n_vals)
5620                         seq_puts(m, ",");
5621
5622                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5623                         seq_puts(m, "stacktrace");
5624                 else
5625                         hist_field_print(m, field);
5626         }
5627
5628         seq_puts(m, ":vals=");
5629
5630         for_each_hist_val_field(i, hist_data) {
5631                 field = hist_data->fields[i];
5632                 if (field->flags & HIST_FIELD_FL_VAR) {
5633                         have_var = true;
5634                         continue;
5635                 }
5636
5637                 if (i == HITCOUNT_IDX)
5638                         seq_puts(m, "hitcount");
5639                 else {
5640                         seq_puts(m, ",");
5641                         hist_field_print(m, field);
5642                 }
5643         }
5644
5645         if (have_var) {
5646                 unsigned int n = 0;
5647
5648                 seq_puts(m, ":");
5649
5650                 for_each_hist_val_field(i, hist_data) {
5651                         field = hist_data->fields[i];
5652
5653                         if (field->flags & HIST_FIELD_FL_VAR) {
5654                                 if (n++)
5655                                         seq_puts(m, ",");
5656                                 hist_field_print(m, field);
5657                         }
5658                 }
5659         }
5660
5661         seq_puts(m, ":sort=");
5662
5663         for (i = 0; i < hist_data->n_sort_keys; i++) {
5664                 struct tracing_map_sort_key *sort_key;
5665                 unsigned int idx, first_key_idx;
5666
5667                 /* skip VAR vals */
5668                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5669
5670                 sort_key = &hist_data->sort_keys[i];
5671                 idx = sort_key->field_idx;
5672
5673                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5674                         return -EINVAL;
5675
5676                 if (i > 0)
5677                         seq_puts(m, ",");
5678
5679                 if (idx == HITCOUNT_IDX)
5680                         seq_puts(m, "hitcount");
5681                 else {
5682                         if (idx >= first_key_idx)
5683                                 idx += hist_data->n_vars;
5684                         hist_field_print(m, hist_data->fields[idx]);
5685                 }
5686
5687                 if (sort_key->descending)
5688                         seq_puts(m, ".descending");
5689         }
5690         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5691         if (hist_data->enable_timestamps)
5692                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5693
5694         print_actions_spec(m, hist_data);
5695
5696         if (data->filter_str)
5697                 seq_printf(m, " if %s", data->filter_str);
5698
5699         if (data->paused)
5700                 seq_puts(m, " [paused]");
5701         else
5702                 seq_puts(m, " [active]");
5703
5704         seq_putc(m, '\n');
5705
5706         return 0;
5707 }
5708
5709 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5710                                    struct event_trigger_data *data)
5711 {
5712         struct hist_trigger_data *hist_data = data->private_data;
5713
5714         if (!data->ref && hist_data->attrs->name)
5715                 save_named_trigger(hist_data->attrs->name, data);
5716
5717         data->ref++;
5718
5719         return 0;
5720 }
5721
5722 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5723 {
5724         struct trace_event_file *file;
5725         unsigned int i;
5726         char *cmd;
5727         int ret;
5728
5729         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5730                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5731                 cmd = hist_data->field_var_hists[i]->cmd;
5732                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5733                                                "!hist", "hist", cmd);
5734                 WARN_ON_ONCE(ret < 0);
5735         }
5736 }
5737
5738 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5739                                     struct event_trigger_data *data)
5740 {
5741         struct hist_trigger_data *hist_data = data->private_data;
5742
5743         if (WARN_ON_ONCE(data->ref <= 0))
5744                 return;
5745
5746         data->ref--;
5747         if (!data->ref) {
5748                 if (data->name)
5749                         del_named_trigger(data);
5750
5751                 trigger_data_free(data);
5752
5753                 remove_hist_vars(hist_data);
5754
5755                 unregister_field_var_hists(hist_data);
5756
5757                 destroy_hist_data(hist_data);
5758         }
5759 }
5760
5761 static struct event_trigger_ops event_hist_trigger_ops = {
5762         .trigger                = event_hist_trigger,
5763         .print                  = event_hist_trigger_print,
5764         .init                   = event_hist_trigger_init,
5765         .free                   = event_hist_trigger_free,
5766 };
5767
5768 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5769                                          struct event_trigger_data *data)
5770 {
5771         data->ref++;
5772
5773         save_named_trigger(data->named_data->name, data);
5774
5775         event_hist_trigger_init(ops, data->named_data);
5776
5777         return 0;
5778 }
5779
5780 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5781                                           struct event_trigger_data *data)
5782 {
5783         if (WARN_ON_ONCE(data->ref <= 0))
5784                 return;
5785
5786         event_hist_trigger_free(ops, data->named_data);
5787
5788         data->ref--;
5789         if (!data->ref) {
5790                 del_named_trigger(data);
5791                 trigger_data_free(data);
5792         }
5793 }
5794
5795 static struct event_trigger_ops event_hist_trigger_named_ops = {
5796         .trigger                = event_hist_trigger,
5797         .print                  = event_hist_trigger_print,
5798         .init                   = event_hist_trigger_named_init,
5799         .free                   = event_hist_trigger_named_free,
5800 };
5801
5802 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5803                                                             char *param)
5804 {
5805         return &event_hist_trigger_ops;
5806 }
5807
5808 static void hist_clear(struct event_trigger_data *data)
5809 {
5810         struct hist_trigger_data *hist_data = data->private_data;
5811
5812         if (data->name)
5813                 pause_named_trigger(data);
5814
5815         tracepoint_synchronize_unregister();
5816
5817         tracing_map_clear(hist_data->map);
5818
5819         if (data->name)
5820                 unpause_named_trigger(data);
5821 }
5822
5823 static bool compatible_field(struct ftrace_event_field *field,
5824                              struct ftrace_event_field *test_field)
5825 {
5826         if (field == test_field)
5827                 return true;
5828         if (field == NULL || test_field == NULL)
5829                 return false;
5830         if (strcmp(field->name, test_field->name) != 0)
5831                 return false;
5832         if (strcmp(field->type, test_field->type) != 0)
5833                 return false;
5834         if (field->size != test_field->size)
5835                 return false;
5836         if (field->is_signed != test_field->is_signed)
5837                 return false;
5838
5839         return true;
5840 }
5841
5842 static bool hist_trigger_match(struct event_trigger_data *data,
5843                                struct event_trigger_data *data_test,
5844                                struct event_trigger_data *named_data,
5845                                bool ignore_filter)
5846 {
5847         struct tracing_map_sort_key *sort_key, *sort_key_test;
5848         struct hist_trigger_data *hist_data, *hist_data_test;
5849         struct hist_field *key_field, *key_field_test;
5850         unsigned int i;
5851
5852         if (named_data && (named_data != data_test) &&
5853             (named_data != data_test->named_data))
5854                 return false;
5855
5856         if (!named_data && is_named_trigger(data_test))
5857                 return false;
5858
5859         hist_data = data->private_data;
5860         hist_data_test = data_test->private_data;
5861
5862         if (hist_data->n_vals != hist_data_test->n_vals ||
5863             hist_data->n_fields != hist_data_test->n_fields ||
5864             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5865                 return false;
5866
5867         if (!ignore_filter) {
5868                 if ((data->filter_str && !data_test->filter_str) ||
5869                    (!data->filter_str && data_test->filter_str))
5870                         return false;
5871         }
5872
5873         for_each_hist_field(i, hist_data) {
5874                 key_field = hist_data->fields[i];
5875                 key_field_test = hist_data_test->fields[i];
5876
5877                 if (key_field->flags != key_field_test->flags)
5878                         return false;
5879                 if (!compatible_field(key_field->field, key_field_test->field))
5880                         return false;
5881                 if (key_field->offset != key_field_test->offset)
5882                         return false;
5883                 if (key_field->size != key_field_test->size)
5884                         return false;
5885                 if (key_field->is_signed != key_field_test->is_signed)
5886                         return false;
5887                 if (!!key_field->var.name != !!key_field_test->var.name)
5888                         return false;
5889                 if (key_field->var.name &&
5890                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5891                         return false;
5892         }
5893
5894         for (i = 0; i < hist_data->n_sort_keys; i++) {
5895                 sort_key = &hist_data->sort_keys[i];
5896                 sort_key_test = &hist_data_test->sort_keys[i];
5897
5898                 if (sort_key->field_idx != sort_key_test->field_idx ||
5899                     sort_key->descending != sort_key_test->descending)
5900                         return false;
5901         }
5902
5903         if (!ignore_filter && data->filter_str &&
5904             (strcmp(data->filter_str, data_test->filter_str) != 0))
5905                 return false;
5906
5907         if (!actions_match(hist_data, hist_data_test))
5908                 return false;
5909
5910         return true;
5911 }
5912
5913 static int hist_register_trigger(char *glob,
5914                                  struct event_trigger_data *data,
5915                                  struct trace_event_file *file)
5916 {
5917         struct hist_trigger_data *hist_data = data->private_data;
5918         struct event_trigger_data *test, *named_data = NULL;
5919         struct trace_array *tr = file->tr;
5920         int ret = 0;
5921
5922         if (hist_data->attrs->name) {
5923                 named_data = find_named_trigger(hist_data->attrs->name);
5924                 if (named_data) {
5925                         if (!hist_trigger_match(data, named_data, named_data,
5926                                                 true)) {
5927                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5928                                 ret = -EINVAL;
5929                                 goto out;
5930                         }
5931                 }
5932         }
5933
5934         if (hist_data->attrs->name && !named_data)
5935                 goto new;
5936
5937         lockdep_assert_held(&event_mutex);
5938
5939         list_for_each_entry(test, &file->triggers, list) {
5940                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5941                         if (!hist_trigger_match(data, test, named_data, false))
5942                                 continue;
5943                         if (hist_data->attrs->pause)
5944                                 test->paused = true;
5945                         else if (hist_data->attrs->cont)
5946                                 test->paused = false;
5947                         else if (hist_data->attrs->clear)
5948                                 hist_clear(test);
5949                         else {
5950                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5951                                 ret = -EEXIST;
5952                         }
5953                         goto out;
5954                 }
5955         }
5956  new:
5957         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5958                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5959                 ret = -ENOENT;
5960                 goto out;
5961         }
5962
5963         if (hist_data->attrs->pause)
5964                 data->paused = true;
5965
5966         if (named_data) {
5967                 data->private_data = named_data->private_data;
5968                 set_named_trigger_data(data, named_data);
5969                 data->ops = &event_hist_trigger_named_ops;
5970         }
5971
5972         if (data->ops->init) {
5973                 ret = data->ops->init(data->ops, data);
5974                 if (ret < 0)
5975                         goto out;
5976         }
5977
5978         if (hist_data->enable_timestamps) {
5979                 char *clock = hist_data->attrs->clock;
5980
5981                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5982                 if (ret) {
5983                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5984                         goto out;
5985                 }
5986
5987                 tracing_set_filter_buffering(file->tr, true);
5988         }
5989
5990         if (named_data)
5991                 destroy_hist_data(hist_data);
5992
5993         ret++;
5994  out:
5995         return ret;
5996 }
5997
5998 static int hist_trigger_enable(struct event_trigger_data *data,
5999                                struct trace_event_file *file)
6000 {
6001         int ret = 0;
6002
6003         list_add_tail_rcu(&data->list, &file->triggers);
6004
6005         update_cond_flag(file);
6006
6007         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6008                 list_del_rcu(&data->list);
6009                 update_cond_flag(file);
6010                 ret--;
6011         }
6012
6013         return ret;
6014 }
6015
6016 static bool have_hist_trigger_match(struct event_trigger_data *data,
6017                                     struct trace_event_file *file)
6018 {
6019         struct hist_trigger_data *hist_data = data->private_data;
6020         struct event_trigger_data *test, *named_data = NULL;
6021         bool match = false;
6022
6023         lockdep_assert_held(&event_mutex);
6024
6025         if (hist_data->attrs->name)
6026                 named_data = find_named_trigger(hist_data->attrs->name);
6027
6028         list_for_each_entry(test, &file->triggers, list) {
6029                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6030                         if (hist_trigger_match(data, test, named_data, false)) {
6031                                 match = true;
6032                                 break;
6033                         }
6034                 }
6035         }
6036
6037         return match;
6038 }
6039
6040 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6041                                     struct trace_event_file *file)
6042 {
6043         struct hist_trigger_data *hist_data = data->private_data;
6044         struct event_trigger_data *test, *named_data = NULL;
6045
6046         lockdep_assert_held(&event_mutex);
6047
6048         if (hist_data->attrs->name)
6049                 named_data = find_named_trigger(hist_data->attrs->name);
6050
6051         list_for_each_entry(test, &file->triggers, list) {
6052                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6053                         if (!hist_trigger_match(data, test, named_data, false))
6054                                 continue;
6055                         hist_data = test->private_data;
6056                         if (check_var_refs(hist_data))
6057                                 return true;
6058                         break;
6059                 }
6060         }
6061
6062         return false;
6063 }
6064
6065 static void hist_unregister_trigger(char *glob,
6066                                     struct event_trigger_data *data,
6067                                     struct trace_event_file *file)
6068 {
6069         struct hist_trigger_data *hist_data = data->private_data;
6070         struct event_trigger_data *test, *named_data = NULL;
6071         bool unregistered = false;
6072
6073         lockdep_assert_held(&event_mutex);
6074
6075         if (hist_data->attrs->name)
6076                 named_data = find_named_trigger(hist_data->attrs->name);
6077
6078         list_for_each_entry(test, &file->triggers, list) {
6079                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6080                         if (!hist_trigger_match(data, test, named_data, false))
6081                                 continue;
6082                         unregistered = true;
6083                         list_del_rcu(&test->list);
6084                         trace_event_trigger_enable_disable(file, 0);
6085                         update_cond_flag(file);
6086                         break;
6087                 }
6088         }
6089
6090         if (unregistered && test->ops->free)
6091                 test->ops->free(test->ops, test);
6092
6093         if (hist_data->enable_timestamps) {
6094                 if (!hist_data->remove || unregistered)
6095                         tracing_set_filter_buffering(file->tr, false);
6096         }
6097 }
6098
6099 static bool hist_file_check_refs(struct trace_event_file *file)
6100 {
6101         struct hist_trigger_data *hist_data;
6102         struct event_trigger_data *test;
6103
6104         lockdep_assert_held(&event_mutex);
6105
6106         list_for_each_entry(test, &file->triggers, list) {
6107                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6108                         hist_data = test->private_data;
6109                         if (check_var_refs(hist_data))
6110                                 return true;
6111                 }
6112         }
6113
6114         return false;
6115 }
6116
6117 static void hist_unreg_all(struct trace_event_file *file)
6118 {
6119         struct event_trigger_data *test, *n;
6120         struct hist_trigger_data *hist_data;
6121         struct synth_event *se;
6122         const char *se_name;
6123
6124         lockdep_assert_held(&event_mutex);
6125
6126         if (hist_file_check_refs(file))
6127                 return;
6128
6129         list_for_each_entry_safe(test, n, &file->triggers, list) {
6130                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6131                         hist_data = test->private_data;
6132                         list_del_rcu(&test->list);
6133                         trace_event_trigger_enable_disable(file, 0);
6134
6135                         se_name = trace_event_name(file->event_call);
6136                         se = find_synth_event(se_name);
6137                         if (se)
6138                                 se->ref--;
6139
6140                         update_cond_flag(file);
6141                         if (hist_data->enable_timestamps)
6142                                 tracing_set_filter_buffering(file->tr, false);
6143                         if (test->ops->free)
6144                                 test->ops->free(test->ops, test);
6145                 }
6146         }
6147 }
6148
6149 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6150                                     struct trace_event_file *file,
6151                                     char *glob, char *cmd, char *param)
6152 {
6153         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6154         struct event_trigger_data *trigger_data;
6155         struct hist_trigger_attrs *attrs;
6156         struct event_trigger_ops *trigger_ops;
6157         struct hist_trigger_data *hist_data;
6158         struct synth_event *se;
6159         const char *se_name;
6160         bool remove = false;
6161         char *trigger, *p, *start;
6162         int ret = 0;
6163
6164         lockdep_assert_held(&event_mutex);
6165
6166         if (glob && strlen(glob)) {
6167                 hist_err_clear();
6168                 last_cmd_set(file, param);
6169         }
6170
6171         if (!param)
6172                 return -EINVAL;
6173
6174         if (glob[0] == '!')
6175                 remove = true;
6176
6177         /*
6178          * separate the trigger from the filter (k:v [if filter])
6179          * allowing for whitespace in the trigger
6180          */
6181         p = trigger = param;
6182         do {
6183                 p = strstr(p, "if");
6184                 if (!p)
6185                         break;
6186                 if (p == param)
6187                         return -EINVAL;
6188                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6189                         p++;
6190                         continue;
6191                 }
6192                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6193                         return -EINVAL;
6194                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6195                         p++;
6196                         continue;
6197                 }
6198                 break;
6199         } while (p);
6200
6201         if (!p)
6202                 param = NULL;
6203         else {
6204                 *(p - 1) = '\0';
6205                 param = strstrip(p);
6206                 trigger = strstrip(trigger);
6207         }
6208
6209         /*
6210          * To simplify arithmetic expression parsing, replace occurrences of
6211          * '.sym-offset' modifier with '.symXoffset'
6212          */
6213         start = strstr(trigger, ".sym-offset");
6214         while (start) {
6215                 *(start + 4) = 'X';
6216                 start = strstr(start + 11, ".sym-offset");
6217         }
6218
6219         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6220         if (IS_ERR(attrs))
6221                 return PTR_ERR(attrs);
6222
6223         if (attrs->map_bits)
6224                 hist_trigger_bits = attrs->map_bits;
6225
6226         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6227         if (IS_ERR(hist_data)) {
6228                 destroy_hist_trigger_attrs(attrs);
6229                 return PTR_ERR(hist_data);
6230         }
6231
6232         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6233
6234         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6235         if (!trigger_data) {
6236                 ret = -ENOMEM;
6237                 goto out_free;
6238         }
6239
6240         trigger_data->count = -1;
6241         trigger_data->ops = trigger_ops;
6242         trigger_data->cmd_ops = cmd_ops;
6243
6244         INIT_LIST_HEAD(&trigger_data->list);
6245         RCU_INIT_POINTER(trigger_data->filter, NULL);
6246
6247         trigger_data->private_data = hist_data;
6248
6249         /* if param is non-empty, it's supposed to be a filter */
6250         if (param && cmd_ops->set_filter) {
6251                 ret = cmd_ops->set_filter(param, trigger_data, file);
6252                 if (ret < 0)
6253                         goto out_free;
6254         }
6255
6256         if (remove) {
6257                 if (!have_hist_trigger_match(trigger_data, file))
6258                         goto out_free;
6259
6260                 if (hist_trigger_check_refs(trigger_data, file)) {
6261                         ret = -EBUSY;
6262                         goto out_free;
6263                 }
6264
6265                 cmd_ops->unreg(glob+1, trigger_data, file);
6266                 se_name = trace_event_name(file->event_call);
6267                 se = find_synth_event(se_name);
6268                 if (se)
6269                         se->ref--;
6270                 ret = 0;
6271                 goto out_free;
6272         }
6273
6274         ret = cmd_ops->reg(glob, trigger_data, file);
6275         /*
6276          * The above returns on success the # of triggers registered,
6277          * but if it didn't register any it returns zero.  Consider no
6278          * triggers registered a failure too.
6279          */
6280         if (!ret) {
6281                 if (!(attrs->pause || attrs->cont || attrs->clear))
6282                         ret = -ENOENT;
6283                 goto out_free;
6284         } else if (ret < 0)
6285                 goto out_free;
6286
6287         if (get_named_trigger_data(trigger_data))
6288                 goto enable;
6289
6290         if (has_hist_vars(hist_data))
6291                 save_hist_vars(hist_data);
6292
6293         ret = create_actions(hist_data);
6294         if (ret)
6295                 goto out_unreg;
6296
6297         ret = tracing_map_init(hist_data->map);
6298         if (ret)
6299                 goto out_unreg;
6300 enable:
6301         ret = hist_trigger_enable(trigger_data, file);
6302         if (ret)
6303                 goto out_unreg;
6304
6305         se_name = trace_event_name(file->event_call);
6306         se = find_synth_event(se_name);
6307         if (se)
6308                 se->ref++;
6309         /* Just return zero, not the number of registered triggers */
6310         ret = 0;
6311  out:
6312         if (ret == 0)
6313                 hist_err_clear();
6314
6315         return ret;
6316  out_unreg:
6317         cmd_ops->unreg(glob+1, trigger_data, file);
6318  out_free:
6319         if (cmd_ops->set_filter)
6320                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6321
6322         remove_hist_vars(hist_data);
6323
6324         kfree(trigger_data);
6325
6326         destroy_hist_data(hist_data);
6327         goto out;
6328 }
6329
6330 static struct event_command trigger_hist_cmd = {
6331         .name                   = "hist",
6332         .trigger_type           = ETT_EVENT_HIST,
6333         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6334         .parse                  = event_hist_trigger_parse,
6335         .reg                    = hist_register_trigger,
6336         .unreg                  = hist_unregister_trigger,
6337         .unreg_all              = hist_unreg_all,
6338         .get_trigger_ops        = event_hist_get_trigger_ops,
6339         .set_filter             = set_trigger_filter,
6340 };
6341
6342 __init int register_trigger_hist_cmd(void)
6343 {
6344         int ret;
6345
6346         ret = register_event_command(&trigger_hist_cmd);
6347         WARN_ON(ret < 0);
6348
6349         return ret;
6350 }
6351
6352 static void
6353 hist_enable_trigger(struct event_trigger_data *data,
6354                     struct trace_buffer *buffer,  void *rec,
6355                     struct ring_buffer_event *event)
6356 {
6357         struct enable_trigger_data *enable_data = data->private_data;
6358         struct event_trigger_data *test;
6359
6360         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6361                                 lockdep_is_held(&event_mutex)) {
6362                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6363                         if (enable_data->enable)
6364                                 test->paused = false;
6365                         else
6366                                 test->paused = true;
6367                 }
6368         }
6369 }
6370
6371 static void
6372 hist_enable_count_trigger(struct event_trigger_data *data,
6373                           struct trace_buffer *buffer,  void *rec,
6374                           struct ring_buffer_event *event)
6375 {
6376         if (!data->count)
6377                 return;
6378
6379         if (data->count != -1)
6380                 (data->count)--;
6381
6382         hist_enable_trigger(data, buffer, rec, event);
6383 }
6384
6385 static struct event_trigger_ops hist_enable_trigger_ops = {
6386         .trigger                = hist_enable_trigger,
6387         .print                  = event_enable_trigger_print,
6388         .init                   = event_trigger_init,
6389         .free                   = event_enable_trigger_free,
6390 };
6391
6392 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6393         .trigger                = hist_enable_count_trigger,
6394         .print                  = event_enable_trigger_print,
6395         .init                   = event_trigger_init,
6396         .free                   = event_enable_trigger_free,
6397 };
6398
6399 static struct event_trigger_ops hist_disable_trigger_ops = {
6400         .trigger                = hist_enable_trigger,
6401         .print                  = event_enable_trigger_print,
6402         .init                   = event_trigger_init,
6403         .free                   = event_enable_trigger_free,
6404 };
6405
6406 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6407         .trigger                = hist_enable_count_trigger,
6408         .print                  = event_enable_trigger_print,
6409         .init                   = event_trigger_init,
6410         .free                   = event_enable_trigger_free,
6411 };
6412
6413 static struct event_trigger_ops *
6414 hist_enable_get_trigger_ops(char *cmd, char *param)
6415 {
6416         struct event_trigger_ops *ops;
6417         bool enable;
6418
6419         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6420
6421         if (enable)
6422                 ops = param ? &hist_enable_count_trigger_ops :
6423                         &hist_enable_trigger_ops;
6424         else
6425                 ops = param ? &hist_disable_count_trigger_ops :
6426                         &hist_disable_trigger_ops;
6427
6428         return ops;
6429 }
6430
6431 static void hist_enable_unreg_all(struct trace_event_file *file)
6432 {
6433         struct event_trigger_data *test, *n;
6434
6435         list_for_each_entry_safe(test, n, &file->triggers, list) {
6436                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6437                         list_del_rcu(&test->list);
6438                         update_cond_flag(file);
6439                         trace_event_trigger_enable_disable(file, 0);
6440                         if (test->ops->free)
6441                                 test->ops->free(test->ops, test);
6442                 }
6443         }
6444 }
6445
6446 static struct event_command trigger_hist_enable_cmd = {
6447         .name                   = ENABLE_HIST_STR,
6448         .trigger_type           = ETT_HIST_ENABLE,
6449         .parse                  = event_enable_trigger_parse,
6450         .reg                    = event_enable_register_trigger,
6451         .unreg                  = event_enable_unregister_trigger,
6452         .unreg_all              = hist_enable_unreg_all,
6453         .get_trigger_ops        = hist_enable_get_trigger_ops,
6454         .set_filter             = set_trigger_filter,
6455 };
6456
6457 static struct event_command trigger_hist_disable_cmd = {
6458         .name                   = DISABLE_HIST_STR,
6459         .trigger_type           = ETT_HIST_ENABLE,
6460         .parse                  = event_enable_trigger_parse,
6461         .reg                    = event_enable_register_trigger,
6462         .unreg                  = event_enable_unregister_trigger,
6463         .unreg_all              = hist_enable_unreg_all,
6464         .get_trigger_ops        = hist_enable_get_trigger_ops,
6465         .set_filter             = set_trigger_filter,
6466 };
6467
6468 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6469 {
6470         unregister_event_command(&trigger_hist_enable_cmd);
6471         unregister_event_command(&trigger_hist_disable_cmd);
6472 }
6473
6474 __init int register_trigger_hist_enable_disable_cmds(void)
6475 {
6476         int ret;
6477
6478         ret = register_event_command(&trigger_hist_enable_cmd);
6479         if (WARN_ON(ret < 0))
6480                 return ret;
6481         ret = register_event_command(&trigger_hist_disable_cmd);
6482         if (WARN_ON(ret < 0))
6483                 unregister_trigger_hist_enable_disable_cmds();
6484
6485         return ret;
6486 }