kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),
  73
  74 #undef C
  75 #define C(a, b)         HIST_ERR_##a
  76
  77 enum { ERRORS };
  78
  79 #undef C
  80 #define C(a, b)         b
  81
  82 static const char *err_text[] = { ERRORS };
  83
  84 struct hist_field;
  85
  86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  87                                 struct tracing_map_elt *elt,
  88                                 struct trace_buffer *buffer,
  89                                 struct ring_buffer_event *rbe,
  90                                 void *event);
  91
  92 #define HIST_FIELD_OPERANDS_MAX 2
  93 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  94 #define HIST_ACTIONS_MAX        8
  95 #define HIST_CONST_DIGITS_MAX   21
  96 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  97
  98 enum field_op_id {
  99         FIELD_OP_NONE,
 100         FIELD_OP_PLUS,
 101         FIELD_OP_MINUS,
 102         FIELD_OP_UNARY_MINUS,
 103         FIELD_OP_DIV,
 104         FIELD_OP_MULT,
 105 };
 106
 107 /*
 108  * A hist_var (histogram variable) contains variable information for
 109  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 110  * flag set.  A hist_var has a variable name e.g. ts0, and is
 111  * associated with a given histogram trigger, as specified by
 112  * hist_data.  The hist_var idx is the unique index assigned to the
 113  * variable by the hist trigger's tracing_map.  The idx is what is
 114  * used to set a variable's value and, by a variable reference, to
 115  * retrieve it.
 116  */
 117 struct hist_var {
 118         char                            *name;
 119         struct hist_trigger_data        *hist_data;
 120         unsigned int                    idx;
 121 };
 122
 123 struct hist_field {
 124         struct ftrace_event_field       *field;
 125         unsigned long                   flags;
 126         hist_field_fn_t                 fn;
 127         unsigned int                    ref;
 128         unsigned int                    size;
 129         unsigned int                    offset;
 130         unsigned int                    is_signed;
 131         unsigned long                   buckets;
 132         const char                      *type;
 133         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 134         struct hist_trigger_data        *hist_data;
 135
 136         /*
 137          * Variable fields contain variable-specific info in var.
 138          */
 139         struct hist_var                 var;
 140         enum field_op_id                operator;
 141         char                            *system;
 142         char                            *event_name;
 143
 144         /*
 145          * The name field is used for EXPR and VAR_REF fields.  VAR
 146          * fields contain the variable name in var.name.
 147          */
 148         char                            *name;
 149
 150         /*
 151          * When a histogram trigger is hit, if it has any references
 152          * to variables, the values of those variables are collected
 153          * into a var_ref_vals array by resolve_var_refs().  The
 154          * current value of each variable is read from the tracing_map
 155          * using the hist field's hist_var.idx and entered into the
 156          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 157          */
 158         unsigned int                    var_ref_idx;
 159         bool                            read_once;
 160
 161         unsigned int                    var_str_idx;
 162
 163         /* Numeric literals are represented as u64 */
 164         u64                             constant;
 165         /* Used to optimize division by constants */
 166         u64                             div_multiplier;
 167 };
 168
 169 static u64 hist_field_none(struct hist_field *field,
 170                            struct tracing_map_elt *elt,
 171                            struct trace_buffer *buffer,
 172                            struct ring_buffer_event *rbe,
 173                            void *event)
 174 {
 175         return 0;
 176 }
 177
 178 static u64 hist_field_const(struct hist_field *field,
 179                            struct tracing_map_elt *elt,
 180                            struct trace_buffer *buffer,
 181                            struct ring_buffer_event *rbe,
 182                            void *event)
 183 {
 184         return field->constant;
 185 }
 186
 187 static u64 hist_field_counter(struct hist_field *field,
 188                               struct tracing_map_elt *elt,
 189                               struct trace_buffer *buffer,
 190                               struct ring_buffer_event *rbe,
 191                               void *event)
 192 {
 193         return 1;
 194 }
 195
 196 static u64 hist_field_string(struct hist_field *hist_field,
 197                              struct tracing_map_elt *elt,
 198                              struct trace_buffer *buffer,
 199                              struct ring_buffer_event *rbe,
 200                              void *event)
 201 {
 202         char *addr = (char *)(event + hist_field->field->offset);
 203
 204         return (u64)(unsigned long)addr;
 205 }
 206
 207 static u64 hist_field_dynstring(struct hist_field *hist_field,
 208                                 struct tracing_map_elt *elt,
 209                                 struct trace_buffer *buffer,
 210                                 struct ring_buffer_event *rbe,
 211                                 void *event)
 212 {
 213         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 214         int str_loc = str_item & 0xffff;
 215         char *addr = (char *)(event + str_loc);
 216
 217         return (u64)(unsigned long)addr;
 218 }
 219
 220 static u64 hist_field_pstring(struct hist_field *hist_field,
 221                               struct tracing_map_elt *elt,
 222                               struct trace_buffer *buffer,
 223                               struct ring_buffer_event *rbe,
 224                               void *event)
 225 {
 226         char **addr = (char **)(event + hist_field->field->offset);
 227
 228         return (u64)(unsigned long)*addr;
 229 }
 230
 231 static u64 hist_field_log2(struct hist_field *hist_field,
 232                            struct tracing_map_elt *elt,
 233                            struct trace_buffer *buffer,
 234                            struct ring_buffer_event *rbe,
 235                            void *event)
 236 {
 237         struct hist_field *operand = hist_field->operands[0];
 238
 239         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 240
 241         return (u64) ilog2(roundup_pow_of_two(val));
 242 }
 243
 244 static u64 hist_field_bucket(struct hist_field *hist_field,
 245                              struct tracing_map_elt *elt,
 246                              struct trace_buffer *buffer,
 247                              struct ring_buffer_event *rbe,
 248                              void *event)
 249 {
 250         struct hist_field *operand = hist_field->operands[0];
 251         unsigned long buckets = hist_field->buckets;
 252
 253         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 254
 255         if (WARN_ON_ONCE(!buckets))
 256                 return val;
 257
 258         if (val >= LONG_MAX)
 259                 val = div64_ul(val, buckets);
 260         else
 261                 val = (u64)((unsigned long)val / buckets);
 262         return val * buckets;
 263 }
 264
 265 static u64 hist_field_plus(struct hist_field *hist_field,
 266                            struct tracing_map_elt *elt,
 267                            struct trace_buffer *buffer,
 268                            struct ring_buffer_event *rbe,
 269                            void *event)
 270 {
 271         struct hist_field *operand1 = hist_field->operands[0];
 272         struct hist_field *operand2 = hist_field->operands[1];
 273
 274         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 275         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 276
 277         return val1 + val2;
 278 }
 279
 280 static u64 hist_field_minus(struct hist_field *hist_field,
 281                             struct tracing_map_elt *elt,
 282                             struct trace_buffer *buffer,
 283                             struct ring_buffer_event *rbe,
 284                             void *event)
 285 {
 286         struct hist_field *operand1 = hist_field->operands[0];
 287         struct hist_field *operand2 = hist_field->operands[1];
 288
 289         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 290         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 291
 292         return val1 - val2;
 293 }
 294
 295 static u64 hist_field_div(struct hist_field *hist_field,
 296                            struct tracing_map_elt *elt,
 297                            struct trace_buffer *buffer,
 298                            struct ring_buffer_event *rbe,
 299                            void *event)
 300 {
 301         struct hist_field *operand1 = hist_field->operands[0];
 302         struct hist_field *operand2 = hist_field->operands[1];
 303
 304         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 305         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 306
 307         /* Return -1 for the undefined case */
 308         if (!val2)
 309                 return -1;
 310
 311         /* Use shift if the divisor is a power of 2 */
 312         if (!(val2 & (val2 - 1)))
 313                 return val1 >> __ffs64(val2);
 314
 315         return div64_u64(val1, val2);
 316 }
 317
 318 static u64 div_by_power_of_two(struct hist_field *hist_field,
 319                                 struct tracing_map_elt *elt,
 320                                 struct trace_buffer *buffer,
 321                                 struct ring_buffer_event *rbe,
 322                                 void *event)
 323 {
 324         struct hist_field *operand1 = hist_field->operands[0];
 325         struct hist_field *operand2 = hist_field->operands[1];
 326
 327         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 328
 329         return val1 >> __ffs64(operand2->constant);
 330 }
 331
 332 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 333                                 struct tracing_map_elt *elt,
 334                                 struct trace_buffer *buffer,
 335                                 struct ring_buffer_event *rbe,
 336                                 void *event)
 337 {
 338         struct hist_field *operand1 = hist_field->operands[0];
 339         struct hist_field *operand2 = hist_field->operands[1];
 340
 341         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 342
 343         return div64_u64(val1, operand2->constant);
 344 }
 345
 346 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 347                                 struct tracing_map_elt *elt,
 348                                 struct trace_buffer *buffer,
 349                                 struct ring_buffer_event *rbe,
 350                                 void *event)
 351 {
 352         struct hist_field *operand1 = hist_field->operands[0];
 353         struct hist_field *operand2 = hist_field->operands[1];
 354
 355         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 356
 357         /*
 358          * If the divisor is a constant, do a multiplication and shift instead.
 359          *
 360          * Choose Z = some power of 2. If Y <= Z, then:
 361          *     X / Y = (X * (Z / Y)) / Z
 362          *
 363          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 364          *     X / Y = (X * mult) / Z
 365          *
 366          * The division by Z can be replaced by a shift since Z is a power of 2:
 367          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 368          *
 369          * As long, as X < Z the results will not be off by more than 1.
 370          */
 371         if (val1 < (1 << HIST_DIV_SHIFT)) {
 372                 u64 mult = operand2->div_multiplier;
 373
 374                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 375         }
 376
 377         return div64_u64(val1, operand2->constant);
 378 }
 379
 380 static u64 hist_field_mult(struct hist_field *hist_field,
 381                            struct tracing_map_elt *elt,
 382                            struct trace_buffer *buffer,
 383                            struct ring_buffer_event *rbe,
 384                            void *event)
 385 {
 386         struct hist_field *operand1 = hist_field->operands[0];
 387         struct hist_field *operand2 = hist_field->operands[1];
 388
 389         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 390         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 391
 392         return val1 * val2;
 393 }
 394
 395 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 396                                   struct tracing_map_elt *elt,
 397                                   struct trace_buffer *buffer,
 398                                   struct ring_buffer_event *rbe,
 399                                   void *event)
 400 {
 401         struct hist_field *operand = hist_field->operands[0];
 402
 403         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 404         u64 val = (u64)-sval;
 405
 406         return val;
 407 }
 408
 409 #define DEFINE_HIST_FIELD_FN(type)                                      \
 410         static u64 hist_field_##type(struct hist_field *hist_field,     \
 411                                      struct tracing_map_elt *elt,       \
 412                                      struct trace_buffer *buffer,       \
 413                                      struct ring_buffer_event *rbe,     \
 414                                      void *event)                       \
 415 {                                                                       \
 416         type *addr = (type *)(event + hist_field->field->offset);       \
 417                                                                         \
 418         return (u64)(unsigned long)*addr;                               \
 419 }
 420
 421 DEFINE_HIST_FIELD_FN(s64);
 422 DEFINE_HIST_FIELD_FN(u64);
 423 DEFINE_HIST_FIELD_FN(s32);
 424 DEFINE_HIST_FIELD_FN(u32);
 425 DEFINE_HIST_FIELD_FN(s16);
 426 DEFINE_HIST_FIELD_FN(u16);
 427 DEFINE_HIST_FIELD_FN(s8);
 428 DEFINE_HIST_FIELD_FN(u8);
 429
 430 #define for_each_hist_field(i, hist_data)       \
 431         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 432
 433 #define for_each_hist_val_field(i, hist_data)   \
 434         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 435
 436 #define for_each_hist_key_field(i, hist_data)   \
 437         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 438
 439 #define HIST_STACKTRACE_DEPTH   16
 440 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 441 #define HIST_STACKTRACE_SKIP    5
 442
 443 #define HITCOUNT_IDX            0
 444 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 445
 446 enum hist_field_flags {
 447         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 448         HIST_FIELD_FL_KEY               = 1 << 1,
 449         HIST_FIELD_FL_STRING            = 1 << 2,
 450         HIST_FIELD_FL_HEX               = 1 << 3,
 451         HIST_FIELD_FL_SYM               = 1 << 4,
 452         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 453         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 454         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 455         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 456         HIST_FIELD_FL_LOG2              = 1 << 9,
 457         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 458         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 459         HIST_FIELD_FL_VAR               = 1 << 12,
 460         HIST_FIELD_FL_EXPR              = 1 << 13,
 461         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 462         HIST_FIELD_FL_CPU               = 1 << 15,
 463         HIST_FIELD_FL_ALIAS             = 1 << 16,
 464         HIST_FIELD_FL_BUCKET            = 1 << 17,
 465         HIST_FIELD_FL_CONST             = 1 << 18,
 466 };
 467
 468 struct var_defs {
 469         unsigned int    n_vars;
 470         char            *name[TRACING_MAP_VARS_MAX];
 471         char            *expr[TRACING_MAP_VARS_MAX];
 472 };
 473
 474 struct hist_trigger_attrs {
 475         char            *keys_str;
 476         char            *vals_str;
 477         char            *sort_key_str;
 478         char            *name;
 479         char            *clock;
 480         bool            pause;
 481         bool            cont;
 482         bool            clear;
 483         bool            ts_in_usecs;
 484         unsigned int    map_bits;
 485
 486         char            *assignment_str[TRACING_MAP_VARS_MAX];
 487         unsigned int    n_assignments;
 488
 489         char            *action_str[HIST_ACTIONS_MAX];
 490         unsigned int    n_actions;
 491
 492         struct var_defs var_defs;
 493 };
 494
 495 struct field_var {
 496         struct hist_field       *var;
 497         struct hist_field       *val;
 498 };
 499
 500 struct field_var_hist {
 501         struct hist_trigger_data        *hist_data;
 502         char                            *cmd;
 503 };
 504
 505 struct hist_trigger_data {
 506         struct hist_field               *fields[HIST_FIELDS_MAX];
 507         unsigned int                    n_vals;
 508         unsigned int                    n_keys;
 509         unsigned int                    n_fields;
 510         unsigned int                    n_vars;
 511         unsigned int                    n_var_str;
 512         unsigned int                    key_size;
 513         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 514         unsigned int                    n_sort_keys;
 515         struct trace_event_file         *event_file;
 516         struct hist_trigger_attrs       *attrs;
 517         struct tracing_map              *map;
 518         bool                            enable_timestamps;
 519         bool                            remove;
 520         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 521         unsigned int                    n_var_refs;
 522
 523         struct action_data              *actions[HIST_ACTIONS_MAX];
 524         unsigned int                    n_actions;
 525
 526         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 527         unsigned int                    n_field_vars;
 528         unsigned int                    n_field_var_str;
 529         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 530         unsigned int                    n_field_var_hists;
 531
 532         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 533         unsigned int                    n_save_vars;
 534         unsigned int                    n_save_var_str;
 535 };
 536
 537 struct action_data;
 538
 539 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 540                              struct tracing_map_elt *elt,
 541                              struct trace_buffer *buffer, void *rec,
 542                              struct ring_buffer_event *rbe, void *key,
 543                              struct action_data *data, u64 *var_ref_vals);
 544
 545 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 546
 547 enum handler_id {
 548         HANDLER_ONMATCH = 1,
 549         HANDLER_ONMAX,
 550         HANDLER_ONCHANGE,
 551 };
 552
 553 enum action_id {
 554         ACTION_SAVE = 1,
 555         ACTION_TRACE,
 556         ACTION_SNAPSHOT,
 557 };
 558
 559 struct action_data {
 560         enum handler_id         handler;
 561         enum action_id          action;
 562         char                    *action_name;
 563         action_fn_t             fn;
 564
 565         unsigned int            n_params;
 566         char                    *params[SYNTH_FIELDS_MAX];
 567
 568         /*
 569          * When a histogram trigger is hit, the values of any
 570          * references to variables, including variables being passed
 571          * as parameters to synthetic events, are collected into a
 572          * var_ref_vals array.  This var_ref_idx array is an array of
 573          * indices into the var_ref_vals array, one for each synthetic
 574          * event param, and is passed to the synthetic event
 575          * invocation.
 576          */
 577         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 578         struct synth_event      *synth_event;
 579         bool                    use_trace_keyword;
 580         char                    *synth_event_name;
 581
 582         union {
 583                 struct {
 584                         char                    *event;
 585                         char                    *event_system;
 586                 } match_data;
 587
 588                 struct {
 589                         /*
 590                          * var_str contains the $-unstripped variable
 591                          * name referenced by var_ref, and used when
 592                          * printing the action.  Because var_ref
 593                          * creation is deferred to create_actions(),
 594                          * we need a per-action way to save it until
 595                          * then, thus var_str.
 596                          */
 597                         char                    *var_str;
 598
 599                         /*
 600                          * var_ref refers to the variable being
 601                          * tracked e.g onmax($var).
 602                          */
 603                         struct hist_field       *var_ref;
 604
 605                         /*
 606                          * track_var contains the 'invisible' tracking
 607                          * variable created to keep the current
 608                          * e.g. max value.
 609                          */
 610                         struct hist_field       *track_var;
 611
 612                         check_track_val_fn_t    check_val;
 613                         action_fn_t             save_data;
 614                 } track_data;
 615         };
 616 };
 617
 618 struct track_data {
 619         u64                             track_val;
 620         bool                            updated;
 621
 622         unsigned int                    key_len;
 623         void                            *key;
 624         struct tracing_map_elt          elt;
 625
 626         struct action_data              *action_data;
 627         struct hist_trigger_data        *hist_data;
 628 };
 629
 630 struct hist_elt_data {
 631         char *comm;
 632         u64 *var_ref_vals;
 633         char **field_var_str;
 634         int n_field_var_str;
 635 };
 636
 637 struct snapshot_context {
 638         struct tracing_map_elt  *elt;
 639         void                    *key;
 640 };
 641
 642 /*
 643  * Returns the specific division function to use if the divisor
 644  * is constant. This avoids extra branches when the trigger is hit.
 645  */
 646 static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
 647 {
 648         u64 div = divisor->constant;
 649
 650         if (!(div & (div - 1)))
 651                 return div_by_power_of_two;
 652
 653         /* If the divisor is too large, do a regular division */
 654         if (div > (1 << HIST_DIV_SHIFT))
 655                 return div_by_not_power_of_two;
 656
 657         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 658         return div_by_mult_and_shift;
 659 }
 660
 661 static void track_data_free(struct track_data *track_data)
 662 {
 663         struct hist_elt_data *elt_data;
 664
 665         if (!track_data)
 666                 return;
 667
 668         kfree(track_data->key);
 669
 670         elt_data = track_data->elt.private_data;
 671         if (elt_data) {
 672                 kfree(elt_data->comm);
 673                 kfree(elt_data);
 674         }
 675
 676         kfree(track_data);
 677 }
 678
 679 static struct track_data *track_data_alloc(unsigned int key_len,
 680                                            struct action_data *action_data,
 681                                            struct hist_trigger_data *hist_data)
 682 {
 683         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 684         struct hist_elt_data *elt_data;
 685
 686         if (!data)
 687                 return ERR_PTR(-ENOMEM);
 688
 689         data->key = kzalloc(key_len, GFP_KERNEL);
 690         if (!data->key) {
 691                 track_data_free(data);
 692                 return ERR_PTR(-ENOMEM);
 693         }
 694
 695         data->key_len = key_len;
 696         data->action_data = action_data;
 697         data->hist_data = hist_data;
 698
 699         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 700         if (!elt_data) {
 701                 track_data_free(data);
 702                 return ERR_PTR(-ENOMEM);
 703         }
 704
 705         data->elt.private_data = elt_data;
 706
 707         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 708         if (!elt_data->comm) {
 709                 track_data_free(data);
 710                 return ERR_PTR(-ENOMEM);
 711         }
 712
 713         return data;
 714 }
 715
 716 static char last_cmd[MAX_FILTER_STR_VAL];
 717 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 718
 719 static int errpos(char *str)
 720 {
 721         return err_pos(last_cmd, str);
 722 }
 723
 724 static void last_cmd_set(struct trace_event_file *file, char *str)
 725 {
 726         const char *system = NULL, *name = NULL;
 727         struct trace_event_call *call;
 728
 729         if (!str)
 730                 return;
 731
 732         strcpy(last_cmd, "hist:");
 733         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 734
 735         if (file) {
 736                 call = file->event_call;
 737                 system = call->class->system;
 738                 if (system) {
 739                         name = trace_event_name(call);
 740                         if (!name)
 741                                 system = NULL;
 742                 }
 743         }
 744
 745         if (system)
 746                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 747 }
 748
 749 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 750 {
 751         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 752                         err_type, err_pos);
 753 }
 754
 755 static void hist_err_clear(void)
 756 {
 757         last_cmd[0] = '\0';
 758         last_cmd_loc[0] = '\0';
 759 }
 760
 761 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 762                                     unsigned int *var_ref_idx);
 763
 764 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 765                                unsigned int *var_ref_idx)
 766 {
 767         struct tracepoint *tp = event->tp;
 768
 769         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 770                 struct tracepoint_func *probe_func_ptr;
 771                 synth_probe_func_t probe_func;
 772                 void *__data;
 773
 774                 if (!(cpu_online(raw_smp_processor_id())))
 775                         return;
 776
 777                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 778                 if (probe_func_ptr) {
 779                         do {
 780                                 probe_func = probe_func_ptr->func;
 781                                 __data = probe_func_ptr->data;
 782                                 probe_func(__data, var_ref_vals, var_ref_idx);
 783                         } while ((++probe_func_ptr)->func);
 784                 }
 785         }
 786 }
 787
 788 static void action_trace(struct hist_trigger_data *hist_data,
 789                          struct tracing_map_elt *elt,
 790                          struct trace_buffer *buffer, void *rec,
 791                          struct ring_buffer_event *rbe, void *key,
 792                          struct action_data *data, u64 *var_ref_vals)
 793 {
 794         struct synth_event *event = data->synth_event;
 795
 796         trace_synth(event, var_ref_vals, data->var_ref_idx);
 797 }
 798
 799 struct hist_var_data {
 800         struct list_head list;
 801         struct hist_trigger_data *hist_data;
 802 };
 803
 804 static u64 hist_field_timestamp(struct hist_field *hist_field,
 805                                 struct tracing_map_elt *elt,
 806                                 struct trace_buffer *buffer,
 807                                 struct ring_buffer_event *rbe,
 808                                 void *event)
 809 {
 810         struct hist_trigger_data *hist_data = hist_field->hist_data;
 811         struct trace_array *tr = hist_data->event_file->tr;
 812
 813         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 814
 815         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 816                 ts = ns2usecs(ts);
 817
 818         return ts;
 819 }
 820
 821 static u64 hist_field_cpu(struct hist_field *hist_field,
 822                           struct tracing_map_elt *elt,
 823                           struct trace_buffer *buffer,
 824                           struct ring_buffer_event *rbe,
 825                           void *event)
 826 {
 827         int cpu = smp_processor_id();
 828
 829         return cpu;
 830 }
 831
 832 /**
 833  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 834  * @hist_field: The VAR_REF field to check
 835  * @var_data: The hist trigger that owns the variable
 836  * @var_idx: The trigger variable identifier
 837  *
 838  * Check the given VAR_REF field to see whether or not it references
 839  * the given variable associated with the given trigger.
 840  *
 841  * Return: The VAR_REF field if it does reference the variable, NULL if not
 842  */
 843 static struct hist_field *
 844 check_field_for_var_ref(struct hist_field *hist_field,
 845                         struct hist_trigger_data *var_data,
 846                         unsigned int var_idx)
 847 {
 848         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 849
 850         if (hist_field && hist_field->var.idx == var_idx &&
 851             hist_field->var.hist_data == var_data)
 852                 return hist_field;
 853
 854         return NULL;
 855 }
 856
 857 /**
 858  * find_var_ref - Check if a trigger has a reference to a trigger variable
 859  * @hist_data: The hist trigger that might have a reference to the variable
 860  * @var_data: The hist trigger that owns the variable
 861  * @var_idx: The trigger variable identifier
 862  *
 863  * Check the list of var_refs[] on the first hist trigger to see
 864  * whether any of them are references to the variable on the second
 865  * trigger.
 866  *
 867  * Return: The VAR_REF field referencing the variable if so, NULL if not
 868  */
 869 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 870                                        struct hist_trigger_data *var_data,
 871                                        unsigned int var_idx)
 872 {
 873         struct hist_field *hist_field;
 874         unsigned int i;
 875
 876         for (i = 0; i < hist_data->n_var_refs; i++) {
 877                 hist_field = hist_data->var_refs[i];
 878                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 879                         return hist_field;
 880         }
 881
 882         return NULL;
 883 }
 884
 885 /**
 886  * find_any_var_ref - Check if there is a reference to a given trigger variable
 887  * @hist_data: The hist trigger
 888  * @var_idx: The trigger variable identifier
 889  *
 890  * Check to see whether the given variable is currently referenced by
 891  * any other trigger.
 892  *
 893  * The trigger the variable is defined on is explicitly excluded - the
 894  * assumption being that a self-reference doesn't prevent a trigger
 895  * from being removed.
 896  *
 897  * Return: The VAR_REF field referencing the variable if so, NULL if not
 898  */
 899 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 900                                            unsigned int var_idx)
 901 {
 902         struct trace_array *tr = hist_data->event_file->tr;
 903         struct hist_field *found = NULL;
 904         struct hist_var_data *var_data;
 905
 906         list_for_each_entry(var_data, &tr->hist_vars, list) {
 907                 if (var_data->hist_data == hist_data)
 908                         continue;
 909                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 910                 if (found)
 911                         break;
 912         }
 913
 914         return found;
 915 }
 916
 917 /**
 918  * check_var_refs - Check if there is a reference to any of trigger's variables
 919  * @hist_data: The hist trigger
 920  *
 921  * A trigger can define one or more variables.  If any one of them is
 922  * currently referenced by any other trigger, this function will
 923  * determine that.
 924
 925  * Typically used to determine whether or not a trigger can be removed
 926  * - if there are any references to a trigger's variables, it cannot.
 927  *
 928  * Return: True if there is a reference to any of trigger's variables
 929  */
 930 static bool check_var_refs(struct hist_trigger_data *hist_data)
 931 {
 932         struct hist_field *field;
 933         bool found = false;
 934         int i;
 935
 936         for_each_hist_field(i, hist_data) {
 937                 field = hist_data->fields[i];
 938                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 939                         if (find_any_var_ref(hist_data, field->var.idx)) {
 940                                 found = true;
 941                                 break;
 942                         }
 943                 }
 944         }
 945
 946         return found;
 947 }
 948
 949 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 950 {
 951         struct trace_array *tr = hist_data->event_file->tr;
 952         struct hist_var_data *var_data, *found = NULL;
 953
 954         list_for_each_entry(var_data, &tr->hist_vars, list) {
 955                 if (var_data->hist_data == hist_data) {
 956                         found = var_data;
 957                         break;
 958                 }
 959         }
 960
 961         return found;
 962 }
 963
 964 static bool field_has_hist_vars(struct hist_field *hist_field,
 965                                 unsigned int level)
 966 {
 967         int i;
 968
 969         if (level > 3)
 970                 return false;
 971
 972         if (!hist_field)
 973                 return false;
 974
 975         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 976             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 977                 return true;
 978
 979         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 980                 struct hist_field *operand;
 981
 982                 operand = hist_field->operands[i];
 983                 if (field_has_hist_vars(operand, level + 1))
 984                         return true;
 985         }
 986
 987         return false;
 988 }
 989
 990 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 991 {
 992         struct hist_field *hist_field;
 993         int i;
 994
 995         for_each_hist_field(i, hist_data) {
 996                 hist_field = hist_data->fields[i];
 997                 if (field_has_hist_vars(hist_field, 0))
 998                         return true;
 999         }
1000
1001         return false;
1002 }
1003
1004 static int save_hist_vars(struct hist_trigger_data *hist_data)
1005 {
1006         struct trace_array *tr = hist_data->event_file->tr;
1007         struct hist_var_data *var_data;
1008
1009         var_data = find_hist_vars(hist_data);
1010         if (var_data)
1011                 return 0;
1012
1013         if (tracing_check_open_get_tr(tr))
1014                 return -ENODEV;
1015
1016         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1017         if (!var_data) {
1018                 trace_array_put(tr);
1019                 return -ENOMEM;
1020         }
1021
1022         var_data->hist_data = hist_data;
1023         list_add(&var_data->list, &tr->hist_vars);
1024
1025         return 0;
1026 }
1027
1028 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1029 {
1030         struct trace_array *tr = hist_data->event_file->tr;
1031         struct hist_var_data *var_data;
1032
1033         var_data = find_hist_vars(hist_data);
1034         if (!var_data)
1035                 return;
1036
1037         if (WARN_ON(check_var_refs(hist_data)))
1038                 return;
1039
1040         list_del(&var_data->list);
1041
1042         kfree(var_data);
1043
1044         trace_array_put(tr);
1045 }
1046
1047 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1048                                          const char *var_name)
1049 {
1050         struct hist_field *hist_field, *found = NULL;
1051         int i;
1052
1053         for_each_hist_field(i, hist_data) {
1054                 hist_field = hist_data->fields[i];
1055                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1056                     strcmp(hist_field->var.name, var_name) == 0) {
1057                         found = hist_field;
1058                         break;
1059                 }
1060         }
1061
1062         return found;
1063 }
1064
1065 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1066                                    struct trace_event_file *file,
1067                                    const char *var_name)
1068 {
1069         struct hist_trigger_data *test_data;
1070         struct event_trigger_data *test;
1071         struct hist_field *hist_field;
1072
1073         lockdep_assert_held(&event_mutex);
1074
1075         hist_field = find_var_field(hist_data, var_name);
1076         if (hist_field)
1077                 return hist_field;
1078
1079         list_for_each_entry(test, &file->triggers, list) {
1080                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1081                         test_data = test->private_data;
1082                         hist_field = find_var_field(test_data, var_name);
1083                         if (hist_field)
1084                                 return hist_field;
1085                 }
1086         }
1087
1088         return NULL;
1089 }
1090
1091 static struct trace_event_file *find_var_file(struct trace_array *tr,
1092                                               char *system,
1093                                               char *event_name,
1094                                               char *var_name)
1095 {
1096         struct hist_trigger_data *var_hist_data;
1097         struct hist_var_data *var_data;
1098         struct trace_event_file *file, *found = NULL;
1099
1100         if (system)
1101                 return find_event_file(tr, system, event_name);
1102
1103         list_for_each_entry(var_data, &tr->hist_vars, list) {
1104                 var_hist_data = var_data->hist_data;
1105                 file = var_hist_data->event_file;
1106                 if (file == found)
1107                         continue;
1108
1109                 if (find_var_field(var_hist_data, var_name)) {
1110                         if (found) {
1111                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1112                                 return NULL;
1113                         }
1114
1115                         found = file;
1116                 }
1117         }
1118
1119         return found;
1120 }
1121
1122 static struct hist_field *find_file_var(struct trace_event_file *file,
1123                                         const char *var_name)
1124 {
1125         struct hist_trigger_data *test_data;
1126         struct event_trigger_data *test;
1127         struct hist_field *hist_field;
1128
1129         lockdep_assert_held(&event_mutex);
1130
1131         list_for_each_entry(test, &file->triggers, list) {
1132                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1133                         test_data = test->private_data;
1134                         hist_field = find_var_field(test_data, var_name);
1135                         if (hist_field)
1136                                 return hist_field;
1137                 }
1138         }
1139
1140         return NULL;
1141 }
1142
1143 static struct hist_field *
1144 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1145 {
1146         struct trace_array *tr = hist_data->event_file->tr;
1147         struct hist_field *hist_field, *found = NULL;
1148         struct trace_event_file *file;
1149         unsigned int i;
1150
1151         for (i = 0; i < hist_data->n_actions; i++) {
1152                 struct action_data *data = hist_data->actions[i];
1153
1154                 if (data->handler == HANDLER_ONMATCH) {
1155                         char *system = data->match_data.event_system;
1156                         char *event_name = data->match_data.event;
1157
1158                         file = find_var_file(tr, system, event_name, var_name);
1159                         if (!file)
1160                                 continue;
1161                         hist_field = find_file_var(file, var_name);
1162                         if (hist_field) {
1163                                 if (found) {
1164                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1165                                                  errpos(var_name));
1166                                         return ERR_PTR(-EINVAL);
1167                                 }
1168
1169                                 found = hist_field;
1170                         }
1171                 }
1172         }
1173         return found;
1174 }
1175
1176 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1177                                          char *system,
1178                                          char *event_name,
1179                                          char *var_name)
1180 {
1181         struct trace_array *tr = hist_data->event_file->tr;
1182         struct hist_field *hist_field = NULL;
1183         struct trace_event_file *file;
1184
1185         if (!system || !event_name) {
1186                 hist_field = find_match_var(hist_data, var_name);
1187                 if (IS_ERR(hist_field))
1188                         return NULL;
1189                 if (hist_field)
1190                         return hist_field;
1191         }
1192
1193         file = find_var_file(tr, system, event_name, var_name);
1194         if (!file)
1195                 return NULL;
1196
1197         hist_field = find_file_var(file, var_name);
1198
1199         return hist_field;
1200 }
1201
1202 static u64 hist_field_var_ref(struct hist_field *hist_field,
1203                               struct tracing_map_elt *elt,
1204                               struct trace_buffer *buffer,
1205                               struct ring_buffer_event *rbe,
1206                               void *event)
1207 {
1208         struct hist_elt_data *elt_data;
1209         u64 var_val = 0;
1210
1211         if (WARN_ON_ONCE(!elt))
1212                 return var_val;
1213
1214         elt_data = elt->private_data;
1215         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1216
1217         return var_val;
1218 }
1219
1220 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1221                              u64 *var_ref_vals, bool self)
1222 {
1223         struct hist_trigger_data *var_data;
1224         struct tracing_map_elt *var_elt;
1225         struct hist_field *hist_field;
1226         unsigned int i, var_idx;
1227         bool resolved = true;
1228         u64 var_val = 0;
1229
1230         for (i = 0; i < hist_data->n_var_refs; i++) {
1231                 hist_field = hist_data->var_refs[i];
1232                 var_idx = hist_field->var.idx;
1233                 var_data = hist_field->var.hist_data;
1234
1235                 if (var_data == NULL) {
1236                         resolved = false;
1237                         break;
1238                 }
1239
1240                 if ((self && var_data != hist_data) ||
1241                     (!self && var_data == hist_data))
1242                         continue;
1243
1244                 var_elt = tracing_map_lookup(var_data->map, key);
1245                 if (!var_elt) {
1246                         resolved = false;
1247                         break;
1248                 }
1249
1250                 if (!tracing_map_var_set(var_elt, var_idx)) {
1251                         resolved = false;
1252                         break;
1253                 }
1254
1255                 if (self || !hist_field->read_once)
1256                         var_val = tracing_map_read_var(var_elt, var_idx);
1257                 else
1258                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1259
1260                 var_ref_vals[i] = var_val;
1261         }
1262
1263         return resolved;
1264 }
1265
1266 static const char *hist_field_name(struct hist_field *field,
1267                                    unsigned int level)
1268 {
1269         const char *field_name = "";
1270
1271         if (level > 1)
1272                 return field_name;
1273
1274         if (field->field)
1275                 field_name = field->field->name;
1276         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1277                  field->flags & HIST_FIELD_FL_ALIAS ||
1278                  field->flags & HIST_FIELD_FL_BUCKET)
1279                 field_name = hist_field_name(field->operands[0], ++level);
1280         else if (field->flags & HIST_FIELD_FL_CPU)
1281                 field_name = "common_cpu";
1282         else if (field->flags & HIST_FIELD_FL_EXPR ||
1283                  field->flags & HIST_FIELD_FL_VAR_REF) {
1284                 if (field->system) {
1285                         static char full_name[MAX_FILTER_STR_VAL];
1286
1287                         strcat(full_name, field->system);
1288                         strcat(full_name, ".");
1289                         strcat(full_name, field->event_name);
1290                         strcat(full_name, ".");
1291                         strcat(full_name, field->name);
1292                         field_name = full_name;
1293                 } else
1294                         field_name = field->name;
1295         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1296                 field_name = "common_timestamp";
1297
1298         if (field_name == NULL)
1299                 field_name = "";
1300
1301         return field_name;
1302 }
1303
1304 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1305 {
1306         hist_field_fn_t fn = NULL;
1307
1308         switch (field_size) {
1309         case 8:
1310                 if (field_is_signed)
1311                         fn = hist_field_s64;
1312                 else
1313                         fn = hist_field_u64;
1314                 break;
1315         case 4:
1316                 if (field_is_signed)
1317                         fn = hist_field_s32;
1318                 else
1319                         fn = hist_field_u32;
1320                 break;
1321         case 2:
1322                 if (field_is_signed)
1323                         fn = hist_field_s16;
1324                 else
1325                         fn = hist_field_u16;
1326                 break;
1327         case 1:
1328                 if (field_is_signed)
1329                         fn = hist_field_s8;
1330                 else
1331                         fn = hist_field_u8;
1332                 break;
1333         }
1334
1335         return fn;
1336 }
1337
1338 static int parse_map_size(char *str)
1339 {
1340         unsigned long size, map_bits;
1341         int ret;
1342
1343         ret = kstrtoul(str, 0, &size);
1344         if (ret)
1345                 goto out;
1346
1347         map_bits = ilog2(roundup_pow_of_two(size));
1348         if (map_bits < TRACING_MAP_BITS_MIN ||
1349             map_bits > TRACING_MAP_BITS_MAX)
1350                 ret = -EINVAL;
1351         else
1352                 ret = map_bits;
1353  out:
1354         return ret;
1355 }
1356
1357 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1358 {
1359         unsigned int i;
1360
1361         if (!attrs)
1362                 return;
1363
1364         for (i = 0; i < attrs->n_assignments; i++)
1365                 kfree(attrs->assignment_str[i]);
1366
1367         for (i = 0; i < attrs->n_actions; i++)
1368                 kfree(attrs->action_str[i]);
1369
1370         kfree(attrs->name);
1371         kfree(attrs->sort_key_str);
1372         kfree(attrs->keys_str);
1373         kfree(attrs->vals_str);
1374         kfree(attrs->clock);
1375         kfree(attrs);
1376 }
1377
1378 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1379 {
1380         int ret = -EINVAL;
1381
1382         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1383                 return ret;
1384
1385         if ((str_has_prefix(str, "onmatch(")) ||
1386             (str_has_prefix(str, "onmax(")) ||
1387             (str_has_prefix(str, "onchange("))) {
1388                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1389                 if (!attrs->action_str[attrs->n_actions]) {
1390                         ret = -ENOMEM;
1391                         return ret;
1392                 }
1393                 attrs->n_actions++;
1394                 ret = 0;
1395         }
1396         return ret;
1397 }
1398
1399 static int parse_assignment(struct trace_array *tr,
1400                             char *str, struct hist_trigger_attrs *attrs)
1401 {
1402         int len, ret = 0;
1403
1404         if ((len = str_has_prefix(str, "key=")) ||
1405             (len = str_has_prefix(str, "keys="))) {
1406                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1407                 if (!attrs->keys_str) {
1408                         ret = -ENOMEM;
1409                         goto out;
1410                 }
1411         } else if ((len = str_has_prefix(str, "val=")) ||
1412                    (len = str_has_prefix(str, "vals=")) ||
1413                    (len = str_has_prefix(str, "values="))) {
1414                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1415                 if (!attrs->vals_str) {
1416                         ret = -ENOMEM;
1417                         goto out;
1418                 }
1419         } else if ((len = str_has_prefix(str, "sort="))) {
1420                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1421                 if (!attrs->sort_key_str) {
1422                         ret = -ENOMEM;
1423                         goto out;
1424                 }
1425         } else if (str_has_prefix(str, "name=")) {
1426                 attrs->name = kstrdup(str, GFP_KERNEL);
1427                 if (!attrs->name) {
1428                         ret = -ENOMEM;
1429                         goto out;
1430                 }
1431         } else if ((len = str_has_prefix(str, "clock="))) {
1432                 str += len;
1433
1434                 str = strstrip(str);
1435                 attrs->clock = kstrdup(str, GFP_KERNEL);
1436                 if (!attrs->clock) {
1437                         ret = -ENOMEM;
1438                         goto out;
1439                 }
1440         } else if ((len = str_has_prefix(str, "size="))) {
1441                 int map_bits = parse_map_size(str + len);
1442
1443                 if (map_bits < 0) {
1444                         ret = map_bits;
1445                         goto out;
1446                 }
1447                 attrs->map_bits = map_bits;
1448         } else {
1449                 char *assignment;
1450
1451                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1452                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1453                         ret = -EINVAL;
1454                         goto out;
1455                 }
1456
1457                 assignment = kstrdup(str, GFP_KERNEL);
1458                 if (!assignment) {
1459                         ret = -ENOMEM;
1460                         goto out;
1461                 }
1462
1463                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1464         }
1465  out:
1466         return ret;
1467 }
1468
1469 static struct hist_trigger_attrs *
1470 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1471 {
1472         struct hist_trigger_attrs *attrs;
1473         int ret = 0;
1474
1475         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1476         if (!attrs)
1477                 return ERR_PTR(-ENOMEM);
1478
1479         while (trigger_str) {
1480                 char *str = strsep(&trigger_str, ":");
1481                 char *rhs;
1482
1483                 rhs = strchr(str, '=');
1484                 if (rhs) {
1485                         if (!strlen(++rhs)) {
1486                                 ret = -EINVAL;
1487                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1488                                 goto free;
1489                         }
1490                         ret = parse_assignment(tr, str, attrs);
1491                         if (ret)
1492                                 goto free;
1493                 } else if (strcmp(str, "pause") == 0)
1494                         attrs->pause = true;
1495                 else if ((strcmp(str, "cont") == 0) ||
1496                          (strcmp(str, "continue") == 0))
1497                         attrs->cont = true;
1498                 else if (strcmp(str, "clear") == 0)
1499                         attrs->clear = true;
1500                 else {
1501                         ret = parse_action(str, attrs);
1502                         if (ret)
1503                                 goto free;
1504                 }
1505         }
1506
1507         if (!attrs->keys_str) {
1508                 ret = -EINVAL;
1509                 goto free;
1510         }
1511
1512         if (!attrs->clock) {
1513                 attrs->clock = kstrdup("global", GFP_KERNEL);
1514                 if (!attrs->clock) {
1515                         ret = -ENOMEM;
1516                         goto free;
1517                 }
1518         }
1519
1520         return attrs;
1521  free:
1522         destroy_hist_trigger_attrs(attrs);
1523
1524         return ERR_PTR(ret);
1525 }
1526
1527 static inline void save_comm(char *comm, struct task_struct *task)
1528 {
1529         if (!task->pid) {
1530                 strcpy(comm, "<idle>");
1531                 return;
1532         }
1533
1534         if (WARN_ON_ONCE(task->pid < 0)) {
1535                 strcpy(comm, "<XXX>");
1536                 return;
1537         }
1538
1539         strncpy(comm, task->comm, TASK_COMM_LEN);
1540 }
1541
1542 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1543 {
1544         unsigned int i;
1545
1546         for (i = 0; i < elt_data->n_field_var_str; i++)
1547                 kfree(elt_data->field_var_str[i]);
1548
1549         kfree(elt_data->field_var_str);
1550
1551         kfree(elt_data->comm);
1552         kfree(elt_data);
1553 }
1554
1555 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1556 {
1557         struct hist_elt_data *elt_data = elt->private_data;
1558
1559         hist_elt_data_free(elt_data);
1560 }
1561
1562 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1563 {
1564         struct hist_trigger_data *hist_data = elt->map->private_data;
1565         unsigned int size = TASK_COMM_LEN;
1566         struct hist_elt_data *elt_data;
1567         struct hist_field *hist_field;
1568         unsigned int i, n_str;
1569
1570         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1571         if (!elt_data)
1572                 return -ENOMEM;
1573
1574         for_each_hist_field(i, hist_data) {
1575                 hist_field = hist_data->fields[i];
1576
1577                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1578                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1579                         if (!elt_data->comm) {
1580                                 kfree(elt_data);
1581                                 return -ENOMEM;
1582                         }
1583                         break;
1584                 }
1585         }
1586
1587         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1588                 hist_data->n_var_str;
1589         if (n_str > SYNTH_FIELDS_MAX) {
1590                 hist_elt_data_free(elt_data);
1591                 return -EINVAL;
1592         }
1593
1594         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1595
1596         size = STR_VAR_LEN_MAX;
1597
1598         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1599         if (!elt_data->field_var_str) {
1600                 hist_elt_data_free(elt_data);
1601                 return -EINVAL;
1602         }
1603         elt_data->n_field_var_str = n_str;
1604
1605         for (i = 0; i < n_str; i++) {
1606                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1607                 if (!elt_data->field_var_str[i]) {
1608                         hist_elt_data_free(elt_data);
1609                         return -ENOMEM;
1610                 }
1611         }
1612
1613         elt->private_data = elt_data;
1614
1615         return 0;
1616 }
1617
1618 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1619 {
1620         struct hist_elt_data *elt_data = elt->private_data;
1621
1622         if (elt_data->comm)
1623                 save_comm(elt_data->comm, current);
1624 }
1625
1626 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1627         .elt_alloc      = hist_trigger_elt_data_alloc,
1628         .elt_free       = hist_trigger_elt_data_free,
1629         .elt_init       = hist_trigger_elt_data_init,
1630 };
1631
1632 static const char *get_hist_field_flags(struct hist_field *hist_field)
1633 {
1634         const char *flags_str = NULL;
1635
1636         if (hist_field->flags & HIST_FIELD_FL_HEX)
1637                 flags_str = "hex";
1638         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1639                 flags_str = "sym";
1640         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1641                 flags_str = "sym-offset";
1642         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1643                 flags_str = "execname";
1644         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1645                 flags_str = "syscall";
1646         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1647                 flags_str = "log2";
1648         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1649                 flags_str = "buckets";
1650         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1651                 flags_str = "usecs";
1652
1653         return flags_str;
1654 }
1655
1656 static void expr_field_str(struct hist_field *field, char *expr)
1657 {
1658         if (field->flags & HIST_FIELD_FL_VAR_REF)
1659                 strcat(expr, "$");
1660         else if (field->flags & HIST_FIELD_FL_CONST) {
1661                 char str[HIST_CONST_DIGITS_MAX];
1662
1663                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1664                 strcat(expr, str);
1665         }
1666
1667         strcat(expr, hist_field_name(field, 0));
1668
1669         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1670                 const char *flags_str = get_hist_field_flags(field);
1671
1672                 if (flags_str) {
1673                         strcat(expr, ".");
1674                         strcat(expr, flags_str);
1675                 }
1676         }
1677 }
1678
1679 static char *expr_str(struct hist_field *field, unsigned int level)
1680 {
1681         char *expr;
1682
1683         if (level > 1)
1684                 return NULL;
1685
1686         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1687         if (!expr)
1688                 return NULL;
1689
1690         if (!field->operands[0]) {
1691                 expr_field_str(field, expr);
1692                 return expr;
1693         }
1694
1695         if (field->operator == FIELD_OP_UNARY_MINUS) {
1696                 char *subexpr;
1697
1698                 strcat(expr, "-(");
1699                 subexpr = expr_str(field->operands[0], ++level);
1700                 if (!subexpr) {
1701                         kfree(expr);
1702                         return NULL;
1703                 }
1704                 strcat(expr, subexpr);
1705                 strcat(expr, ")");
1706
1707                 kfree(subexpr);
1708
1709                 return expr;
1710         }
1711
1712         expr_field_str(field->operands[0], expr);
1713
1714         switch (field->operator) {
1715         case FIELD_OP_MINUS:
1716                 strcat(expr, "-");
1717                 break;
1718         case FIELD_OP_PLUS:
1719                 strcat(expr, "+");
1720                 break;
1721         case FIELD_OP_DIV:
1722                 strcat(expr, "/");
1723                 break;
1724         case FIELD_OP_MULT:
1725                 strcat(expr, "*");
1726                 break;
1727         default:
1728                 kfree(expr);
1729                 return NULL;
1730         }
1731
1732         expr_field_str(field->operands[1], expr);
1733
1734         return expr;
1735 }
1736
1737 /*
1738  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1739  * of the expression tree to be evaluated.
1740  */
1741 static int contains_operator(char *str, char **sep)
1742 {
1743         enum field_op_id field_op = FIELD_OP_NONE;
1744         char *minus_op, *plus_op, *div_op, *mult_op;
1745
1746
1747         /*
1748          * Report the last occurrence of the operators first, so that the
1749          * expression is evaluated left to right. This is important since
1750          * subtraction and division are not associative.
1751          *
1752          *      e.g
1753          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1754          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1755          */
1756
1757         /*
1758          * First, find lower precedence addition and subtraction
1759          * since the expression will be evaluated recursively.
1760          */
1761         minus_op = strrchr(str, '-');
1762         if (minus_op) {
1763                 /*
1764                  * Unary minus is not supported in sub-expressions. If
1765                  * present, it is always the next root operator.
1766                  */
1767                 if (minus_op == str) {
1768                         field_op = FIELD_OP_UNARY_MINUS;
1769                         goto out;
1770                 }
1771
1772                 field_op = FIELD_OP_MINUS;
1773         }
1774
1775         plus_op = strrchr(str, '+');
1776         if (plus_op || minus_op) {
1777                 /*
1778                  * For operators of the same precedence use to rightmost as the
1779                  * root, so that the expression is evaluated left to right.
1780                  */
1781                 if (plus_op > minus_op)
1782                         field_op = FIELD_OP_PLUS;
1783                 goto out;
1784         }
1785
1786         /*
1787          * Multiplication and division have higher precedence than addition and
1788          * subtraction.
1789          */
1790         div_op = strrchr(str, '/');
1791         if (div_op)
1792                 field_op = FIELD_OP_DIV;
1793
1794         mult_op = strrchr(str, '*');
1795         /*
1796          * For operators of the same precedence use to rightmost as the
1797          * root, so that the expression is evaluated left to right.
1798          */
1799         if (mult_op > div_op)
1800                 field_op = FIELD_OP_MULT;
1801
1802 out:
1803         if (sep) {
1804                 switch (field_op) {
1805                 case FIELD_OP_UNARY_MINUS:
1806                 case FIELD_OP_MINUS:
1807                         *sep = minus_op;
1808                         break;
1809                 case FIELD_OP_PLUS:
1810                         *sep = plus_op;
1811                         break;
1812                 case FIELD_OP_DIV:
1813                         *sep = div_op;
1814                         break;
1815                 case FIELD_OP_MULT:
1816                         *sep = mult_op;
1817                         break;
1818                 case FIELD_OP_NONE:
1819                 default:
1820                         *sep = NULL;
1821                         break;
1822                 }
1823         }
1824
1825         return field_op;
1826 }
1827
1828 static void get_hist_field(struct hist_field *hist_field)
1829 {
1830         hist_field->ref++;
1831 }
1832
1833 static void __destroy_hist_field(struct hist_field *hist_field)
1834 {
1835         if (--hist_field->ref > 1)
1836                 return;
1837
1838         kfree(hist_field->var.name);
1839         kfree(hist_field->name);
1840
1841         /* Can likely be a const */
1842         kfree_const(hist_field->type);
1843
1844         kfree(hist_field->system);
1845         kfree(hist_field->event_name);
1846
1847         kfree(hist_field);
1848 }
1849
1850 static void destroy_hist_field(struct hist_field *hist_field,
1851                                unsigned int level)
1852 {
1853         unsigned int i;
1854
1855         if (level > 3)
1856                 return;
1857
1858         if (!hist_field)
1859                 return;
1860
1861         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1862                 return; /* var refs will be destroyed separately */
1863
1864         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1865                 destroy_hist_field(hist_field->operands[i], level + 1);
1866
1867         __destroy_hist_field(hist_field);
1868 }
1869
1870 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1871                                             struct ftrace_event_field *field,
1872                                             unsigned long flags,
1873                                             char *var_name)
1874 {
1875         struct hist_field *hist_field;
1876
1877         if (field && is_function_field(field))
1878                 return NULL;
1879
1880         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1881         if (!hist_field)
1882                 return NULL;
1883
1884         hist_field->ref = 1;
1885
1886         hist_field->hist_data = hist_data;
1887
1888         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1889                 goto out; /* caller will populate */
1890
1891         if (flags & HIST_FIELD_FL_VAR_REF) {
1892                 hist_field->fn = hist_field_var_ref;
1893                 goto out;
1894         }
1895
1896         if (flags & HIST_FIELD_FL_HITCOUNT) {
1897                 hist_field->fn = hist_field_counter;
1898                 hist_field->size = sizeof(u64);
1899                 hist_field->type = "u64";
1900                 goto out;
1901         }
1902
1903         if (flags & HIST_FIELD_FL_CONST) {
1904                 hist_field->fn = hist_field_const;
1905                 hist_field->size = sizeof(u64);
1906                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1907                 if (!hist_field->type)
1908                         goto free;
1909                 goto out;
1910         }
1911
1912         if (flags & HIST_FIELD_FL_STACKTRACE) {
1913                 hist_field->fn = hist_field_none;
1914                 goto out;
1915         }
1916
1917         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1918                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1919                 hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1920                         hist_field_bucket;
1921                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1922                 hist_field->size = hist_field->operands[0]->size;
1923                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1924                 if (!hist_field->type)
1925                         goto free;
1926                 goto out;
1927         }
1928
1929         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1930                 hist_field->fn = hist_field_timestamp;
1931                 hist_field->size = sizeof(u64);
1932                 hist_field->type = "u64";
1933                 goto out;
1934         }
1935
1936         if (flags & HIST_FIELD_FL_CPU) {
1937                 hist_field->fn = hist_field_cpu;
1938                 hist_field->size = sizeof(int);
1939                 hist_field->type = "unsigned int";
1940                 goto out;
1941         }
1942
1943         if (WARN_ON_ONCE(!field))
1944                 goto out;
1945
1946         /* Pointers to strings are just pointers and dangerous to dereference */
1947         if (is_string_field(field) &&
1948             (field->filter_type != FILTER_PTR_STRING)) {
1949                 flags |= HIST_FIELD_FL_STRING;
1950
1951                 hist_field->size = MAX_FILTER_STR_VAL;
1952                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1953                 if (!hist_field->type)
1954                         goto free;
1955
1956                 if (field->filter_type == FILTER_STATIC_STRING) {
1957                         hist_field->fn = hist_field_string;
1958                         hist_field->size = field->size;
1959                 } else if (field->filter_type == FILTER_DYN_STRING)
1960                         hist_field->fn = hist_field_dynstring;
1961                 else
1962                         hist_field->fn = hist_field_pstring;
1963         } else {
1964                 hist_field->size = field->size;
1965                 hist_field->is_signed = field->is_signed;
1966                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1967                 if (!hist_field->type)
1968                         goto free;
1969
1970                 hist_field->fn = select_value_fn(field->size,
1971                                                  field->is_signed);
1972                 if (!hist_field->fn) {
1973                         destroy_hist_field(hist_field, 0);
1974                         return NULL;
1975                 }
1976         }
1977  out:
1978         hist_field->field = field;
1979         hist_field->flags = flags;
1980
1981         if (var_name) {
1982                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1983                 if (!hist_field->var.name)
1984                         goto free;
1985         }
1986
1987         return hist_field;
1988  free:
1989         destroy_hist_field(hist_field, 0);
1990         return NULL;
1991 }
1992
1993 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1994 {
1995         unsigned int i;
1996
1997         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1998                 if (hist_data->fields[i]) {
1999                         destroy_hist_field(hist_data->fields[i], 0);
2000                         hist_data->fields[i] = NULL;
2001                 }
2002         }
2003
2004         for (i = 0; i < hist_data->n_var_refs; i++) {
2005                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2006                 __destroy_hist_field(hist_data->var_refs[i]);
2007                 hist_data->var_refs[i] = NULL;
2008         }
2009 }
2010
2011 static int init_var_ref(struct hist_field *ref_field,
2012                         struct hist_field *var_field,
2013                         char *system, char *event_name)
2014 {
2015         int err = 0;
2016
2017         ref_field->var.idx = var_field->var.idx;
2018         ref_field->var.hist_data = var_field->hist_data;
2019         ref_field->size = var_field->size;
2020         ref_field->is_signed = var_field->is_signed;
2021         ref_field->flags |= var_field->flags &
2022                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2023
2024         if (system) {
2025                 ref_field->system = kstrdup(system, GFP_KERNEL);
2026                 if (!ref_field->system)
2027                         return -ENOMEM;
2028         }
2029
2030         if (event_name) {
2031                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2032                 if (!ref_field->event_name) {
2033                         err = -ENOMEM;
2034                         goto free;
2035                 }
2036         }
2037
2038         if (var_field->var.name) {
2039                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2040                 if (!ref_field->name) {
2041                         err = -ENOMEM;
2042                         goto free;
2043                 }
2044         } else if (var_field->name) {
2045                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2046                 if (!ref_field->name) {
2047                         err = -ENOMEM;
2048                         goto free;
2049                 }
2050         }
2051
2052         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2053         if (!ref_field->type) {
2054                 err = -ENOMEM;
2055                 goto free;
2056         }
2057  out:
2058         return err;
2059  free:
2060         kfree(ref_field->system);
2061         kfree(ref_field->event_name);
2062         kfree(ref_field->name);
2063
2064         goto out;
2065 }
2066
2067 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2068                             struct hist_field *var_field)
2069 {
2070         struct hist_field *ref_field;
2071         int i;
2072
2073         for (i = 0; i < hist_data->n_var_refs; i++) {
2074                 ref_field = hist_data->var_refs[i];
2075                 if (ref_field->var.idx == var_field->var.idx &&
2076                     ref_field->var.hist_data == var_field->hist_data)
2077                         return i;
2078         }
2079
2080         return -ENOENT;
2081 }
2082
2083 /**
2084  * create_var_ref - Create a variable reference and attach it to trigger
2085  * @hist_data: The trigger that will be referencing the variable
2086  * @var_field: The VAR field to create a reference to
2087  * @system: The optional system string
2088  * @event_name: The optional event_name string
2089  *
2090  * Given a variable hist_field, create a VAR_REF hist_field that
2091  * represents a reference to it.
2092  *
2093  * This function also adds the reference to the trigger that
2094  * now references the variable.
2095  *
2096  * Return: The VAR_REF field if successful, NULL if not
2097  */
2098 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2099                                          struct hist_field *var_field,
2100                                          char *system, char *event_name)
2101 {
2102         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2103         struct hist_field *ref_field;
2104         int i;
2105
2106         /* Check if the variable already exists */
2107         for (i = 0; i < hist_data->n_var_refs; i++) {
2108                 ref_field = hist_data->var_refs[i];
2109                 if (ref_field->var.idx == var_field->var.idx &&
2110                     ref_field->var.hist_data == var_field->hist_data) {
2111                         get_hist_field(ref_field);
2112                         return ref_field;
2113                 }
2114         }
2115
2116         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2117         if (ref_field) {
2118                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2119                         destroy_hist_field(ref_field, 0);
2120                         return NULL;
2121                 }
2122
2123                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2124                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2125         }
2126
2127         return ref_field;
2128 }
2129
2130 static bool is_var_ref(char *var_name)
2131 {
2132         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2133                 return false;
2134
2135         return true;
2136 }
2137
2138 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2139                                  char *var_name)
2140 {
2141         char *name, *field;
2142         unsigned int i;
2143
2144         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2145                 name = hist_data->attrs->var_defs.name[i];
2146
2147                 if (strcmp(var_name, name) == 0) {
2148                         field = hist_data->attrs->var_defs.expr[i];
2149                         if (contains_operator(field, NULL) || is_var_ref(field))
2150                                 continue;
2151                         return field;
2152                 }
2153         }
2154
2155         return NULL;
2156 }
2157
2158 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2159                                  char *system, char *event_name,
2160                                  char *var_name)
2161 {
2162         struct trace_event_call *call;
2163
2164         if (system && event_name) {
2165                 call = hist_data->event_file->event_call;
2166
2167                 if (strcmp(system, call->class->system) != 0)
2168                         return NULL;
2169
2170                 if (strcmp(event_name, trace_event_name(call)) != 0)
2171                         return NULL;
2172         }
2173
2174         if (!!system != !!event_name)
2175                 return NULL;
2176
2177         if (!is_var_ref(var_name))
2178                 return NULL;
2179
2180         var_name++;
2181
2182         return field_name_from_var(hist_data, var_name);
2183 }
2184
2185 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2186                                         char *system, char *event_name,
2187                                         char *var_name)
2188 {
2189         struct hist_field *var_field = NULL, *ref_field = NULL;
2190         struct trace_array *tr = hist_data->event_file->tr;
2191
2192         if (!is_var_ref(var_name))
2193                 return NULL;
2194
2195         var_name++;
2196
2197         var_field = find_event_var(hist_data, system, event_name, var_name);
2198         if (var_field)
2199                 ref_field = create_var_ref(hist_data, var_field,
2200                                            system, event_name);
2201
2202         if (!ref_field)
2203                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2204
2205         return ref_field;
2206 }
2207
2208 static struct ftrace_event_field *
2209 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2210             char *field_str, unsigned long *flags, unsigned long *buckets)
2211 {
2212         struct ftrace_event_field *field = NULL;
2213         char *field_name, *modifier, *str;
2214         struct trace_array *tr = file->tr;
2215
2216         modifier = str = kstrdup(field_str, GFP_KERNEL);
2217         if (!modifier)
2218                 return ERR_PTR(-ENOMEM);
2219
2220         field_name = strsep(&modifier, ".");
2221         if (modifier) {
2222                 if (strcmp(modifier, "hex") == 0)
2223                         *flags |= HIST_FIELD_FL_HEX;
2224                 else if (strcmp(modifier, "sym") == 0)
2225                         *flags |= HIST_FIELD_FL_SYM;
2226                 /*
2227                  * 'sym-offset' occurrences in the trigger string are modified
2228                  * to 'symXoffset' to simplify arithmetic expression parsing.
2229                  */
2230                 else if (strcmp(modifier, "symXoffset") == 0)
2231                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2232                 else if ((strcmp(modifier, "execname") == 0) &&
2233                          (strcmp(field_name, "common_pid") == 0))
2234                         *flags |= HIST_FIELD_FL_EXECNAME;
2235                 else if (strcmp(modifier, "syscall") == 0)
2236                         *flags |= HIST_FIELD_FL_SYSCALL;
2237                 else if (strcmp(modifier, "log2") == 0)
2238                         *flags |= HIST_FIELD_FL_LOG2;
2239                 else if (strcmp(modifier, "usecs") == 0)
2240                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2241                 else if (strncmp(modifier, "bucket", 6) == 0) {
2242                         int ret;
2243
2244                         modifier += 6;
2245
2246                         if (*modifier == 's')
2247                                 modifier++;
2248                         if (*modifier != '=')
2249                                 goto error;
2250                         modifier++;
2251                         ret = kstrtoul(modifier, 0, buckets);
2252                         if (ret || !(*buckets))
2253                                 goto error;
2254                         *flags |= HIST_FIELD_FL_BUCKET;
2255                 } else {
2256  error:
2257                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2258                         field = ERR_PTR(-EINVAL);
2259                         goto out;
2260                 }
2261         }
2262
2263         if (strcmp(field_name, "common_timestamp") == 0) {
2264                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2265                 hist_data->enable_timestamps = true;
2266                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2267                         hist_data->attrs->ts_in_usecs = true;
2268         } else if (strcmp(field_name, "common_cpu") == 0)
2269                 *flags |= HIST_FIELD_FL_CPU;
2270         else {
2271                 field = trace_find_event_field(file->event_call, field_name);
2272                 if (!field || !field->size) {
2273                         /*
2274                          * For backward compatibility, if field_name
2275                          * was "cpu", then we treat this the same as
2276                          * common_cpu.
2277                          */
2278                         if (strcmp(field_name, "cpu") == 0) {
2279                                 *flags |= HIST_FIELD_FL_CPU;
2280                         } else {
2281                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2282                                          errpos(field_name));
2283                                 field = ERR_PTR(-EINVAL);
2284                                 goto out;
2285                         }
2286                 }
2287         }
2288  out:
2289         kfree(str);
2290
2291         return field;
2292 }
2293
2294 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2295                                        struct hist_field *var_ref,
2296                                        char *var_name)
2297 {
2298         struct hist_field *alias = NULL;
2299         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2300
2301         alias = create_hist_field(hist_data, NULL, flags, var_name);
2302         if (!alias)
2303                 return NULL;
2304
2305         alias->fn = var_ref->fn;
2306         alias->operands[0] = var_ref;
2307
2308         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2309                 destroy_hist_field(alias, 0);
2310                 return NULL;
2311         }
2312
2313         alias->var_ref_idx = var_ref->var_ref_idx;
2314
2315         return alias;
2316 }
2317
2318 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2319                                       char *str, char *var_name,
2320                                       unsigned long *flags)
2321 {
2322         struct trace_array *tr = hist_data->event_file->tr;
2323         struct hist_field *field = NULL;
2324         u64 constant;
2325
2326         if (kstrtoull(str, 0, &constant)) {
2327                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2328                 return NULL;
2329         }
2330
2331         *flags |= HIST_FIELD_FL_CONST;
2332         field = create_hist_field(hist_data, NULL, *flags, var_name);
2333         if (!field)
2334                 return NULL;
2335
2336         field->constant = constant;
2337
2338         return field;
2339 }
2340
2341 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2342                                      struct trace_event_file *file, char *str,
2343                                      unsigned long *flags, char *var_name)
2344 {
2345         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2346         struct ftrace_event_field *field = NULL;
2347         struct hist_field *hist_field = NULL;
2348         unsigned long buckets = 0;
2349         int ret = 0;
2350
2351         if (isdigit(str[0])) {
2352                 hist_field = parse_const(hist_data, str, var_name, flags);
2353                 if (!hist_field) {
2354                         ret = -EINVAL;
2355                         goto out;
2356                 }
2357                 return hist_field;
2358         }
2359
2360         s = strchr(str, '.');
2361         if (s) {
2362                 s = strchr(++s, '.');
2363                 if (s) {
2364                         ref_system = strsep(&str, ".");
2365                         if (!str) {
2366                                 ret = -EINVAL;
2367                                 goto out;
2368                         }
2369                         ref_event = strsep(&str, ".");
2370                         if (!str) {
2371                                 ret = -EINVAL;
2372                                 goto out;
2373                         }
2374                         ref_var = str;
2375                 }
2376         }
2377
2378         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2379         if (!s) {
2380                 hist_field = parse_var_ref(hist_data, ref_system,
2381                                            ref_event, ref_var);
2382                 if (hist_field) {
2383                         if (var_name) {
2384                                 hist_field = create_alias(hist_data, hist_field, var_name);
2385                                 if (!hist_field) {
2386                                         ret = -ENOMEM;
2387                                         goto out;
2388                                 }
2389                         }
2390                         return hist_field;
2391                 }
2392         } else
2393                 str = s;
2394
2395         field = parse_field(hist_data, file, str, flags, &buckets);
2396         if (IS_ERR(field)) {
2397                 ret = PTR_ERR(field);
2398                 goto out;
2399         }
2400
2401         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2402         if (!hist_field) {
2403                 ret = -ENOMEM;
2404                 goto out;
2405         }
2406         hist_field->buckets = buckets;
2407
2408         return hist_field;
2409  out:
2410         return ERR_PTR(ret);
2411 }
2412
2413 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2414                                      struct trace_event_file *file,
2415                                      char *str, unsigned long flags,
2416                                      char *var_name, unsigned int *n_subexprs);
2417
2418 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2419                                       struct trace_event_file *file,
2420                                       char *str, unsigned long flags,
2421                                       char *var_name, unsigned int *n_subexprs)
2422 {
2423         struct hist_field *operand1, *expr = NULL;
2424         unsigned long operand_flags;
2425         int ret = 0;
2426         char *s;
2427
2428         /* Unary minus operator, increment n_subexprs */
2429         ++*n_subexprs;
2430
2431         /* we support only -(xxx) i.e. explicit parens required */
2432
2433         if (*n_subexprs > 3) {
2434                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2435                 ret = -EINVAL;
2436                 goto free;
2437         }
2438
2439         str++; /* skip leading '-' */
2440
2441         s = strchr(str, '(');
2442         if (s)
2443                 str++;
2444         else {
2445                 ret = -EINVAL;
2446                 goto free;
2447         }
2448
2449         s = strrchr(str, ')');
2450         if (s) {
2451                  /* unary minus not supported in sub-expressions */
2452                 if (*(s+1) != '\0') {
2453                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2454                                  errpos(str));
2455                         ret = -EINVAL;
2456                         goto free;
2457                 }
2458                 *s = '\0';
2459         }
2460         else {
2461                 ret = -EINVAL; /* no closing ')' */
2462                 goto free;
2463         }
2464
2465         flags |= HIST_FIELD_FL_EXPR;
2466         expr = create_hist_field(hist_data, NULL, flags, var_name);
2467         if (!expr) {
2468                 ret = -ENOMEM;
2469                 goto free;
2470         }
2471
2472         operand_flags = 0;
2473         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2474         if (IS_ERR(operand1)) {
2475                 ret = PTR_ERR(operand1);
2476                 goto free;
2477         }
2478         if (operand1->flags & HIST_FIELD_FL_STRING) {
2479                 /* String type can not be the operand of unary operator. */
2480                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2481                 destroy_hist_field(operand1, 0);
2482                 ret = -EINVAL;
2483                 goto free;
2484         }
2485
2486         expr->flags |= operand1->flags &
2487                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2488         expr->fn = hist_field_unary_minus;
2489         expr->operands[0] = operand1;
2490         expr->operator = FIELD_OP_UNARY_MINUS;
2491         expr->name = expr_str(expr, 0);
2492         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2493         if (!expr->type) {
2494                 ret = -ENOMEM;
2495                 goto free;
2496         }
2497
2498         return expr;
2499  free:
2500         destroy_hist_field(expr, 0);
2501         return ERR_PTR(ret);
2502 }
2503
2504 /*
2505  * If the operands are var refs, return pointers the
2506  * variable(s) referenced in var1 and var2, else NULL.
2507  */
2508 static int check_expr_operands(struct trace_array *tr,
2509                                struct hist_field *operand1,
2510                                struct hist_field *operand2,
2511                                struct hist_field **var1,
2512                                struct hist_field **var2)
2513 {
2514         unsigned long operand1_flags = operand1->flags;
2515         unsigned long operand2_flags = operand2->flags;
2516
2517         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2518             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2519                 struct hist_field *var;
2520
2521                 var = find_var_field(operand1->var.hist_data, operand1->name);
2522                 if (!var)
2523                         return -EINVAL;
2524                 operand1_flags = var->flags;
2525                 *var1 = var;
2526         }
2527
2528         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2529             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2530                 struct hist_field *var;
2531
2532                 var = find_var_field(operand2->var.hist_data, operand2->name);
2533                 if (!var)
2534                         return -EINVAL;
2535                 operand2_flags = var->flags;
2536                 *var2 = var;
2537         }
2538
2539         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2540             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2541                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2542                 return -EINVAL;
2543         }
2544
2545         return 0;
2546 }
2547
2548 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2549                                      struct trace_event_file *file,
2550                                      char *str, unsigned long flags,
2551                                      char *var_name, unsigned int *n_subexprs)
2552 {
2553         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2554         struct hist_field *var1 = NULL, *var2 = NULL;
2555         unsigned long operand_flags, operand2_flags;
2556         int field_op, ret = -EINVAL;
2557         char *sep, *operand1_str;
2558         hist_field_fn_t op_fn;
2559         bool combine_consts;
2560
2561         if (*n_subexprs > 3) {
2562                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2563                 return ERR_PTR(-EINVAL);
2564         }
2565
2566         field_op = contains_operator(str, &sep);
2567
2568         if (field_op == FIELD_OP_NONE)
2569                 return parse_atom(hist_data, file, str, &flags, var_name);
2570
2571         if (field_op == FIELD_OP_UNARY_MINUS)
2572                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2573
2574         /* Binary operator found, increment n_subexprs */
2575         ++*n_subexprs;
2576
2577         /* Split the expression string at the root operator */
2578         if (!sep)
2579                 return ERR_PTR(-EINVAL);
2580
2581         *sep = '\0';
2582         operand1_str = str;
2583         str = sep+1;
2584
2585         /* Binary operator requires both operands */
2586         if (*operand1_str == '\0' || *str == '\0')
2587                 return ERR_PTR(-EINVAL);
2588
2589         operand_flags = 0;
2590
2591         /* LHS of string is an expression e.g. a+b in a+b+c */
2592         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2593         if (IS_ERR(operand1))
2594                 return ERR_CAST(operand1);
2595
2596         if (operand1->flags & HIST_FIELD_FL_STRING) {
2597                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2598                 ret = -EINVAL;
2599                 goto free_op1;
2600         }
2601
2602         /* RHS of string is another expression e.g. c in a+b+c */
2603         operand_flags = 0;
2604         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2605         if (IS_ERR(operand2)) {
2606                 ret = PTR_ERR(operand2);
2607                 goto free_op1;
2608         }
2609         if (operand2->flags & HIST_FIELD_FL_STRING) {
2610                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2611                 ret = -EINVAL;
2612                 goto free_operands;
2613         }
2614
2615         switch (field_op) {
2616         case FIELD_OP_MINUS:
2617                 op_fn = hist_field_minus;
2618                 break;
2619         case FIELD_OP_PLUS:
2620                 op_fn = hist_field_plus;
2621                 break;
2622         case FIELD_OP_DIV:
2623                 op_fn = hist_field_div;
2624                 break;
2625         case FIELD_OP_MULT:
2626                 op_fn = hist_field_mult;
2627                 break;
2628         default:
2629                 ret = -EINVAL;
2630                 goto free_operands;
2631         }
2632
2633         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2634         if (ret)
2635                 goto free_operands;
2636
2637         operand_flags = var1 ? var1->flags : operand1->flags;
2638         operand2_flags = var2 ? var2->flags : operand2->flags;
2639
2640         /*
2641          * If both operands are constant, the expression can be
2642          * collapsed to a single constant.
2643          */
2644         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2645
2646         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2647
2648         flags |= operand1->flags &
2649                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2650
2651         expr = create_hist_field(hist_data, NULL, flags, var_name);
2652         if (!expr) {
2653                 ret = -ENOMEM;
2654                 goto free_operands;
2655         }
2656
2657         operand1->read_once = true;
2658         operand2->read_once = true;
2659
2660         /* The operands are now owned and free'd by 'expr' */
2661         expr->operands[0] = operand1;
2662         expr->operands[1] = operand2;
2663
2664         if (field_op == FIELD_OP_DIV &&
2665                         operand2_flags & HIST_FIELD_FL_CONST) {
2666                 u64 divisor = var2 ? var2->constant : operand2->constant;
2667
2668                 if (!divisor) {
2669                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2670                         ret = -EDOM;
2671                         goto free_expr;
2672                 }
2673
2674                 /*
2675                  * Copy the divisor here so we don't have to look it up
2676                  * later if this is a var ref
2677                  */
2678                 operand2->constant = divisor;
2679                 op_fn = hist_field_get_div_fn(operand2);
2680         }
2681
2682         if (combine_consts) {
2683                 if (var1)
2684                         expr->operands[0] = var1;
2685                 if (var2)
2686                         expr->operands[1] = var2;
2687
2688                 expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2689
2690                 expr->operands[0] = NULL;
2691                 expr->operands[1] = NULL;
2692
2693                 /*
2694                  * var refs won't be destroyed immediately
2695                  * See: destroy_hist_field()
2696                  */
2697                 destroy_hist_field(operand2, 0);
2698                 destroy_hist_field(operand1, 0);
2699
2700                 expr->name = expr_str(expr, 0);
2701         } else {
2702                 expr->fn = op_fn;
2703
2704                 /* The operand sizes should be the same, so just pick one */
2705                 expr->size = operand1->size;
2706
2707                 expr->operator = field_op;
2708                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2709                 if (!expr->type) {
2710                         ret = -ENOMEM;
2711                         goto free_expr;
2712                 }
2713
2714                 expr->name = expr_str(expr, 0);
2715         }
2716
2717         return expr;
2718
2719 free_operands:
2720         destroy_hist_field(operand2, 0);
2721 free_op1:
2722         destroy_hist_field(operand1, 0);
2723         return ERR_PTR(ret);
2724
2725 free_expr:
2726         destroy_hist_field(expr, 0);
2727         return ERR_PTR(ret);
2728 }
2729
2730 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2731                                  struct trace_event_file *file)
2732 {
2733         struct event_trigger_data *test;
2734
2735         lockdep_assert_held(&event_mutex);
2736
2737         list_for_each_entry(test, &file->triggers, list) {
2738                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2739                         if (test->private_data == hist_data)
2740                                 return test->filter_str;
2741                 }
2742         }
2743
2744         return NULL;
2745 }
2746
2747 static struct event_command trigger_hist_cmd;
2748 static int event_hist_trigger_func(struct event_command *cmd_ops,
2749                                    struct trace_event_file *file,
2750                                    char *glob, char *cmd, char *param);
2751
2752 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2753                             struct hist_trigger_data *hist_data,
2754                             unsigned int n_keys)
2755 {
2756         struct hist_field *target_hist_field, *hist_field;
2757         unsigned int n, i, j;
2758
2759         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2760                 return false;
2761
2762         i = hist_data->n_vals;
2763         j = target_hist_data->n_vals;
2764
2765         for (n = 0; n < n_keys; n++) {
2766                 hist_field = hist_data->fields[i + n];
2767                 target_hist_field = target_hist_data->fields[j + n];
2768
2769                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2770                         return false;
2771                 if (hist_field->size != target_hist_field->size)
2772                         return false;
2773                 if (hist_field->is_signed != target_hist_field->is_signed)
2774                         return false;
2775         }
2776
2777         return true;
2778 }
2779
2780 static struct hist_trigger_data *
2781 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2782                      struct trace_event_file *file)
2783 {
2784         struct hist_trigger_data *hist_data;
2785         struct event_trigger_data *test;
2786         unsigned int n_keys;
2787
2788         lockdep_assert_held(&event_mutex);
2789
2790         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2791
2792         list_for_each_entry(test, &file->triggers, list) {
2793                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2794                         hist_data = test->private_data;
2795
2796                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2797                                 return hist_data;
2798                 }
2799         }
2800
2801         return NULL;
2802 }
2803
2804 static struct trace_event_file *event_file(struct trace_array *tr,
2805                                            char *system, char *event_name)
2806 {
2807         struct trace_event_file *file;
2808
2809         file = __find_event_file(tr, system, event_name);
2810         if (!file)
2811                 return ERR_PTR(-EINVAL);
2812
2813         return file;
2814 }
2815
2816 static struct hist_field *
2817 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2818                          char *system, char *event_name, char *field_name)
2819 {
2820         struct hist_field *event_var;
2821         char *synthetic_name;
2822
2823         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2824         if (!synthetic_name)
2825                 return ERR_PTR(-ENOMEM);
2826
2827         strcpy(synthetic_name, "synthetic_");
2828         strcat(synthetic_name, field_name);
2829
2830         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2831
2832         kfree(synthetic_name);
2833
2834         return event_var;
2835 }
2836
2837 /**
2838  * create_field_var_hist - Automatically create a histogram and var for a field
2839  * @target_hist_data: The target hist trigger
2840  * @subsys_name: Optional subsystem name
2841  * @event_name: Optional event name
2842  * @field_name: The name of the field (and the resulting variable)
2843  *
2844  * Hist trigger actions fetch data from variables, not directly from
2845  * events.  However, for convenience, users are allowed to directly
2846  * specify an event field in an action, which will be automatically
2847  * converted into a variable on their behalf.
2848  *
2849  * If a user specifies a field on an event that isn't the event the
2850  * histogram currently being defined (the target event histogram), the
2851  * only way that can be accomplished is if a new hist trigger is
2852  * created and the field variable defined on that.
2853  *
2854  * This function creates a new histogram compatible with the target
2855  * event (meaning a histogram with the same key as the target
2856  * histogram), and creates a variable for the specified field, but
2857  * with 'synthetic_' prepended to the variable name in order to avoid
2858  * collision with normal field variables.
2859  *
2860  * Return: The variable created for the field.
2861  */
2862 static struct hist_field *
2863 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2864                       char *subsys_name, char *event_name, char *field_name)
2865 {
2866         struct trace_array *tr = target_hist_data->event_file->tr;
2867         struct hist_trigger_data *hist_data;
2868         unsigned int i, n, first = true;
2869         struct field_var_hist *var_hist;
2870         struct trace_event_file *file;
2871         struct hist_field *key_field;
2872         struct hist_field *event_var;
2873         char *saved_filter;
2874         char *cmd;
2875         int ret;
2876
2877         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2878                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2879                 return ERR_PTR(-EINVAL);
2880         }
2881
2882         file = event_file(tr, subsys_name, event_name);
2883
2884         if (IS_ERR(file)) {
2885                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2886                 ret = PTR_ERR(file);
2887                 return ERR_PTR(ret);
2888         }
2889
2890         /*
2891          * Look for a histogram compatible with target.  We'll use the
2892          * found histogram specification to create a new matching
2893          * histogram with our variable on it.  target_hist_data is not
2894          * yet a registered histogram so we can't use that.
2895          */
2896         hist_data = find_compatible_hist(target_hist_data, file);
2897         if (!hist_data) {
2898                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2899                 return ERR_PTR(-EINVAL);
2900         }
2901
2902         /* See if a synthetic field variable has already been created */
2903         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2904                                              event_name, field_name);
2905         if (!IS_ERR_OR_NULL(event_var))
2906                 return event_var;
2907
2908         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2909         if (!var_hist)
2910                 return ERR_PTR(-ENOMEM);
2911
2912         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2913         if (!cmd) {
2914                 kfree(var_hist);
2915                 return ERR_PTR(-ENOMEM);
2916         }
2917
2918         /* Use the same keys as the compatible histogram */
2919         strcat(cmd, "keys=");
2920
2921         for_each_hist_key_field(i, hist_data) {
2922                 key_field = hist_data->fields[i];
2923                 if (!first)
2924                         strcat(cmd, ",");
2925                 strcat(cmd, key_field->field->name);
2926                 first = false;
2927         }
2928
2929         /* Create the synthetic field variable specification */
2930         strcat(cmd, ":synthetic_");
2931         strcat(cmd, field_name);
2932         strcat(cmd, "=");
2933         strcat(cmd, field_name);
2934
2935         /* Use the same filter as the compatible histogram */
2936         saved_filter = find_trigger_filter(hist_data, file);
2937         if (saved_filter) {
2938                 strcat(cmd, " if ");
2939                 strcat(cmd, saved_filter);
2940         }
2941
2942         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2943         if (!var_hist->cmd) {
2944                 kfree(cmd);
2945                 kfree(var_hist);
2946                 return ERR_PTR(-ENOMEM);
2947         }
2948
2949         /* Save the compatible histogram information */
2950         var_hist->hist_data = hist_data;
2951
2952         /* Create the new histogram with our variable */
2953         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2954                                       "", "hist", cmd);
2955         if (ret) {
2956                 kfree(cmd);
2957                 kfree(var_hist->cmd);
2958                 kfree(var_hist);
2959                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2960                 return ERR_PTR(ret);
2961         }
2962
2963         kfree(cmd);
2964
2965         /* If we can't find the variable, something went wrong */
2966         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2967                                              event_name, field_name);
2968         if (IS_ERR_OR_NULL(event_var)) {
2969                 kfree(var_hist->cmd);
2970                 kfree(var_hist);
2971                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2972                 return ERR_PTR(-EINVAL);
2973         }
2974
2975         n = target_hist_data->n_field_var_hists;
2976         target_hist_data->field_var_hists[n] = var_hist;
2977         target_hist_data->n_field_var_hists++;
2978
2979         return event_var;
2980 }
2981
2982 static struct hist_field *
2983 find_target_event_var(struct hist_trigger_data *hist_data,
2984                       char *subsys_name, char *event_name, char *var_name)
2985 {
2986         struct trace_event_file *file = hist_data->event_file;
2987         struct hist_field *hist_field = NULL;
2988
2989         if (subsys_name) {
2990                 struct trace_event_call *call;
2991
2992                 if (!event_name)
2993                         return NULL;
2994
2995                 call = file->event_call;
2996
2997                 if (strcmp(subsys_name, call->class->system) != 0)
2998                         return NULL;
2999
3000                 if (strcmp(event_name, trace_event_name(call)) != 0)
3001                         return NULL;
3002         }
3003
3004         hist_field = find_var_field(hist_data, var_name);
3005
3006         return hist_field;
3007 }
3008
3009 static inline void __update_field_vars(struct tracing_map_elt *elt,
3010                                        struct trace_buffer *buffer,
3011                                        struct ring_buffer_event *rbe,
3012                                        void *rec,
3013                                        struct field_var **field_vars,
3014                                        unsigned int n_field_vars,
3015                                        unsigned int field_var_str_start)
3016 {
3017         struct hist_elt_data *elt_data = elt->private_data;
3018         unsigned int i, j, var_idx;
3019         u64 var_val;
3020
3021         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3022                 struct field_var *field_var = field_vars[i];
3023                 struct hist_field *var = field_var->var;
3024                 struct hist_field *val = field_var->val;
3025
3026                 var_val = val->fn(val, elt, buffer, rbe, rec);
3027                 var_idx = var->var.idx;
3028
3029                 if (val->flags & HIST_FIELD_FL_STRING) {
3030                         char *str = elt_data->field_var_str[j++];
3031                         char *val_str = (char *)(uintptr_t)var_val;
3032                         unsigned int size;
3033
3034                         size = min(val->size, STR_VAR_LEN_MAX);
3035                         strscpy(str, val_str, size);
3036                         var_val = (u64)(uintptr_t)str;
3037                 }
3038                 tracing_map_set_var(elt, var_idx, var_val);
3039         }
3040 }
3041
3042 static void update_field_vars(struct hist_trigger_data *hist_data,
3043                               struct tracing_map_elt *elt,
3044                               struct trace_buffer *buffer,
3045                               struct ring_buffer_event *rbe,
3046                               void *rec)
3047 {
3048         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3049                             hist_data->n_field_vars, 0);
3050 }
3051
3052 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3053                                  struct tracing_map_elt *elt,
3054                                  struct trace_buffer *buffer,  void *rec,
3055                                  struct ring_buffer_event *rbe, void *key,
3056                                  struct action_data *data, u64 *var_ref_vals)
3057 {
3058         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3059                             hist_data->n_save_vars, hist_data->n_field_var_str);
3060 }
3061
3062 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3063                                      struct trace_event_file *file,
3064                                      char *name, int size, const char *type)
3065 {
3066         struct hist_field *var;
3067         int idx;
3068
3069         if (find_var(hist_data, file, name) && !hist_data->remove) {
3070                 var = ERR_PTR(-EINVAL);
3071                 goto out;
3072         }
3073
3074         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3075         if (!var) {
3076                 var = ERR_PTR(-ENOMEM);
3077                 goto out;
3078         }
3079
3080         idx = tracing_map_add_var(hist_data->map);
3081         if (idx < 0) {
3082                 kfree(var);
3083                 var = ERR_PTR(-EINVAL);
3084                 goto out;
3085         }
3086
3087         var->ref = 1;
3088         var->flags = HIST_FIELD_FL_VAR;
3089         var->var.idx = idx;
3090         var->var.hist_data = var->hist_data = hist_data;
3091         var->size = size;
3092         var->var.name = kstrdup(name, GFP_KERNEL);
3093         var->type = kstrdup_const(type, GFP_KERNEL);
3094         if (!var->var.name || !var->type) {
3095                 kfree_const(var->type);
3096                 kfree(var->var.name);
3097                 kfree(var);
3098                 var = ERR_PTR(-ENOMEM);
3099         }
3100  out:
3101         return var;
3102 }
3103
3104 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3105                                           struct trace_event_file *file,
3106                                           char *field_name)
3107 {
3108         struct hist_field *val = NULL, *var = NULL;
3109         unsigned long flags = HIST_FIELD_FL_VAR;
3110         struct trace_array *tr = file->tr;
3111         struct field_var *field_var;
3112         int ret = 0;
3113
3114         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3115                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3116                 ret = -EINVAL;
3117                 goto err;
3118         }
3119
3120         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3121         if (IS_ERR(val)) {
3122                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3123                 ret = PTR_ERR(val);
3124                 goto err;
3125         }
3126
3127         var = create_var(hist_data, file, field_name, val->size, val->type);
3128         if (IS_ERR(var)) {
3129                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3130                 kfree(val);
3131                 ret = PTR_ERR(var);
3132                 goto err;
3133         }
3134
3135         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3136         if (!field_var) {
3137                 kfree(val);
3138                 kfree(var);
3139                 ret =  -ENOMEM;
3140                 goto err;
3141         }
3142
3143         field_var->var = var;
3144         field_var->val = val;
3145  out:
3146         return field_var;
3147  err:
3148         field_var = ERR_PTR(ret);
3149         goto out;
3150 }
3151
3152 /**
3153  * create_target_field_var - Automatically create a variable for a field
3154  * @target_hist_data: The target hist trigger
3155  * @subsys_name: Optional subsystem name
3156  * @event_name: Optional event name
3157  * @var_name: The name of the field (and the resulting variable)
3158  *
3159  * Hist trigger actions fetch data from variables, not directly from
3160  * events.  However, for convenience, users are allowed to directly
3161  * specify an event field in an action, which will be automatically
3162  * converted into a variable on their behalf.
3163
3164  * This function creates a field variable with the name var_name on
3165  * the hist trigger currently being defined on the target event.  If
3166  * subsys_name and event_name are specified, this function simply
3167  * verifies that they do in fact match the target event subsystem and
3168  * event name.
3169  *
3170  * Return: The variable created for the field.
3171  */
3172 static struct field_var *
3173 create_target_field_var(struct hist_trigger_data *target_hist_data,
3174                         char *subsys_name, char *event_name, char *var_name)
3175 {
3176         struct trace_event_file *file = target_hist_data->event_file;
3177
3178         if (subsys_name) {
3179                 struct trace_event_call *call;
3180
3181                 if (!event_name)
3182                         return NULL;
3183
3184                 call = file->event_call;
3185
3186                 if (strcmp(subsys_name, call->class->system) != 0)
3187                         return NULL;
3188
3189                 if (strcmp(event_name, trace_event_name(call)) != 0)
3190                         return NULL;
3191         }
3192
3193         return create_field_var(target_hist_data, file, var_name);
3194 }
3195
3196 static bool check_track_val_max(u64 track_val, u64 var_val)
3197 {
3198         if (var_val <= track_val)
3199                 return false;
3200
3201         return true;
3202 }
3203
3204 static bool check_track_val_changed(u64 track_val, u64 var_val)
3205 {
3206         if (var_val == track_val)
3207                 return false;
3208
3209         return true;
3210 }
3211
3212 static u64 get_track_val(struct hist_trigger_data *hist_data,
3213                          struct tracing_map_elt *elt,
3214                          struct action_data *data)
3215 {
3216         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3217         u64 track_val;
3218
3219         track_val = tracing_map_read_var(elt, track_var_idx);
3220
3221         return track_val;
3222 }
3223
3224 static void save_track_val(struct hist_trigger_data *hist_data,
3225                            struct tracing_map_elt *elt,
3226                            struct action_data *data, u64 var_val)
3227 {
3228         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3229
3230         tracing_map_set_var(elt, track_var_idx, var_val);
3231 }
3232
3233 static void save_track_data(struct hist_trigger_data *hist_data,
3234                             struct tracing_map_elt *elt,
3235                             struct trace_buffer *buffer, void *rec,
3236                             struct ring_buffer_event *rbe, void *key,
3237                             struct action_data *data, u64 *var_ref_vals)
3238 {
3239         if (data->track_data.save_data)
3240                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3241                                            key, data, var_ref_vals);
3242 }
3243
3244 static bool check_track_val(struct tracing_map_elt *elt,
3245                             struct action_data *data,
3246                             u64 var_val)
3247 {
3248         struct hist_trigger_data *hist_data;
3249         u64 track_val;
3250
3251         hist_data = data->track_data.track_var->hist_data;
3252         track_val = get_track_val(hist_data, elt, data);
3253
3254         return data->track_data.check_val(track_val, var_val);
3255 }
3256
3257 #ifdef CONFIG_TRACER_SNAPSHOT
3258 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3259 {
3260         /* called with tr->max_lock held */
3261         struct track_data *track_data = tr->cond_snapshot->cond_data;
3262         struct hist_elt_data *elt_data, *track_elt_data;
3263         struct snapshot_context *context = cond_data;
3264         struct action_data *action;
3265         u64 track_val;
3266
3267         if (!track_data)
3268                 return false;
3269
3270         action = track_data->action_data;
3271
3272         track_val = get_track_val(track_data->hist_data, context->elt,
3273                                   track_data->action_data);
3274
3275         if (!action->track_data.check_val(track_data->track_val, track_val))
3276                 return false;
3277
3278         track_data->track_val = track_val;
3279         memcpy(track_data->key, context->key, track_data->key_len);
3280
3281         elt_data = context->elt->private_data;
3282         track_elt_data = track_data->elt.private_data;
3283         if (elt_data->comm)
3284                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3285
3286         track_data->updated = true;
3287
3288         return true;
3289 }
3290
3291 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3292                                      struct tracing_map_elt *elt,
3293                                      struct trace_buffer *buffer, void *rec,
3294                                      struct ring_buffer_event *rbe, void *key,
3295                                      struct action_data *data,
3296                                      u64 *var_ref_vals)
3297 {
3298         struct trace_event_file *file = hist_data->event_file;
3299         struct snapshot_context context;
3300
3301         context.elt = elt;
3302         context.key = key;
3303
3304         tracing_snapshot_cond(file->tr, &context);
3305 }
3306
3307 static void hist_trigger_print_key(struct seq_file *m,
3308                                    struct hist_trigger_data *hist_data,
3309                                    void *key,
3310                                    struct tracing_map_elt *elt);
3311
3312 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3313 {
3314         unsigned int i;
3315
3316         if (!hist_data->n_actions)
3317                 return NULL;
3318
3319         for (i = 0; i < hist_data->n_actions; i++) {
3320                 struct action_data *data = hist_data->actions[i];
3321
3322                 if (data->action == ACTION_SNAPSHOT)
3323                         return data;
3324         }
3325
3326         return NULL;
3327 }
3328
3329 static void track_data_snapshot_print(struct seq_file *m,
3330                                       struct hist_trigger_data *hist_data)
3331 {
3332         struct trace_event_file *file = hist_data->event_file;
3333         struct track_data *track_data;
3334         struct action_data *action;
3335
3336         track_data = tracing_cond_snapshot_data(file->tr);
3337         if (!track_data)
3338                 return;
3339
3340         if (!track_data->updated)
3341                 return;
3342
3343         action = snapshot_action(hist_data);
3344         if (!action)
3345                 return;
3346
3347         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3348         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3349                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3350                    action->track_data.var_str, track_data->track_val);
3351
3352         seq_puts(m, "\ttriggered by event with key: ");
3353         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3354         seq_putc(m, '\n');
3355 }
3356 #else
3357 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3358 {
3359         return false;
3360 }
3361 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3362                                      struct tracing_map_elt *elt,
3363                                      struct trace_buffer *buffer, void *rec,
3364                                      struct ring_buffer_event *rbe, void *key,
3365                                      struct action_data *data,
3366                                      u64 *var_ref_vals) {}
3367 static void track_data_snapshot_print(struct seq_file *m,
3368                                       struct hist_trigger_data *hist_data) {}
3369 #endif /* CONFIG_TRACER_SNAPSHOT */
3370
3371 static void track_data_print(struct seq_file *m,
3372                              struct hist_trigger_data *hist_data,
3373                              struct tracing_map_elt *elt,
3374                              struct action_data *data)
3375 {
3376         u64 track_val = get_track_val(hist_data, elt, data);
3377         unsigned int i, save_var_idx;
3378
3379         if (data->handler == HANDLER_ONMAX)
3380                 seq_printf(m, "\n\tmax: %10llu", track_val);
3381         else if (data->handler == HANDLER_ONCHANGE)
3382                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3383
3384         if (data->action == ACTION_SNAPSHOT)
3385                 return;
3386
3387         for (i = 0; i < hist_data->n_save_vars; i++) {
3388                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3389                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3390                 u64 val;
3391
3392                 save_var_idx = save_var->var.idx;
3393
3394                 val = tracing_map_read_var(elt, save_var_idx);
3395
3396                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3397                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3398                                    (char *)(uintptr_t)(val));
3399                 } else
3400                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3401         }
3402 }
3403
3404 static void ontrack_action(struct hist_trigger_data *hist_data,
3405                            struct tracing_map_elt *elt,
3406                            struct trace_buffer *buffer, void *rec,
3407                            struct ring_buffer_event *rbe, void *key,
3408                            struct action_data *data, u64 *var_ref_vals)
3409 {
3410         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3411
3412         if (check_track_val(elt, data, var_val)) {
3413                 save_track_val(hist_data, elt, data, var_val);
3414                 save_track_data(hist_data, elt, buffer, rec, rbe,
3415                                 key, data, var_ref_vals);
3416         }
3417 }
3418
3419 static void action_data_destroy(struct action_data *data)
3420 {
3421         unsigned int i;
3422
3423         lockdep_assert_held(&event_mutex);
3424
3425         kfree(data->action_name);
3426
3427         for (i = 0; i < data->n_params; i++)
3428                 kfree(data->params[i]);
3429
3430         if (data->synth_event)
3431                 data->synth_event->ref--;
3432
3433         kfree(data->synth_event_name);
3434
3435         kfree(data);
3436 }
3437
3438 static void track_data_destroy(struct hist_trigger_data *hist_data,
3439                                struct action_data *data)
3440 {
3441         struct trace_event_file *file = hist_data->event_file;
3442
3443         destroy_hist_field(data->track_data.track_var, 0);
3444
3445         if (data->action == ACTION_SNAPSHOT) {
3446                 struct track_data *track_data;
3447
3448                 track_data = tracing_cond_snapshot_data(file->tr);
3449                 if (track_data && track_data->hist_data == hist_data) {
3450                         tracing_snapshot_cond_disable(file->tr);
3451                         track_data_free(track_data);
3452                 }
3453         }
3454
3455         kfree(data->track_data.var_str);
3456
3457         action_data_destroy(data);
3458 }
3459
3460 static int action_create(struct hist_trigger_data *hist_data,
3461                          struct action_data *data);
3462
3463 static int track_data_create(struct hist_trigger_data *hist_data,
3464                              struct action_data *data)
3465 {
3466         struct hist_field *var_field, *ref_field, *track_var = NULL;
3467         struct trace_event_file *file = hist_data->event_file;
3468         struct trace_array *tr = file->tr;
3469         char *track_data_var_str;
3470         int ret = 0;
3471
3472         track_data_var_str = data->track_data.var_str;
3473         if (track_data_var_str[0] != '$') {
3474                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3475                 return -EINVAL;
3476         }
3477         track_data_var_str++;
3478
3479         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3480         if (!var_field) {
3481                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3482                 return -EINVAL;
3483         }
3484
3485         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3486         if (!ref_field)
3487                 return -ENOMEM;
3488
3489         data->track_data.var_ref = ref_field;
3490
3491         if (data->handler == HANDLER_ONMAX)
3492                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3493         if (IS_ERR(track_var)) {
3494                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3495                 ret = PTR_ERR(track_var);
3496                 goto out;
3497         }
3498
3499         if (data->handler == HANDLER_ONCHANGE)
3500                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3501         if (IS_ERR(track_var)) {
3502                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3503                 ret = PTR_ERR(track_var);
3504                 goto out;
3505         }
3506         data->track_data.track_var = track_var;
3507
3508         ret = action_create(hist_data, data);
3509  out:
3510         return ret;
3511 }
3512
3513 static int parse_action_params(struct trace_array *tr, char *params,
3514                                struct action_data *data)
3515 {
3516         char *param, *saved_param;
3517         bool first_param = true;
3518         int ret = 0;
3519
3520         while (params) {
3521                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3522                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3523                         goto out;
3524                 }
3525
3526                 param = strsep(&params, ",");
3527                 if (!param) {
3528                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3529                         ret = -EINVAL;
3530                         goto out;
3531                 }
3532
3533                 param = strstrip(param);
3534                 if (strlen(param) < 2) {
3535                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3536                         ret = -EINVAL;
3537                         goto out;
3538                 }
3539
3540                 saved_param = kstrdup(param, GFP_KERNEL);
3541                 if (!saved_param) {
3542                         ret = -ENOMEM;
3543                         goto out;
3544                 }
3545
3546                 if (first_param && data->use_trace_keyword) {
3547                         data->synth_event_name = saved_param;
3548                         first_param = false;
3549                         continue;
3550                 }
3551                 first_param = false;
3552
3553                 data->params[data->n_params++] = saved_param;
3554         }
3555  out:
3556         return ret;
3557 }
3558
3559 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3560                         enum handler_id handler)
3561 {
3562         char *action_name;
3563         int ret = 0;
3564
3565         strsep(&str, ".");
3566         if (!str) {
3567                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3568                 ret = -EINVAL;
3569                 goto out;
3570         }
3571
3572         action_name = strsep(&str, "(");
3573         if (!action_name || !str) {
3574                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3575                 ret = -EINVAL;
3576                 goto out;
3577         }
3578
3579         if (str_has_prefix(action_name, "save")) {
3580                 char *params = strsep(&str, ")");
3581
3582                 if (!params) {
3583                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3584                         ret = -EINVAL;
3585                         goto out;
3586                 }
3587
3588                 ret = parse_action_params(tr, params, data);
3589                 if (ret)
3590                         goto out;
3591
3592                 if (handler == HANDLER_ONMAX)
3593                         data->track_data.check_val = check_track_val_max;
3594                 else if (handler == HANDLER_ONCHANGE)
3595                         data->track_data.check_val = check_track_val_changed;
3596                 else {
3597                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3598                         ret = -EINVAL;
3599                         goto out;
3600                 }
3601
3602                 data->track_data.save_data = save_track_data_vars;
3603                 data->fn = ontrack_action;
3604                 data->action = ACTION_SAVE;
3605         } else if (str_has_prefix(action_name, "snapshot")) {
3606                 char *params = strsep(&str, ")");
3607
3608                 if (!str) {
3609                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3610                         ret = -EINVAL;
3611                         goto out;
3612                 }
3613
3614                 if (handler == HANDLER_ONMAX)
3615                         data->track_data.check_val = check_track_val_max;
3616                 else if (handler == HANDLER_ONCHANGE)
3617                         data->track_data.check_val = check_track_val_changed;
3618                 else {
3619                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3620                         ret = -EINVAL;
3621                         goto out;
3622                 }
3623
3624                 data->track_data.save_data = save_track_data_snapshot;
3625                 data->fn = ontrack_action;
3626                 data->action = ACTION_SNAPSHOT;
3627         } else {
3628                 char *params = strsep(&str, ")");
3629
3630                 if (str_has_prefix(action_name, "trace"))
3631                         data->use_trace_keyword = true;
3632
3633                 if (params) {
3634                         ret = parse_action_params(tr, params, data);
3635                         if (ret)
3636                                 goto out;
3637                 }
3638
3639                 if (handler == HANDLER_ONMAX)
3640                         data->track_data.check_val = check_track_val_max;
3641                 else if (handler == HANDLER_ONCHANGE)
3642                         data->track_data.check_val = check_track_val_changed;
3643
3644                 if (handler != HANDLER_ONMATCH) {
3645                         data->track_data.save_data = action_trace;
3646                         data->fn = ontrack_action;
3647                 } else
3648                         data->fn = action_trace;
3649
3650                 data->action = ACTION_TRACE;
3651         }
3652
3653         data->action_name = kstrdup(action_name, GFP_KERNEL);
3654         if (!data->action_name) {
3655                 ret = -ENOMEM;
3656                 goto out;
3657         }
3658
3659         data->handler = handler;
3660  out:
3661         return ret;
3662 }
3663
3664 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3665                                             char *str, enum handler_id handler)
3666 {
3667         struct action_data *data;
3668         int ret = -EINVAL;
3669         char *var_str;
3670
3671         data = kzalloc(sizeof(*data), GFP_KERNEL);
3672         if (!data)
3673                 return ERR_PTR(-ENOMEM);
3674
3675         var_str = strsep(&str, ")");
3676         if (!var_str || !str) {
3677                 ret = -EINVAL;
3678                 goto free;
3679         }
3680
3681         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3682         if (!data->track_data.var_str) {
3683                 ret = -ENOMEM;
3684                 goto free;
3685         }
3686
3687         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3688         if (ret)
3689                 goto free;
3690  out:
3691         return data;
3692  free:
3693         track_data_destroy(hist_data, data);
3694         data = ERR_PTR(ret);
3695         goto out;
3696 }
3697
3698 static void onmatch_destroy(struct action_data *data)
3699 {
3700         kfree(data->match_data.event);
3701         kfree(data->match_data.event_system);
3702
3703         action_data_destroy(data);
3704 }
3705
3706 static void destroy_field_var(struct field_var *field_var)
3707 {
3708         if (!field_var)
3709                 return;
3710
3711         destroy_hist_field(field_var->var, 0);
3712         destroy_hist_field(field_var->val, 0);
3713
3714         kfree(field_var);
3715 }
3716
3717 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3718 {
3719         unsigned int i;
3720
3721         for (i = 0; i < hist_data->n_field_vars; i++)
3722                 destroy_field_var(hist_data->field_vars[i]);
3723
3724         for (i = 0; i < hist_data->n_save_vars; i++)
3725                 destroy_field_var(hist_data->save_vars[i]);
3726 }
3727
3728 static void save_field_var(struct hist_trigger_data *hist_data,
3729                            struct field_var *field_var)
3730 {
3731         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3732
3733         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3734                 hist_data->n_field_var_str++;
3735 }
3736
3737
3738 static int check_synth_field(struct synth_event *event,
3739                              struct hist_field *hist_field,
3740                              unsigned int field_pos)
3741 {
3742         struct synth_field *field;
3743
3744         if (field_pos >= event->n_fields)
3745                 return -EINVAL;
3746
3747         field = event->fields[field_pos];
3748
3749         /*
3750          * A dynamic string synth field can accept static or
3751          * dynamic. A static string synth field can only accept a
3752          * same-sized static string, which is checked for later.
3753          */
3754         if (strstr(hist_field->type, "char[") && field->is_string
3755             && field->is_dynamic)
3756                 return 0;
3757
3758         if (strcmp(field->type, hist_field->type) != 0) {
3759                 if (field->size != hist_field->size ||
3760                     (!field->is_string && field->is_signed != hist_field->is_signed))
3761                         return -EINVAL;
3762         }
3763
3764         return 0;
3765 }
3766
3767 static struct hist_field *
3768 trace_action_find_var(struct hist_trigger_data *hist_data,
3769                       struct action_data *data,
3770                       char *system, char *event, char *var)
3771 {
3772         struct trace_array *tr = hist_data->event_file->tr;
3773         struct hist_field *hist_field;
3774
3775         var++; /* skip '$' */
3776
3777         hist_field = find_target_event_var(hist_data, system, event, var);
3778         if (!hist_field) {
3779                 if (!system && data->handler == HANDLER_ONMATCH) {
3780                         system = data->match_data.event_system;
3781                         event = data->match_data.event;
3782                 }
3783
3784                 hist_field = find_event_var(hist_data, system, event, var);
3785         }
3786
3787         if (!hist_field)
3788                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3789
3790         return hist_field;
3791 }
3792
3793 static struct hist_field *
3794 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3795                               struct action_data *data, char *system,
3796                               char *event, char *var)
3797 {
3798         struct hist_field *hist_field = NULL;
3799         struct field_var *field_var;
3800
3801         /*
3802          * First try to create a field var on the target event (the
3803          * currently being defined).  This will create a variable for
3804          * unqualified fields on the target event, or if qualified,
3805          * target fields that have qualified names matching the target.
3806          */
3807         field_var = create_target_field_var(hist_data, system, event, var);
3808
3809         if (field_var && !IS_ERR(field_var)) {
3810                 save_field_var(hist_data, field_var);
3811                 hist_field = field_var->var;
3812         } else {
3813                 field_var = NULL;
3814                 /*
3815                  * If no explicit system.event is specified, default to
3816                  * looking for fields on the onmatch(system.event.xxx)
3817                  * event.
3818                  */
3819                 if (!system && data->handler == HANDLER_ONMATCH) {
3820                         system = data->match_data.event_system;
3821                         event = data->match_data.event;
3822                 }
3823
3824                 if (!event)
3825                         goto free;
3826                 /*
3827                  * At this point, we're looking at a field on another
3828                  * event.  Because we can't modify a hist trigger on
3829                  * another event to add a variable for a field, we need
3830                  * to create a new trigger on that event and create the
3831                  * variable at the same time.
3832                  */
3833                 hist_field = create_field_var_hist(hist_data, system, event, var);
3834                 if (IS_ERR(hist_field))
3835                         goto free;
3836         }
3837  out:
3838         return hist_field;
3839  free:
3840         destroy_field_var(field_var);
3841         hist_field = NULL;
3842         goto out;
3843 }
3844
3845 static int trace_action_create(struct hist_trigger_data *hist_data,
3846                                struct action_data *data)
3847 {
3848         struct trace_array *tr = hist_data->event_file->tr;
3849         char *event_name, *param, *system = NULL;
3850         struct hist_field *hist_field, *var_ref;
3851         unsigned int i;
3852         unsigned int field_pos = 0;
3853         struct synth_event *event;
3854         char *synth_event_name;
3855         int var_ref_idx, ret = 0;
3856
3857         lockdep_assert_held(&event_mutex);
3858
3859         if (data->use_trace_keyword)
3860                 synth_event_name = data->synth_event_name;
3861         else
3862                 synth_event_name = data->action_name;
3863
3864         event = find_synth_event(synth_event_name);
3865         if (!event) {
3866                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3867                 return -EINVAL;
3868         }
3869
3870         event->ref++;
3871
3872         for (i = 0; i < data->n_params; i++) {
3873                 char *p;
3874
3875                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3876                 if (!param) {
3877                         ret = -ENOMEM;
3878                         goto err;
3879                 }
3880
3881                 system = strsep(&param, ".");
3882                 if (!param) {
3883                         param = (char *)system;
3884                         system = event_name = NULL;
3885                 } else {
3886                         event_name = strsep(&param, ".");
3887                         if (!param) {
3888                                 kfree(p);
3889                                 ret = -EINVAL;
3890                                 goto err;
3891                         }
3892                 }
3893
3894                 if (param[0] == '$')
3895                         hist_field = trace_action_find_var(hist_data, data,
3896                                                            system, event_name,
3897                                                            param);
3898                 else
3899                         hist_field = trace_action_create_field_var(hist_data,
3900                                                                    data,
3901                                                                    system,
3902                                                                    event_name,
3903                                                                    param);
3904
3905                 if (!hist_field) {
3906                         kfree(p);
3907                         ret = -EINVAL;
3908                         goto err;
3909                 }
3910
3911                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3912                         var_ref = create_var_ref(hist_data, hist_field,
3913                                                  system, event_name);
3914                         if (!var_ref) {
3915                                 kfree(p);
3916                                 ret = -ENOMEM;
3917                                 goto err;
3918                         }
3919
3920                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3921                         if (WARN_ON(var_ref_idx < 0)) {
3922                                 ret = var_ref_idx;
3923                                 goto err;
3924                         }
3925
3926                         data->var_ref_idx[i] = var_ref_idx;
3927
3928                         field_pos++;
3929                         kfree(p);
3930                         continue;
3931                 }
3932
3933                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3934                 kfree(p);
3935                 ret = -EINVAL;
3936                 goto err;
3937         }
3938
3939         if (field_pos != event->n_fields) {
3940                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3941                 ret = -EINVAL;
3942                 goto err;
3943         }
3944
3945         data->synth_event = event;
3946  out:
3947         return ret;
3948  err:
3949         event->ref--;
3950
3951         goto out;
3952 }
3953
3954 static int action_create(struct hist_trigger_data *hist_data,
3955                          struct action_data *data)
3956 {
3957         struct trace_event_file *file = hist_data->event_file;
3958         struct trace_array *tr = file->tr;
3959         struct track_data *track_data;
3960         struct field_var *field_var;
3961         unsigned int i;
3962         char *param;
3963         int ret = 0;
3964
3965         if (data->action == ACTION_TRACE)
3966                 return trace_action_create(hist_data, data);
3967
3968         if (data->action == ACTION_SNAPSHOT) {
3969                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3970                 if (IS_ERR(track_data)) {
3971                         ret = PTR_ERR(track_data);
3972                         goto out;
3973                 }
3974
3975                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3976                                                    cond_snapshot_update);
3977                 if (ret)
3978                         track_data_free(track_data);
3979
3980                 goto out;
3981         }
3982
3983         if (data->action == ACTION_SAVE) {
3984                 if (hist_data->n_save_vars) {
3985                         ret = -EEXIST;
3986                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3987                         goto out;
3988                 }
3989
3990                 for (i = 0; i < data->n_params; i++) {
3991                         param = kstrdup(data->params[i], GFP_KERNEL);
3992                         if (!param) {
3993                                 ret = -ENOMEM;
3994                                 goto out;
3995                         }
3996
3997                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3998                         if (IS_ERR(field_var)) {
3999                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4000                                          errpos(param));
4001                                 ret = PTR_ERR(field_var);
4002                                 kfree(param);
4003                                 goto out;
4004                         }
4005
4006                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4007                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4008                                 hist_data->n_save_var_str++;
4009                         kfree(param);
4010                 }
4011         }
4012  out:
4013         return ret;
4014 }
4015
4016 static int onmatch_create(struct hist_trigger_data *hist_data,
4017                           struct action_data *data)
4018 {
4019         return action_create(hist_data, data);
4020 }
4021
4022 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4023 {
4024         char *match_event, *match_event_system;
4025         struct action_data *data;
4026         int ret = -EINVAL;
4027
4028         data = kzalloc(sizeof(*data), GFP_KERNEL);
4029         if (!data)
4030                 return ERR_PTR(-ENOMEM);
4031
4032         match_event = strsep(&str, ")");
4033         if (!match_event || !str) {
4034                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4035                 goto free;
4036         }
4037
4038         match_event_system = strsep(&match_event, ".");
4039         if (!match_event) {
4040                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4041                 goto free;
4042         }
4043
4044         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4045                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4046                 goto free;
4047         }
4048
4049         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4050         if (!data->match_data.event) {
4051                 ret = -ENOMEM;
4052                 goto free;
4053         }
4054
4055         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4056         if (!data->match_data.event_system) {
4057                 ret = -ENOMEM;
4058                 goto free;
4059         }
4060
4061         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4062         if (ret)
4063                 goto free;
4064  out:
4065         return data;
4066  free:
4067         onmatch_destroy(data);
4068         data = ERR_PTR(ret);
4069         goto out;
4070 }
4071
4072 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4073 {
4074         hist_data->fields[HITCOUNT_IDX] =
4075                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4076         if (!hist_data->fields[HITCOUNT_IDX])
4077                 return -ENOMEM;
4078
4079         hist_data->n_vals++;
4080         hist_data->n_fields++;
4081
4082         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4083                 return -EINVAL;
4084
4085         return 0;
4086 }
4087
4088 static int __create_val_field(struct hist_trigger_data *hist_data,
4089                               unsigned int val_idx,
4090                               struct trace_event_file *file,
4091                               char *var_name, char *field_str,
4092                               unsigned long flags)
4093 {
4094         struct hist_field *hist_field;
4095         int ret = 0, n_subexprs = 0;
4096
4097         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4098         if (IS_ERR(hist_field)) {
4099                 ret = PTR_ERR(hist_field);
4100                 goto out;
4101         }
4102
4103         hist_data->fields[val_idx] = hist_field;
4104
4105         ++hist_data->n_vals;
4106         ++hist_data->n_fields;
4107
4108         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4109                 ret = -EINVAL;
4110  out:
4111         return ret;
4112 }
4113
4114 static int create_val_field(struct hist_trigger_data *hist_data,
4115                             unsigned int val_idx,
4116                             struct trace_event_file *file,
4117                             char *field_str)
4118 {
4119         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4120                 return -EINVAL;
4121
4122         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4123 }
4124
4125 static const char *no_comm = "(no comm)";
4126
4127 static u64 hist_field_execname(struct hist_field *hist_field,
4128                                struct tracing_map_elt *elt,
4129                                struct trace_buffer *buffer,
4130                                struct ring_buffer_event *rbe,
4131                                void *event)
4132 {
4133         struct hist_elt_data *elt_data;
4134
4135         if (WARN_ON_ONCE(!elt))
4136                 return (u64)(unsigned long)no_comm;
4137
4138         elt_data = elt->private_data;
4139
4140         if (WARN_ON_ONCE(!elt_data->comm))
4141                 return (u64)(unsigned long)no_comm;
4142
4143         return (u64)(unsigned long)(elt_data->comm);
4144 }
4145
4146 /* Convert a var that points to common_pid.execname to a string */
4147 static void update_var_execname(struct hist_field *hist_field)
4148 {
4149         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4150                 HIST_FIELD_FL_EXECNAME;
4151         hist_field->size = MAX_FILTER_STR_VAL;
4152         hist_field->is_signed = 0;
4153
4154         kfree_const(hist_field->type);
4155         hist_field->type = "char[]";
4156
4157         hist_field->fn = hist_field_execname;
4158 }
4159
4160 static int create_var_field(struct hist_trigger_data *hist_data,
4161                             unsigned int val_idx,
4162                             struct trace_event_file *file,
4163                             char *var_name, char *expr_str)
4164 {
4165         struct trace_array *tr = hist_data->event_file->tr;
4166         unsigned long flags = 0;
4167         int ret;
4168
4169         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4170                 return -EINVAL;
4171
4172         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4173                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4174                 return -EINVAL;
4175         }
4176
4177         flags |= HIST_FIELD_FL_VAR;
4178         hist_data->n_vars++;
4179         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4180                 return -EINVAL;
4181
4182         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4183
4184         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4185                 update_var_execname(hist_data->fields[val_idx]);
4186
4187         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4188                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4189
4190         return ret;
4191 }
4192
4193 static int create_val_fields(struct hist_trigger_data *hist_data,
4194                              struct trace_event_file *file)
4195 {
4196         char *fields_str, *field_str;
4197         unsigned int i, j = 1;
4198         int ret;
4199
4200         ret = create_hitcount_val(hist_data);
4201         if (ret)
4202                 goto out;
4203
4204         fields_str = hist_data->attrs->vals_str;
4205         if (!fields_str)
4206                 goto out;
4207
4208         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4209                      j < TRACING_MAP_VALS_MAX; i++) {
4210                 field_str = strsep(&fields_str, ",");
4211                 if (!field_str)
4212                         break;
4213
4214                 if (strcmp(field_str, "hitcount") == 0)
4215                         continue;
4216
4217                 ret = create_val_field(hist_data, j++, file, field_str);
4218                 if (ret)
4219                         goto out;
4220         }
4221
4222         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4223                 ret = -EINVAL;
4224  out:
4225         return ret;
4226 }
4227
4228 static int create_key_field(struct hist_trigger_data *hist_data,
4229                             unsigned int key_idx,
4230                             unsigned int key_offset,
4231                             struct trace_event_file *file,
4232                             char *field_str)
4233 {
4234         struct trace_array *tr = hist_data->event_file->tr;
4235         struct hist_field *hist_field = NULL;
4236         unsigned long flags = 0;
4237         unsigned int key_size;
4238         int ret = 0, n_subexprs = 0;
4239
4240         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4241                 return -EINVAL;
4242
4243         flags |= HIST_FIELD_FL_KEY;
4244
4245         if (strcmp(field_str, "stacktrace") == 0) {
4246                 flags |= HIST_FIELD_FL_STACKTRACE;
4247                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4248                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4249         } else {
4250                 hist_field = parse_expr(hist_data, file, field_str, flags,
4251                                         NULL, &n_subexprs);
4252                 if (IS_ERR(hist_field)) {
4253                         ret = PTR_ERR(hist_field);
4254                         goto out;
4255                 }
4256
4257                 if (field_has_hist_vars(hist_field, 0)) {
4258                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4259                         destroy_hist_field(hist_field, 0);
4260                         ret = -EINVAL;
4261                         goto out;
4262                 }
4263
4264                 key_size = hist_field->size;
4265         }
4266
4267         hist_data->fields[key_idx] = hist_field;
4268
4269         key_size = ALIGN(key_size, sizeof(u64));
4270         hist_data->fields[key_idx]->size = key_size;
4271         hist_data->fields[key_idx]->offset = key_offset;
4272
4273         hist_data->key_size += key_size;
4274
4275         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4276                 ret = -EINVAL;
4277                 goto out;
4278         }
4279
4280         hist_data->n_keys++;
4281         hist_data->n_fields++;
4282
4283         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4284                 return -EINVAL;
4285
4286         ret = key_size;
4287  out:
4288         return ret;
4289 }
4290
4291 static int create_key_fields(struct hist_trigger_data *hist_data,
4292                              struct trace_event_file *file)
4293 {
4294         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4295         char *fields_str, *field_str;
4296         int ret = -EINVAL;
4297
4298         fields_str = hist_data->attrs->keys_str;
4299         if (!fields_str)
4300                 goto out;
4301
4302         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4303                 field_str = strsep(&fields_str, ",");
4304                 if (!field_str)
4305                         break;
4306                 ret = create_key_field(hist_data, i, key_offset,
4307                                        file, field_str);
4308                 if (ret < 0)
4309                         goto out;
4310                 key_offset += ret;
4311         }
4312         if (fields_str) {
4313                 ret = -EINVAL;
4314                 goto out;
4315         }
4316         ret = 0;
4317  out:
4318         return ret;
4319 }
4320
4321 static int create_var_fields(struct hist_trigger_data *hist_data,
4322                              struct trace_event_file *file)
4323 {
4324         unsigned int i, j = hist_data->n_vals;
4325         int ret = 0;
4326
4327         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4328
4329         for (i = 0; i < n_vars; i++) {
4330                 char *var_name = hist_data->attrs->var_defs.name[i];
4331                 char *expr = hist_data->attrs->var_defs.expr[i];
4332
4333                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4334                 if (ret)
4335                         goto out;
4336         }
4337  out:
4338         return ret;
4339 }
4340
4341 static void free_var_defs(struct hist_trigger_data *hist_data)
4342 {
4343         unsigned int i;
4344
4345         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4346                 kfree(hist_data->attrs->var_defs.name[i]);
4347                 kfree(hist_data->attrs->var_defs.expr[i]);
4348         }
4349
4350         hist_data->attrs->var_defs.n_vars = 0;
4351 }
4352
4353 static int parse_var_defs(struct hist_trigger_data *hist_data)
4354 {
4355         struct trace_array *tr = hist_data->event_file->tr;
4356         char *s, *str, *var_name, *field_str;
4357         unsigned int i, j, n_vars = 0;
4358         int ret = 0;
4359
4360         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4361                 str = hist_data->attrs->assignment_str[i];
4362                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4363                         field_str = strsep(&str, ",");
4364                         if (!field_str)
4365                                 break;
4366
4367                         var_name = strsep(&field_str, "=");
4368                         if (!var_name || !field_str) {
4369                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4370                                          errpos(var_name));
4371                                 ret = -EINVAL;
4372                                 goto free;
4373                         }
4374
4375                         if (n_vars == TRACING_MAP_VARS_MAX) {
4376                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4377                                 ret = -EINVAL;
4378                                 goto free;
4379                         }
4380
4381                         s = kstrdup(var_name, GFP_KERNEL);
4382                         if (!s) {
4383                                 ret = -ENOMEM;
4384                                 goto free;
4385                         }
4386                         hist_data->attrs->var_defs.name[n_vars] = s;
4387
4388                         s = kstrdup(field_str, GFP_KERNEL);
4389                         if (!s) {
4390                                 ret = -ENOMEM;
4391                                 goto free;
4392                         }
4393                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4394
4395                         hist_data->attrs->var_defs.n_vars = n_vars;
4396                 }
4397         }
4398
4399         return ret;
4400  free:
4401         free_var_defs(hist_data);
4402
4403         return ret;
4404 }
4405
4406 static int create_hist_fields(struct hist_trigger_data *hist_data,
4407                               struct trace_event_file *file)
4408 {
4409         int ret;
4410
4411         ret = parse_var_defs(hist_data);
4412         if (ret)
4413                 goto out;
4414
4415         ret = create_val_fields(hist_data, file);
4416         if (ret)
4417                 goto out;
4418
4419         ret = create_var_fields(hist_data, file);
4420         if (ret)
4421                 goto out;
4422
4423         ret = create_key_fields(hist_data, file);
4424         if (ret)
4425                 goto out;
4426  out:
4427         free_var_defs(hist_data);
4428
4429         return ret;
4430 }
4431
4432 static int is_descending(struct trace_array *tr, const char *str)
4433 {
4434         if (!str)
4435                 return 0;
4436
4437         if (strcmp(str, "descending") == 0)
4438                 return 1;
4439
4440         if (strcmp(str, "ascending") == 0)
4441                 return 0;
4442
4443         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4444
4445         return -EINVAL;
4446 }
4447
4448 static int create_sort_keys(struct hist_trigger_data *hist_data)
4449 {
4450         struct trace_array *tr = hist_data->event_file->tr;
4451         char *fields_str = hist_data->attrs->sort_key_str;
4452         struct tracing_map_sort_key *sort_key;
4453         int descending, ret = 0;
4454         unsigned int i, j, k;
4455
4456         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4457
4458         if (!fields_str)
4459                 goto out;
4460
4461         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4462                 struct hist_field *hist_field;
4463                 char *field_str, *field_name;
4464                 const char *test_name;
4465
4466                 sort_key = &hist_data->sort_keys[i];
4467
4468                 field_str = strsep(&fields_str, ",");
4469                 if (!field_str)
4470                         break;
4471
4472                 if (!*field_str) {
4473                         ret = -EINVAL;
4474                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4475                         break;
4476                 }
4477
4478                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4479                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4480                         ret = -EINVAL;
4481                         break;
4482                 }
4483
4484                 field_name = strsep(&field_str, ".");
4485                 if (!field_name || !*field_name) {
4486                         ret = -EINVAL;
4487                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4488                         break;
4489                 }
4490
4491                 if (strcmp(field_name, "hitcount") == 0) {
4492                         descending = is_descending(tr, field_str);
4493                         if (descending < 0) {
4494                                 ret = descending;
4495                                 break;
4496                         }
4497                         sort_key->descending = descending;
4498                         continue;
4499                 }
4500
4501                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4502                         unsigned int idx;
4503
4504                         hist_field = hist_data->fields[j];
4505                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4506                                 continue;
4507
4508                         idx = k++;
4509
4510                         test_name = hist_field_name(hist_field, 0);
4511
4512                         if (strcmp(field_name, test_name) == 0) {
4513                                 sort_key->field_idx = idx;
4514                                 descending = is_descending(tr, field_str);
4515                                 if (descending < 0) {
4516                                         ret = descending;
4517                                         goto out;
4518                                 }
4519                                 sort_key->descending = descending;
4520                                 break;
4521                         }
4522                 }
4523                 if (j == hist_data->n_fields) {
4524                         ret = -EINVAL;
4525                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4526                         break;
4527                 }
4528         }
4529
4530         hist_data->n_sort_keys = i;
4531  out:
4532         return ret;
4533 }
4534
4535 static void destroy_actions(struct hist_trigger_data *hist_data)
4536 {
4537         unsigned int i;
4538
4539         for (i = 0; i < hist_data->n_actions; i++) {
4540                 struct action_data *data = hist_data->actions[i];
4541
4542                 if (data->handler == HANDLER_ONMATCH)
4543                         onmatch_destroy(data);
4544                 else if (data->handler == HANDLER_ONMAX ||
4545                          data->handler == HANDLER_ONCHANGE)
4546                         track_data_destroy(hist_data, data);
4547                 else
4548                         kfree(data);
4549         }
4550 }
4551
4552 static int parse_actions(struct hist_trigger_data *hist_data)
4553 {
4554         struct trace_array *tr = hist_data->event_file->tr;
4555         struct action_data *data;
4556         unsigned int i;
4557         int ret = 0;
4558         char *str;
4559         int len;
4560
4561         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4562                 str = hist_data->attrs->action_str[i];
4563
4564                 if ((len = str_has_prefix(str, "onmatch("))) {
4565                         char *action_str = str + len;
4566
4567                         data = onmatch_parse(tr, action_str);
4568                         if (IS_ERR(data)) {
4569                                 ret = PTR_ERR(data);
4570                                 break;
4571                         }
4572                 } else if ((len = str_has_prefix(str, "onmax("))) {
4573                         char *action_str = str + len;
4574
4575                         data = track_data_parse(hist_data, action_str,
4576                                                 HANDLER_ONMAX);
4577                         if (IS_ERR(data)) {
4578                                 ret = PTR_ERR(data);
4579                                 break;
4580                         }
4581                 } else if ((len = str_has_prefix(str, "onchange("))) {
4582                         char *action_str = str + len;
4583
4584                         data = track_data_parse(hist_data, action_str,
4585                                                 HANDLER_ONCHANGE);
4586                         if (IS_ERR(data)) {
4587                                 ret = PTR_ERR(data);
4588                                 break;
4589                         }
4590                 } else {
4591                         ret = -EINVAL;
4592                         break;
4593                 }
4594
4595                 hist_data->actions[hist_data->n_actions++] = data;
4596         }
4597
4598         return ret;
4599 }
4600
4601 static int create_actions(struct hist_trigger_data *hist_data)
4602 {
4603         struct action_data *data;
4604         unsigned int i;
4605         int ret = 0;
4606
4607         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4608                 data = hist_data->actions[i];
4609
4610                 if (data->handler == HANDLER_ONMATCH) {
4611                         ret = onmatch_create(hist_data, data);
4612                         if (ret)
4613                                 break;
4614                 } else if (data->handler == HANDLER_ONMAX ||
4615                            data->handler == HANDLER_ONCHANGE) {
4616                         ret = track_data_create(hist_data, data);
4617                         if (ret)
4618                                 break;
4619                 } else {
4620                         ret = -EINVAL;
4621                         break;
4622                 }
4623         }
4624
4625         return ret;
4626 }
4627
4628 static void print_actions(struct seq_file *m,
4629                           struct hist_trigger_data *hist_data,
4630                           struct tracing_map_elt *elt)
4631 {
4632         unsigned int i;
4633
4634         for (i = 0; i < hist_data->n_actions; i++) {
4635                 struct action_data *data = hist_data->actions[i];
4636
4637                 if (data->action == ACTION_SNAPSHOT)
4638                         continue;
4639
4640                 if (data->handler == HANDLER_ONMAX ||
4641                     data->handler == HANDLER_ONCHANGE)
4642                         track_data_print(m, hist_data, elt, data);
4643         }
4644 }
4645
4646 static void print_action_spec(struct seq_file *m,
4647                               struct hist_trigger_data *hist_data,
4648                               struct action_data *data)
4649 {
4650         unsigned int i;
4651
4652         if (data->action == ACTION_SAVE) {
4653                 for (i = 0; i < hist_data->n_save_vars; i++) {
4654                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4655                         if (i < hist_data->n_save_vars - 1)
4656                                 seq_puts(m, ",");
4657                 }
4658         } else if (data->action == ACTION_TRACE) {
4659                 if (data->use_trace_keyword)
4660                         seq_printf(m, "%s", data->synth_event_name);
4661                 for (i = 0; i < data->n_params; i++) {
4662                         if (i || data->use_trace_keyword)
4663                                 seq_puts(m, ",");
4664                         seq_printf(m, "%s", data->params[i]);
4665                 }
4666         }
4667 }
4668
4669 static void print_track_data_spec(struct seq_file *m,
4670                                   struct hist_trigger_data *hist_data,
4671                                   struct action_data *data)
4672 {
4673         if (data->handler == HANDLER_ONMAX)
4674                 seq_puts(m, ":onmax(");
4675         else if (data->handler == HANDLER_ONCHANGE)
4676                 seq_puts(m, ":onchange(");
4677         seq_printf(m, "%s", data->track_data.var_str);
4678         seq_printf(m, ").%s(", data->action_name);
4679
4680         print_action_spec(m, hist_data, data);
4681
4682         seq_puts(m, ")");
4683 }
4684
4685 static void print_onmatch_spec(struct seq_file *m,
4686                                struct hist_trigger_data *hist_data,
4687                                struct action_data *data)
4688 {
4689         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4690                    data->match_data.event);
4691
4692         seq_printf(m, "%s(", data->action_name);
4693
4694         print_action_spec(m, hist_data, data);
4695
4696         seq_puts(m, ")");
4697 }
4698
4699 static bool actions_match(struct hist_trigger_data *hist_data,
4700                           struct hist_trigger_data *hist_data_test)
4701 {
4702         unsigned int i, j;
4703
4704         if (hist_data->n_actions != hist_data_test->n_actions)
4705                 return false;
4706
4707         for (i = 0; i < hist_data->n_actions; i++) {
4708                 struct action_data *data = hist_data->actions[i];
4709                 struct action_data *data_test = hist_data_test->actions[i];
4710                 char *action_name, *action_name_test;
4711
4712                 if (data->handler != data_test->handler)
4713                         return false;
4714                 if (data->action != data_test->action)
4715                         return false;
4716
4717                 if (data->n_params != data_test->n_params)
4718                         return false;
4719
4720                 for (j = 0; j < data->n_params; j++) {
4721                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4722                                 return false;
4723                 }
4724
4725                 if (data->use_trace_keyword)
4726                         action_name = data->synth_event_name;
4727                 else
4728                         action_name = data->action_name;
4729
4730                 if (data_test->use_trace_keyword)
4731                         action_name_test = data_test->synth_event_name;
4732                 else
4733                         action_name_test = data_test->action_name;
4734
4735                 if (strcmp(action_name, action_name_test) != 0)
4736                         return false;
4737
4738                 if (data->handler == HANDLER_ONMATCH) {
4739                         if (strcmp(data->match_data.event_system,
4740                                    data_test->match_data.event_system) != 0)
4741                                 return false;
4742                         if (strcmp(data->match_data.event,
4743                                    data_test->match_data.event) != 0)
4744                                 return false;
4745                 } else if (data->handler == HANDLER_ONMAX ||
4746                            data->handler == HANDLER_ONCHANGE) {
4747                         if (strcmp(data->track_data.var_str,
4748                                    data_test->track_data.var_str) != 0)
4749                                 return false;
4750                 }
4751         }
4752
4753         return true;
4754 }
4755
4756
4757 static void print_actions_spec(struct seq_file *m,
4758                                struct hist_trigger_data *hist_data)
4759 {
4760         unsigned int i;
4761
4762         for (i = 0; i < hist_data->n_actions; i++) {
4763                 struct action_data *data = hist_data->actions[i];
4764
4765                 if (data->handler == HANDLER_ONMATCH)
4766                         print_onmatch_spec(m, hist_data, data);
4767                 else if (data->handler == HANDLER_ONMAX ||
4768                          data->handler == HANDLER_ONCHANGE)
4769                         print_track_data_spec(m, hist_data, data);
4770         }
4771 }
4772
4773 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4774 {
4775         unsigned int i;
4776
4777         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4778                 kfree(hist_data->field_var_hists[i]->cmd);
4779                 kfree(hist_data->field_var_hists[i]);
4780         }
4781 }
4782
4783 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4784 {
4785         if (!hist_data)
4786                 return;
4787
4788         destroy_hist_trigger_attrs(hist_data->attrs);
4789         destroy_hist_fields(hist_data);
4790         tracing_map_destroy(hist_data->map);
4791
4792         destroy_actions(hist_data);
4793         destroy_field_vars(hist_data);
4794         destroy_field_var_hists(hist_data);
4795
4796         kfree(hist_data);
4797 }
4798
4799 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4800 {
4801         struct tracing_map *map = hist_data->map;
4802         struct ftrace_event_field *field;
4803         struct hist_field *hist_field;
4804         int i, idx = 0;
4805
4806         for_each_hist_field(i, hist_data) {
4807                 hist_field = hist_data->fields[i];
4808                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4809                         tracing_map_cmp_fn_t cmp_fn;
4810
4811                         field = hist_field->field;
4812
4813                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4814                                 cmp_fn = tracing_map_cmp_none;
4815                         else if (!field)
4816                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4817                                                              hist_field->is_signed);
4818                         else if (is_string_field(field))
4819                                 cmp_fn = tracing_map_cmp_string;
4820                         else
4821                                 cmp_fn = tracing_map_cmp_num(field->size,
4822                                                              field->is_signed);
4823                         idx = tracing_map_add_key_field(map,
4824                                                         hist_field->offset,
4825                                                         cmp_fn);
4826                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4827                         idx = tracing_map_add_sum_field(map);
4828
4829                 if (idx < 0)
4830                         return idx;
4831
4832                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4833                         idx = tracing_map_add_var(map);
4834                         if (idx < 0)
4835                                 return idx;
4836                         hist_field->var.idx = idx;
4837                         hist_field->var.hist_data = hist_data;
4838                 }
4839         }
4840
4841         return 0;
4842 }
4843
4844 static struct hist_trigger_data *
4845 create_hist_data(unsigned int map_bits,
4846                  struct hist_trigger_attrs *attrs,
4847                  struct trace_event_file *file,
4848                  bool remove)
4849 {
4850         const struct tracing_map_ops *map_ops = NULL;
4851         struct hist_trigger_data *hist_data;
4852         int ret = 0;
4853
4854         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4855         if (!hist_data)
4856                 return ERR_PTR(-ENOMEM);
4857
4858         hist_data->attrs = attrs;
4859         hist_data->remove = remove;
4860         hist_data->event_file = file;
4861
4862         ret = parse_actions(hist_data);
4863         if (ret)
4864                 goto free;
4865
4866         ret = create_hist_fields(hist_data, file);
4867         if (ret)
4868                 goto free;
4869
4870         ret = create_sort_keys(hist_data);
4871         if (ret)
4872                 goto free;
4873
4874         map_ops = &hist_trigger_elt_data_ops;
4875
4876         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4877                                             map_ops, hist_data);
4878         if (IS_ERR(hist_data->map)) {
4879                 ret = PTR_ERR(hist_data->map);
4880                 hist_data->map = NULL;
4881                 goto free;
4882         }
4883
4884         ret = create_tracing_map_fields(hist_data);
4885         if (ret)
4886                 goto free;
4887  out:
4888         return hist_data;
4889  free:
4890         hist_data->attrs = NULL;
4891
4892         destroy_hist_data(hist_data);
4893
4894         hist_data = ERR_PTR(ret);
4895
4896         goto out;
4897 }
4898
4899 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4900                                     struct tracing_map_elt *elt,
4901                                     struct trace_buffer *buffer, void *rec,
4902                                     struct ring_buffer_event *rbe,
4903                                     u64 *var_ref_vals)
4904 {
4905         struct hist_elt_data *elt_data;
4906         struct hist_field *hist_field;
4907         unsigned int i, var_idx;
4908         u64 hist_val;
4909
4910         elt_data = elt->private_data;
4911         elt_data->var_ref_vals = var_ref_vals;
4912
4913         for_each_hist_val_field(i, hist_data) {
4914                 hist_field = hist_data->fields[i];
4915                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4916                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4917                         var_idx = hist_field->var.idx;
4918
4919                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4920                                 unsigned int str_start, var_str_idx, idx;
4921                                 char *str, *val_str;
4922                                 unsigned int size;
4923
4924                                 str_start = hist_data->n_field_var_str +
4925                                         hist_data->n_save_var_str;
4926                                 var_str_idx = hist_field->var_str_idx;
4927                                 idx = str_start + var_str_idx;
4928
4929                                 str = elt_data->field_var_str[idx];
4930                                 val_str = (char *)(uintptr_t)hist_val;
4931
4932                                 size = min(hist_field->size, STR_VAR_LEN_MAX);
4933                                 strscpy(str, val_str, size);
4934
4935                                 hist_val = (u64)(uintptr_t)str;
4936                         }
4937                         tracing_map_set_var(elt, var_idx, hist_val);
4938                         continue;
4939                 }
4940                 tracing_map_update_sum(elt, i, hist_val);
4941         }
4942
4943         for_each_hist_key_field(i, hist_data) {
4944                 hist_field = hist_data->fields[i];
4945                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4946                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4947                         var_idx = hist_field->var.idx;
4948                         tracing_map_set_var(elt, var_idx, hist_val);
4949                 }
4950         }
4951
4952         update_field_vars(hist_data, elt, buffer, rbe, rec);
4953 }
4954
4955 static inline void add_to_key(char *compound_key, void *key,
4956                               struct hist_field *key_field, void *rec)
4957 {
4958         size_t size = key_field->size;
4959
4960         if (key_field->flags & HIST_FIELD_FL_STRING) {
4961                 struct ftrace_event_field *field;
4962
4963                 field = key_field->field;
4964                 if (field->filter_type == FILTER_DYN_STRING)
4965                         size = *(u32 *)(rec + field->offset) >> 16;
4966                 else if (field->filter_type == FILTER_STATIC_STRING)
4967                         size = field->size;
4968
4969                 /* ensure NULL-termination */
4970                 if (size > key_field->size - 1)
4971                         size = key_field->size - 1;
4972
4973                 strncpy(compound_key + key_field->offset, (char *)key, size);
4974         } else
4975                 memcpy(compound_key + key_field->offset, key, size);
4976 }
4977
4978 static void
4979 hist_trigger_actions(struct hist_trigger_data *hist_data,
4980                      struct tracing_map_elt *elt,
4981                      struct trace_buffer *buffer, void *rec,
4982                      struct ring_buffer_event *rbe, void *key,
4983                      u64 *var_ref_vals)
4984 {
4985         struct action_data *data;
4986         unsigned int i;
4987
4988         for (i = 0; i < hist_data->n_actions; i++) {
4989                 data = hist_data->actions[i];
4990                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4991         }
4992 }
4993
4994 static void event_hist_trigger(struct event_trigger_data *data,
4995                                struct trace_buffer *buffer, void *rec,
4996                                struct ring_buffer_event *rbe)
4997 {
4998         struct hist_trigger_data *hist_data = data->private_data;
4999         bool use_compound_key = (hist_data->n_keys > 1);
5000         unsigned long entries[HIST_STACKTRACE_DEPTH];
5001         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5002         char compound_key[HIST_KEY_SIZE_MAX];
5003         struct tracing_map_elt *elt = NULL;
5004         struct hist_field *key_field;
5005         u64 field_contents;
5006         void *key = NULL;
5007         unsigned int i;
5008
5009         memset(compound_key, 0, hist_data->key_size);
5010
5011         for_each_hist_key_field(i, hist_data) {
5012                 key_field = hist_data->fields[i];
5013
5014                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5015                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5016                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5017                                          HIST_STACKTRACE_SKIP);
5018                         key = entries;
5019                 } else {
5020                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5021                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5022                                 key = (void *)(unsigned long)field_contents;
5023                                 use_compound_key = true;
5024                         } else
5025                                 key = (void *)&field_contents;
5026                 }
5027
5028                 if (use_compound_key)
5029                         add_to_key(compound_key, key, key_field, rec);
5030         }
5031
5032         if (use_compound_key)
5033                 key = compound_key;
5034
5035         if (hist_data->n_var_refs &&
5036             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5037                 return;
5038
5039         elt = tracing_map_insert(hist_data->map, key);
5040         if (!elt)
5041                 return;
5042
5043         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5044
5045         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5046                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5047 }
5048
5049 static void hist_trigger_stacktrace_print(struct seq_file *m,
5050                                           unsigned long *stacktrace_entries,
5051                                           unsigned int max_entries)
5052 {
5053         unsigned int spaces = 8;
5054         unsigned int i;
5055
5056         for (i = 0; i < max_entries; i++) {
5057                 if (!stacktrace_entries[i])
5058                         return;
5059
5060                 seq_printf(m, "%*c", 1 + spaces, ' ');
5061                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5062         }
5063 }
5064
5065 static void hist_trigger_print_key(struct seq_file *m,
5066                                    struct hist_trigger_data *hist_data,
5067                                    void *key,
5068                                    struct tracing_map_elt *elt)
5069 {
5070         struct hist_field *key_field;
5071         bool multiline = false;
5072         const char *field_name;
5073         unsigned int i;
5074         u64 uval;
5075
5076         seq_puts(m, "{ ");
5077
5078         for_each_hist_key_field(i, hist_data) {
5079                 key_field = hist_data->fields[i];
5080
5081                 if (i > hist_data->n_vals)
5082                         seq_puts(m, ", ");
5083
5084                 field_name = hist_field_name(key_field, 0);
5085
5086                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5087                         uval = *(u64 *)(key + key_field->offset);
5088                         seq_printf(m, "%s: %llx", field_name, uval);
5089                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5090                         uval = *(u64 *)(key + key_field->offset);
5091                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5092                                    uval, (void *)(uintptr_t)uval);
5093                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5094                         uval = *(u64 *)(key + key_field->offset);
5095                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5096                                    uval, (void *)(uintptr_t)uval);
5097                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5098                         struct hist_elt_data *elt_data = elt->private_data;
5099                         char *comm;
5100
5101                         if (WARN_ON_ONCE(!elt_data))
5102                                 return;
5103
5104                         comm = elt_data->comm;
5105
5106                         uval = *(u64 *)(key + key_field->offset);
5107                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5108                                    comm, uval);
5109                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5110                         const char *syscall_name;
5111
5112                         uval = *(u64 *)(key + key_field->offset);
5113                         syscall_name = get_syscall_name(uval);
5114                         if (!syscall_name)
5115                                 syscall_name = "unknown_syscall";
5116
5117                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5118                                    syscall_name, uval);
5119                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5120                         seq_puts(m, "stacktrace:\n");
5121                         hist_trigger_stacktrace_print(m,
5122                                                       key + key_field->offset,
5123                                                       HIST_STACKTRACE_DEPTH);
5124                         multiline = true;
5125                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5126                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5127                                    *(u64 *)(key + key_field->offset));
5128                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5129                         unsigned long buckets = key_field->buckets;
5130                         uval = *(u64 *)(key + key_field->offset);
5131                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5132                                    uval, uval + buckets -1);
5133                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5134                         seq_printf(m, "%s: %-50s", field_name,
5135                                    (char *)(key + key_field->offset));
5136                 } else {
5137                         uval = *(u64 *)(key + key_field->offset);
5138                         seq_printf(m, "%s: %10llu", field_name, uval);
5139                 }
5140         }
5141
5142         if (!multiline)
5143                 seq_puts(m, " ");
5144
5145         seq_puts(m, "}");
5146 }
5147
5148 static void hist_trigger_entry_print(struct seq_file *m,
5149                                      struct hist_trigger_data *hist_data,
5150                                      void *key,
5151                                      struct tracing_map_elt *elt)
5152 {
5153         const char *field_name;
5154         unsigned int i;
5155
5156         hist_trigger_print_key(m, hist_data, key, elt);
5157
5158         seq_printf(m, " hitcount: %10llu",
5159                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5160
5161         for (i = 1; i < hist_data->n_vals; i++) {
5162                 field_name = hist_field_name(hist_data->fields[i], 0);
5163
5164                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5165                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5166                         continue;
5167
5168                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5169                         seq_printf(m, "  %s: %10llx", field_name,
5170                                    tracing_map_read_sum(elt, i));
5171                 } else {
5172                         seq_printf(m, "  %s: %10llu", field_name,
5173                                    tracing_map_read_sum(elt, i));
5174                 }
5175         }
5176
5177         print_actions(m, hist_data, elt);
5178
5179         seq_puts(m, "\n");
5180 }
5181
5182 static int print_entries(struct seq_file *m,
5183                          struct hist_trigger_data *hist_data)
5184 {
5185         struct tracing_map_sort_entry **sort_entries = NULL;
5186         struct tracing_map *map = hist_data->map;
5187         int i, n_entries;
5188
5189         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5190                                              hist_data->n_sort_keys,
5191                                              &sort_entries);
5192         if (n_entries < 0)
5193                 return n_entries;
5194
5195         for (i = 0; i < n_entries; i++)
5196                 hist_trigger_entry_print(m, hist_data,
5197                                          sort_entries[i]->key,
5198                                          sort_entries[i]->elt);
5199
5200         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5201
5202         return n_entries;
5203 }
5204
5205 static void hist_trigger_show(struct seq_file *m,
5206                               struct event_trigger_data *data, int n)
5207 {
5208         struct hist_trigger_data *hist_data;
5209         int n_entries;
5210
5211         if (n > 0)
5212                 seq_puts(m, "\n\n");
5213
5214         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5215         data->ops->print(m, data->ops, data);
5216         seq_puts(m, "#\n\n");
5217
5218         hist_data = data->private_data;
5219         n_entries = print_entries(m, hist_data);
5220         if (n_entries < 0)
5221                 n_entries = 0;
5222
5223         track_data_snapshot_print(m, hist_data);
5224
5225         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5226                    (u64)atomic64_read(&hist_data->map->hits),
5227                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5228 }
5229
5230 static int hist_show(struct seq_file *m, void *v)
5231 {
5232         struct event_trigger_data *data;
5233         struct trace_event_file *event_file;
5234         int n = 0, ret = 0;
5235
5236         mutex_lock(&event_mutex);
5237
5238         event_file = event_file_data(m->private);
5239         if (unlikely(!event_file)) {
5240                 ret = -ENODEV;
5241                 goto out_unlock;
5242         }
5243
5244         list_for_each_entry(data, &event_file->triggers, list) {
5245                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5246                         hist_trigger_show(m, data, n++);
5247         }
5248
5249  out_unlock:
5250         mutex_unlock(&event_mutex);
5251
5252         return ret;
5253 }
5254
5255 static int event_hist_open(struct inode *inode, struct file *file)
5256 {
5257         int ret;
5258
5259         ret = security_locked_down(LOCKDOWN_TRACEFS);
5260         if (ret)
5261                 return ret;
5262
5263         return single_open(file, hist_show, file);
5264 }
5265
5266 const struct file_operations event_hist_fops = {
5267         .open = event_hist_open,
5268         .read = seq_read,
5269         .llseek = seq_lseek,
5270         .release = single_release,
5271 };
5272
5273 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5274 static void hist_field_debug_show_flags(struct seq_file *m,
5275                                         unsigned long flags)
5276 {
5277         seq_puts(m, "      flags:\n");
5278
5279         if (flags & HIST_FIELD_FL_KEY)
5280                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5281         else if (flags & HIST_FIELD_FL_HITCOUNT)
5282                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5283         else if (flags & HIST_FIELD_FL_VAR)
5284                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5285         else if (flags & HIST_FIELD_FL_VAR_REF)
5286                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5287         else
5288                 seq_puts(m, "        VAL: normal u64 value\n");
5289
5290         if (flags & HIST_FIELD_FL_ALIAS)
5291                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5292         else if (flags & HIST_FIELD_FL_CONST)
5293                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5294 }
5295
5296 static int hist_field_debug_show(struct seq_file *m,
5297                                  struct hist_field *field, unsigned long flags)
5298 {
5299         if ((field->flags & flags) != flags) {
5300                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5301                 return -EINVAL;
5302         }
5303
5304         hist_field_debug_show_flags(m, field->flags);
5305         if (field->field)
5306                 seq_printf(m, "      ftrace_event_field name: %s\n",
5307                            field->field->name);
5308
5309         if (field->flags & HIST_FIELD_FL_VAR) {
5310                 seq_printf(m, "      var.name: %s\n", field->var.name);
5311                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5312                            field->var.idx);
5313         }
5314
5315         if (field->flags & HIST_FIELD_FL_CONST)
5316                 seq_printf(m, "      constant: %llu\n", field->constant);
5317
5318         if (field->flags & HIST_FIELD_FL_ALIAS)
5319                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5320                            field->var_ref_idx);
5321
5322         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5323                 seq_printf(m, "      name: %s\n", field->name);
5324                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5325                            field->var.idx);
5326                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5327                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5328                            field->var_ref_idx);
5329                 if (field->system)
5330                         seq_printf(m, "      system: %s\n", field->system);
5331                 if (field->event_name)
5332                         seq_printf(m, "      event_name: %s\n", field->event_name);
5333         }
5334
5335         seq_printf(m, "      type: %s\n", field->type);
5336         seq_printf(m, "      size: %u\n", field->size);
5337         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5338
5339         return 0;
5340 }
5341
5342 static int field_var_debug_show(struct seq_file *m,
5343                                 struct field_var *field_var, unsigned int i,
5344                                 bool save_vars)
5345 {
5346         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5347         struct hist_field *field;
5348         int ret = 0;
5349
5350         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5351
5352         field = field_var->var;
5353
5354         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5355
5356         hist_field_debug_show_flags(m, field->flags);
5357         seq_printf(m, "      var.name: %s\n", field->var.name);
5358         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5359                    field->var.idx);
5360
5361         field = field_var->val;
5362
5363         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5364         if (field->field)
5365                 seq_printf(m, "      ftrace_event_field name: %s\n",
5366                            field->field->name);
5367         else {
5368                 ret = -EINVAL;
5369                 goto out;
5370         }
5371
5372         seq_printf(m, "      type: %s\n", field->type);
5373         seq_printf(m, "      size: %u\n", field->size);
5374         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5375 out:
5376         return ret;
5377 }
5378
5379 static int hist_action_debug_show(struct seq_file *m,
5380                                   struct action_data *data, int i)
5381 {
5382         int ret = 0;
5383
5384         if (data->handler == HANDLER_ONMAX ||
5385             data->handler == HANDLER_ONCHANGE) {
5386                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5387                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5388                                             HIST_FIELD_FL_VAR_REF);
5389                 if (ret)
5390                         goto out;
5391
5392                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5393                 ret = hist_field_debug_show(m, data->track_data.track_var,
5394                                             HIST_FIELD_FL_VAR);
5395                 if (ret)
5396                         goto out;
5397         }
5398
5399         if (data->handler == HANDLER_ONMATCH) {
5400                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5401                            i, data->match_data.event_system);
5402                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5403                            i, data->match_data.event);
5404         }
5405 out:
5406         return ret;
5407 }
5408
5409 static int hist_actions_debug_show(struct seq_file *m,
5410                                    struct hist_trigger_data *hist_data)
5411 {
5412         int i, ret = 0;
5413
5414         if (hist_data->n_actions)
5415                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5416
5417         for (i = 0; i < hist_data->n_actions; i++) {
5418                 struct action_data *action = hist_data->actions[i];
5419
5420                 ret = hist_action_debug_show(m, action, i);
5421                 if (ret)
5422                         goto out;
5423         }
5424
5425         if (hist_data->n_save_vars)
5426                 seq_puts(m, "\n  save action variables (save() params):\n");
5427
5428         for (i = 0; i < hist_data->n_save_vars; i++) {
5429                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5430                 if (ret)
5431                         goto out;
5432         }
5433 out:
5434         return ret;
5435 }
5436
5437 static void hist_trigger_debug_show(struct seq_file *m,
5438                                     struct event_trigger_data *data, int n)
5439 {
5440         struct hist_trigger_data *hist_data;
5441         int i, ret;
5442
5443         if (n > 0)
5444                 seq_puts(m, "\n\n");
5445
5446         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5447         data->ops->print(m, data->ops, data);
5448         seq_puts(m, "#\n\n");
5449
5450         hist_data = data->private_data;
5451
5452         seq_printf(m, "hist_data: %p\n\n", hist_data);
5453         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5454         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5455         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5456
5457         seq_puts(m, "\n  val fields:\n\n");
5458
5459         seq_puts(m, "    hist_data->fields[0]:\n");
5460         ret = hist_field_debug_show(m, hist_data->fields[0],
5461                                     HIST_FIELD_FL_HITCOUNT);
5462         if (ret)
5463                 return;
5464
5465         for (i = 1; i < hist_data->n_vals; i++) {
5466                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5467                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5468                 if (ret)
5469                         return;
5470         }
5471
5472         seq_puts(m, "\n  key fields:\n");
5473
5474         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5475                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5476                 ret = hist_field_debug_show(m, hist_data->fields[i],
5477                                             HIST_FIELD_FL_KEY);
5478                 if (ret)
5479                         return;
5480         }
5481
5482         if (hist_data->n_var_refs)
5483                 seq_puts(m, "\n  variable reference fields:\n");
5484
5485         for (i = 0; i < hist_data->n_var_refs; i++) {
5486                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5487                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5488                                             HIST_FIELD_FL_VAR_REF);
5489                 if (ret)
5490                         return;
5491         }
5492
5493         if (hist_data->n_field_vars)
5494                 seq_puts(m, "\n  field variables:\n");
5495
5496         for (i = 0; i < hist_data->n_field_vars; i++) {
5497                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5498                 if (ret)
5499                         return;
5500         }
5501
5502         ret = hist_actions_debug_show(m, hist_data);
5503         if (ret)
5504                 return;
5505 }
5506
5507 static int hist_debug_show(struct seq_file *m, void *v)
5508 {
5509         struct event_trigger_data *data;
5510         struct trace_event_file *event_file;
5511         int n = 0, ret = 0;
5512
5513         mutex_lock(&event_mutex);
5514
5515         event_file = event_file_data(m->private);
5516         if (unlikely(!event_file)) {
5517                 ret = -ENODEV;
5518                 goto out_unlock;
5519         }
5520
5521         list_for_each_entry(data, &event_file->triggers, list) {
5522                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5523                         hist_trigger_debug_show(m, data, n++);
5524         }
5525
5526  out_unlock:
5527         mutex_unlock(&event_mutex);
5528
5529         return ret;
5530 }
5531
5532 static int event_hist_debug_open(struct inode *inode, struct file *file)
5533 {
5534         int ret;
5535
5536         ret = security_locked_down(LOCKDOWN_TRACEFS);
5537         if (ret)
5538                 return ret;
5539
5540         return single_open(file, hist_debug_show, file);
5541 }
5542
5543 const struct file_operations event_hist_debug_fops = {
5544         .open = event_hist_debug_open,
5545         .read = seq_read,
5546         .llseek = seq_lseek,
5547         .release = single_release,
5548 };
5549 #endif
5550
5551 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5552 {
5553         const char *field_name = hist_field_name(hist_field, 0);
5554
5555         if (hist_field->var.name)
5556                 seq_printf(m, "%s=", hist_field->var.name);
5557
5558         if (hist_field->flags & HIST_FIELD_FL_CPU)
5559                 seq_puts(m, "common_cpu");
5560         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5561                 seq_printf(m, "%llu", hist_field->constant);
5562         else if (field_name) {
5563                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5564                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5565                         seq_putc(m, '$');
5566                 seq_printf(m, "%s", field_name);
5567         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5568                 seq_puts(m, "common_timestamp");
5569
5570         if (hist_field->flags) {
5571                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5572                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5573                         const char *flags = get_hist_field_flags(hist_field);
5574
5575                         if (flags)
5576                                 seq_printf(m, ".%s", flags);
5577                 }
5578         }
5579         if (hist_field->buckets)
5580                 seq_printf(m, "=%ld", hist_field->buckets);
5581 }
5582
5583 static int event_hist_trigger_print(struct seq_file *m,
5584                                     struct event_trigger_ops *ops,
5585                                     struct event_trigger_data *data)
5586 {
5587         struct hist_trigger_data *hist_data = data->private_data;
5588         struct hist_field *field;
5589         bool have_var = false;
5590         unsigned int i;
5591
5592         seq_puts(m, "hist:");
5593
5594         if (data->name)
5595                 seq_printf(m, "%s:", data->name);
5596
5597         seq_puts(m, "keys=");
5598
5599         for_each_hist_key_field(i, hist_data) {
5600                 field = hist_data->fields[i];
5601
5602                 if (i > hist_data->n_vals)
5603                         seq_puts(m, ",");
5604
5605                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5606                         seq_puts(m, "stacktrace");
5607                 else
5608                         hist_field_print(m, field);
5609         }
5610
5611         seq_puts(m, ":vals=");
5612
5613         for_each_hist_val_field(i, hist_data) {
5614                 field = hist_data->fields[i];
5615                 if (field->flags & HIST_FIELD_FL_VAR) {
5616                         have_var = true;
5617                         continue;
5618                 }
5619
5620                 if (i == HITCOUNT_IDX)
5621                         seq_puts(m, "hitcount");
5622                 else {
5623                         seq_puts(m, ",");
5624                         hist_field_print(m, field);
5625                 }
5626         }
5627
5628         if (have_var) {
5629                 unsigned int n = 0;
5630
5631                 seq_puts(m, ":");
5632
5633                 for_each_hist_val_field(i, hist_data) {
5634                         field = hist_data->fields[i];
5635
5636                         if (field->flags & HIST_FIELD_FL_VAR) {
5637                                 if (n++)
5638                                         seq_puts(m, ",");
5639                                 hist_field_print(m, field);
5640                         }
5641                 }
5642         }
5643
5644         seq_puts(m, ":sort=");
5645
5646         for (i = 0; i < hist_data->n_sort_keys; i++) {
5647                 struct tracing_map_sort_key *sort_key;
5648                 unsigned int idx, first_key_idx;
5649
5650                 /* skip VAR vals */
5651                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5652
5653                 sort_key = &hist_data->sort_keys[i];
5654                 idx = sort_key->field_idx;
5655
5656                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5657                         return -EINVAL;
5658
5659                 if (i > 0)
5660                         seq_puts(m, ",");
5661
5662                 if (idx == HITCOUNT_IDX)
5663                         seq_puts(m, "hitcount");
5664                 else {
5665                         if (idx >= first_key_idx)
5666                                 idx += hist_data->n_vars;
5667                         hist_field_print(m, hist_data->fields[idx]);
5668                 }
5669
5670                 if (sort_key->descending)
5671                         seq_puts(m, ".descending");
5672         }
5673         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5674         if (hist_data->enable_timestamps)
5675                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5676
5677         print_actions_spec(m, hist_data);
5678
5679         if (data->filter_str)
5680                 seq_printf(m, " if %s", data->filter_str);
5681
5682         if (data->paused)
5683                 seq_puts(m, " [paused]");
5684         else
5685                 seq_puts(m, " [active]");
5686
5687         seq_putc(m, '\n');
5688
5689         return 0;
5690 }
5691
5692 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5693                                    struct event_trigger_data *data)
5694 {
5695         struct hist_trigger_data *hist_data = data->private_data;
5696
5697         if (!data->ref && hist_data->attrs->name)
5698                 save_named_trigger(hist_data->attrs->name, data);
5699
5700         data->ref++;
5701
5702         return 0;
5703 }
5704
5705 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5706 {
5707         struct trace_event_file *file;
5708         unsigned int i;
5709         char *cmd;
5710         int ret;
5711
5712         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5713                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5714                 cmd = hist_data->field_var_hists[i]->cmd;
5715                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5716                                               "!hist", "hist", cmd);
5717                 WARN_ON_ONCE(ret < 0);
5718         }
5719 }
5720
5721 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5722                                     struct event_trigger_data *data)
5723 {
5724         struct hist_trigger_data *hist_data = data->private_data;
5725
5726         if (WARN_ON_ONCE(data->ref <= 0))
5727                 return;
5728
5729         data->ref--;
5730         if (!data->ref) {
5731                 if (data->name)
5732                         del_named_trigger(data);
5733
5734                 trigger_data_free(data);
5735
5736                 remove_hist_vars(hist_data);
5737
5738                 unregister_field_var_hists(hist_data);
5739
5740                 destroy_hist_data(hist_data);
5741         }
5742 }
5743
5744 static struct event_trigger_ops event_hist_trigger_ops = {
5745         .func                   = event_hist_trigger,
5746         .print                  = event_hist_trigger_print,
5747         .init                   = event_hist_trigger_init,
5748         .free                   = event_hist_trigger_free,
5749 };
5750
5751 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5752                                          struct event_trigger_data *data)
5753 {
5754         data->ref++;
5755
5756         save_named_trigger(data->named_data->name, data);
5757
5758         event_hist_trigger_init(ops, data->named_data);
5759
5760         return 0;
5761 }
5762
5763 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5764                                           struct event_trigger_data *data)
5765 {
5766         if (WARN_ON_ONCE(data->ref <= 0))
5767                 return;
5768
5769         event_hist_trigger_free(ops, data->named_data);
5770
5771         data->ref--;
5772         if (!data->ref) {
5773                 del_named_trigger(data);
5774                 trigger_data_free(data);
5775         }
5776 }
5777
5778 static struct event_trigger_ops event_hist_trigger_named_ops = {
5779         .func                   = event_hist_trigger,
5780         .print                  = event_hist_trigger_print,
5781         .init                   = event_hist_trigger_named_init,
5782         .free                   = event_hist_trigger_named_free,
5783 };
5784
5785 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5786                                                             char *param)
5787 {
5788         return &event_hist_trigger_ops;
5789 }
5790
5791 static void hist_clear(struct event_trigger_data *data)
5792 {
5793         struct hist_trigger_data *hist_data = data->private_data;
5794
5795         if (data->name)
5796                 pause_named_trigger(data);
5797
5798         tracepoint_synchronize_unregister();
5799
5800         tracing_map_clear(hist_data->map);
5801
5802         if (data->name)
5803                 unpause_named_trigger(data);
5804 }
5805
5806 static bool compatible_field(struct ftrace_event_field *field,
5807                              struct ftrace_event_field *test_field)
5808 {
5809         if (field == test_field)
5810                 return true;
5811         if (field == NULL || test_field == NULL)
5812                 return false;
5813         if (strcmp(field->name, test_field->name) != 0)
5814                 return false;
5815         if (strcmp(field->type, test_field->type) != 0)
5816                 return false;
5817         if (field->size != test_field->size)
5818                 return false;
5819         if (field->is_signed != test_field->is_signed)
5820                 return false;
5821
5822         return true;
5823 }
5824
5825 static bool hist_trigger_match(struct event_trigger_data *data,
5826                                struct event_trigger_data *data_test,
5827                                struct event_trigger_data *named_data,
5828                                bool ignore_filter)
5829 {
5830         struct tracing_map_sort_key *sort_key, *sort_key_test;
5831         struct hist_trigger_data *hist_data, *hist_data_test;
5832         struct hist_field *key_field, *key_field_test;
5833         unsigned int i;
5834
5835         if (named_data && (named_data != data_test) &&
5836             (named_data != data_test->named_data))
5837                 return false;
5838
5839         if (!named_data && is_named_trigger(data_test))
5840                 return false;
5841
5842         hist_data = data->private_data;
5843         hist_data_test = data_test->private_data;
5844
5845         if (hist_data->n_vals != hist_data_test->n_vals ||
5846             hist_data->n_fields != hist_data_test->n_fields ||
5847             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5848                 return false;
5849
5850         if (!ignore_filter) {
5851                 if ((data->filter_str && !data_test->filter_str) ||
5852                    (!data->filter_str && data_test->filter_str))
5853                         return false;
5854         }
5855
5856         for_each_hist_field(i, hist_data) {
5857                 key_field = hist_data->fields[i];
5858                 key_field_test = hist_data_test->fields[i];
5859
5860                 if (key_field->flags != key_field_test->flags)
5861                         return false;
5862                 if (!compatible_field(key_field->field, key_field_test->field))
5863                         return false;
5864                 if (key_field->offset != key_field_test->offset)
5865                         return false;
5866                 if (key_field->size != key_field_test->size)
5867                         return false;
5868                 if (key_field->is_signed != key_field_test->is_signed)
5869                         return false;
5870                 if (!!key_field->var.name != !!key_field_test->var.name)
5871                         return false;
5872                 if (key_field->var.name &&
5873                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5874                         return false;
5875         }
5876
5877         for (i = 0; i < hist_data->n_sort_keys; i++) {
5878                 sort_key = &hist_data->sort_keys[i];
5879                 sort_key_test = &hist_data_test->sort_keys[i];
5880
5881                 if (sort_key->field_idx != sort_key_test->field_idx ||
5882                     sort_key->descending != sort_key_test->descending)
5883                         return false;
5884         }
5885
5886         if (!ignore_filter && data->filter_str &&
5887             (strcmp(data->filter_str, data_test->filter_str) != 0))
5888                 return false;
5889
5890         if (!actions_match(hist_data, hist_data_test))
5891                 return false;
5892
5893         return true;
5894 }
5895
5896 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5897                                  struct event_trigger_data *data,
5898                                  struct trace_event_file *file)
5899 {
5900         struct hist_trigger_data *hist_data = data->private_data;
5901         struct event_trigger_data *test, *named_data = NULL;
5902         struct trace_array *tr = file->tr;
5903         int ret = 0;
5904
5905         if (hist_data->attrs->name) {
5906                 named_data = find_named_trigger(hist_data->attrs->name);
5907                 if (named_data) {
5908                         if (!hist_trigger_match(data, named_data, named_data,
5909                                                 true)) {
5910                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5911                                 ret = -EINVAL;
5912                                 goto out;
5913                         }
5914                 }
5915         }
5916
5917         if (hist_data->attrs->name && !named_data)
5918                 goto new;
5919
5920         lockdep_assert_held(&event_mutex);
5921
5922         list_for_each_entry(test, &file->triggers, list) {
5923                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5924                         if (!hist_trigger_match(data, test, named_data, false))
5925                                 continue;
5926                         if (hist_data->attrs->pause)
5927                                 test->paused = true;
5928                         else if (hist_data->attrs->cont)
5929                                 test->paused = false;
5930                         else if (hist_data->attrs->clear)
5931                                 hist_clear(test);
5932                         else {
5933                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5934                                 ret = -EEXIST;
5935                         }
5936                         goto out;
5937                 }
5938         }
5939  new:
5940         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5941                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5942                 ret = -ENOENT;
5943                 goto out;
5944         }
5945
5946         if (hist_data->attrs->pause)
5947                 data->paused = true;
5948
5949         if (named_data) {
5950                 data->private_data = named_data->private_data;
5951                 set_named_trigger_data(data, named_data);
5952                 data->ops = &event_hist_trigger_named_ops;
5953         }
5954
5955         if (data->ops->init) {
5956                 ret = data->ops->init(data->ops, data);
5957                 if (ret < 0)
5958                         goto out;
5959         }
5960
5961         if (hist_data->enable_timestamps) {
5962                 char *clock = hist_data->attrs->clock;
5963
5964                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5965                 if (ret) {
5966                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5967                         goto out;
5968                 }
5969
5970                 tracing_set_filter_buffering(file->tr, true);
5971         }
5972
5973         if (named_data)
5974                 destroy_hist_data(hist_data);
5975
5976         ret++;
5977  out:
5978         return ret;
5979 }
5980
5981 static int hist_trigger_enable(struct event_trigger_data *data,
5982                                struct trace_event_file *file)
5983 {
5984         int ret = 0;
5985
5986         list_add_tail_rcu(&data->list, &file->triggers);
5987
5988         update_cond_flag(file);
5989
5990         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5991                 list_del_rcu(&data->list);
5992                 update_cond_flag(file);
5993                 ret--;
5994         }
5995
5996         return ret;
5997 }
5998
5999 static bool have_hist_trigger_match(struct event_trigger_data *data,
6000                                     struct trace_event_file *file)
6001 {
6002         struct hist_trigger_data *hist_data = data->private_data;
6003         struct event_trigger_data *test, *named_data = NULL;
6004         bool match = false;
6005
6006         lockdep_assert_held(&event_mutex);
6007
6008         if (hist_data->attrs->name)
6009                 named_data = find_named_trigger(hist_data->attrs->name);
6010
6011         list_for_each_entry(test, &file->triggers, list) {
6012                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6013                         if (hist_trigger_match(data, test, named_data, false)) {
6014                                 match = true;
6015                                 break;
6016                         }
6017                 }
6018         }
6019
6020         return match;
6021 }
6022
6023 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6024                                     struct trace_event_file *file)
6025 {
6026         struct hist_trigger_data *hist_data = data->private_data;
6027         struct event_trigger_data *test, *named_data = NULL;
6028
6029         lockdep_assert_held(&event_mutex);
6030
6031         if (hist_data->attrs->name)
6032                 named_data = find_named_trigger(hist_data->attrs->name);
6033
6034         list_for_each_entry(test, &file->triggers, list) {
6035                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6036                         if (!hist_trigger_match(data, test, named_data, false))
6037                                 continue;
6038                         hist_data = test->private_data;
6039                         if (check_var_refs(hist_data))
6040                                 return true;
6041                         break;
6042                 }
6043         }
6044
6045         return false;
6046 }
6047
6048 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6049                                     struct event_trigger_data *data,
6050                                     struct trace_event_file *file)
6051 {
6052         struct hist_trigger_data *hist_data = data->private_data;
6053         struct event_trigger_data *test, *named_data = NULL;
6054         bool unregistered = false;
6055
6056         lockdep_assert_held(&event_mutex);
6057
6058         if (hist_data->attrs->name)
6059                 named_data = find_named_trigger(hist_data->attrs->name);
6060
6061         list_for_each_entry(test, &file->triggers, list) {
6062                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6063                         if (!hist_trigger_match(data, test, named_data, false))
6064                                 continue;
6065                         unregistered = true;
6066                         list_del_rcu(&test->list);
6067                         trace_event_trigger_enable_disable(file, 0);
6068                         update_cond_flag(file);
6069                         break;
6070                 }
6071         }
6072
6073         if (unregistered && test->ops->free)
6074                 test->ops->free(test->ops, test);
6075
6076         if (hist_data->enable_timestamps) {
6077                 if (!hist_data->remove || unregistered)
6078                         tracing_set_filter_buffering(file->tr, false);
6079         }
6080 }
6081
6082 static bool hist_file_check_refs(struct trace_event_file *file)
6083 {
6084         struct hist_trigger_data *hist_data;
6085         struct event_trigger_data *test;
6086
6087         lockdep_assert_held(&event_mutex);
6088
6089         list_for_each_entry(test, &file->triggers, list) {
6090                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6091                         hist_data = test->private_data;
6092                         if (check_var_refs(hist_data))
6093                                 return true;
6094                 }
6095         }
6096
6097         return false;
6098 }
6099
6100 static void hist_unreg_all(struct trace_event_file *file)
6101 {
6102         struct event_trigger_data *test, *n;
6103         struct hist_trigger_data *hist_data;
6104         struct synth_event *se;
6105         const char *se_name;
6106
6107         lockdep_assert_held(&event_mutex);
6108
6109         if (hist_file_check_refs(file))
6110                 return;
6111
6112         list_for_each_entry_safe(test, n, &file->triggers, list) {
6113                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6114                         hist_data = test->private_data;
6115                         list_del_rcu(&test->list);
6116                         trace_event_trigger_enable_disable(file, 0);
6117
6118                         se_name = trace_event_name(file->event_call);
6119                         se = find_synth_event(se_name);
6120                         if (se)
6121                                 se->ref--;
6122
6123                         update_cond_flag(file);
6124                         if (hist_data->enable_timestamps)
6125                                 tracing_set_filter_buffering(file->tr, false);
6126                         if (test->ops->free)
6127                                 test->ops->free(test->ops, test);
6128                 }
6129         }
6130 }
6131
6132 static int event_hist_trigger_func(struct event_command *cmd_ops,
6133                                    struct trace_event_file *file,
6134                                    char *glob, char *cmd, char *param)
6135 {
6136         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6137         struct event_trigger_data *trigger_data;
6138         struct hist_trigger_attrs *attrs;
6139         struct event_trigger_ops *trigger_ops;
6140         struct hist_trigger_data *hist_data;
6141         struct synth_event *se;
6142         const char *se_name;
6143         bool remove = false;
6144         char *trigger, *p, *start;
6145         int ret = 0;
6146
6147         lockdep_assert_held(&event_mutex);
6148
6149         if (glob && strlen(glob)) {
6150                 hist_err_clear();
6151                 last_cmd_set(file, param);
6152         }
6153
6154         if (!param)
6155                 return -EINVAL;
6156
6157         if (glob[0] == '!')
6158                 remove = true;
6159
6160         /*
6161          * separate the trigger from the filter (k:v [if filter])
6162          * allowing for whitespace in the trigger
6163          */
6164         p = trigger = param;
6165         do {
6166                 p = strstr(p, "if");
6167                 if (!p)
6168                         break;
6169                 if (p == param)
6170                         return -EINVAL;
6171                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6172                         p++;
6173                         continue;
6174                 }
6175                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6176                         return -EINVAL;
6177                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6178                         p++;
6179                         continue;
6180                 }
6181                 break;
6182         } while (p);
6183
6184         if (!p)
6185                 param = NULL;
6186         else {
6187                 *(p - 1) = '\0';
6188                 param = strstrip(p);
6189                 trigger = strstrip(trigger);
6190         }
6191
6192         /*
6193          * To simplify arithmetic expression parsing, replace occurrences of
6194          * '.sym-offset' modifier with '.symXoffset'
6195          */
6196         start = strstr(trigger, ".sym-offset");
6197         while (start) {
6198                 *(start + 4) = 'X';
6199                 start = strstr(start + 11, ".sym-offset");
6200         }
6201
6202         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6203         if (IS_ERR(attrs))
6204                 return PTR_ERR(attrs);
6205
6206         if (attrs->map_bits)
6207                 hist_trigger_bits = attrs->map_bits;
6208
6209         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6210         if (IS_ERR(hist_data)) {
6211                 destroy_hist_trigger_attrs(attrs);
6212                 return PTR_ERR(hist_data);
6213         }
6214
6215         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6216
6217         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6218         if (!trigger_data) {
6219                 ret = -ENOMEM;
6220                 goto out_free;
6221         }
6222
6223         trigger_data->count = -1;
6224         trigger_data->ops = trigger_ops;
6225         trigger_data->cmd_ops = cmd_ops;
6226
6227         INIT_LIST_HEAD(&trigger_data->list);
6228         RCU_INIT_POINTER(trigger_data->filter, NULL);
6229
6230         trigger_data->private_data = hist_data;
6231
6232         /* if param is non-empty, it's supposed to be a filter */
6233         if (param && cmd_ops->set_filter) {
6234                 ret = cmd_ops->set_filter(param, trigger_data, file);
6235                 if (ret < 0)
6236                         goto out_free;
6237         }
6238
6239         if (remove) {
6240                 if (!have_hist_trigger_match(trigger_data, file))
6241                         goto out_free;
6242
6243                 if (hist_trigger_check_refs(trigger_data, file)) {
6244                         ret = -EBUSY;
6245                         goto out_free;
6246                 }
6247
6248                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6249                 se_name = trace_event_name(file->event_call);
6250                 se = find_synth_event(se_name);
6251                 if (se)
6252                         se->ref--;
6253                 ret = 0;
6254                 goto out_free;
6255         }
6256
6257         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6258         /*
6259          * The above returns on success the # of triggers registered,
6260          * but if it didn't register any it returns zero.  Consider no
6261          * triggers registered a failure too.
6262          */
6263         if (!ret) {
6264                 if (!(attrs->pause || attrs->cont || attrs->clear))
6265                         ret = -ENOENT;
6266                 goto out_free;
6267         } else if (ret < 0)
6268                 goto out_free;
6269
6270         if (get_named_trigger_data(trigger_data))
6271                 goto enable;
6272
6273         if (has_hist_vars(hist_data))
6274                 save_hist_vars(hist_data);
6275
6276         ret = create_actions(hist_data);
6277         if (ret)
6278                 goto out_unreg;
6279
6280         ret = tracing_map_init(hist_data->map);
6281         if (ret)
6282                 goto out_unreg;
6283 enable:
6284         ret = hist_trigger_enable(trigger_data, file);
6285         if (ret)
6286                 goto out_unreg;
6287
6288         se_name = trace_event_name(file->event_call);
6289         se = find_synth_event(se_name);
6290         if (se)
6291                 se->ref++;
6292         /* Just return zero, not the number of registered triggers */
6293         ret = 0;
6294  out:
6295         if (ret == 0)
6296                 hist_err_clear();
6297
6298         return ret;
6299  out_unreg:
6300         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6301  out_free:
6302         if (cmd_ops->set_filter)
6303                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6304
6305         remove_hist_vars(hist_data);
6306
6307         kfree(trigger_data);
6308
6309         destroy_hist_data(hist_data);
6310         goto out;
6311 }
6312
6313 static struct event_command trigger_hist_cmd = {
6314         .name                   = "hist",
6315         .trigger_type           = ETT_EVENT_HIST,
6316         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6317         .func                   = event_hist_trigger_func,
6318         .reg                    = hist_register_trigger,
6319         .unreg                  = hist_unregister_trigger,
6320         .unreg_all              = hist_unreg_all,
6321         .get_trigger_ops        = event_hist_get_trigger_ops,
6322         .set_filter             = set_trigger_filter,
6323 };
6324
6325 __init int register_trigger_hist_cmd(void)
6326 {
6327         int ret;
6328
6329         ret = register_event_command(&trigger_hist_cmd);
6330         WARN_ON(ret < 0);
6331
6332         return ret;
6333 }
6334
6335 static void
6336 hist_enable_trigger(struct event_trigger_data *data,
6337                     struct trace_buffer *buffer,  void *rec,
6338                     struct ring_buffer_event *event)
6339 {
6340         struct enable_trigger_data *enable_data = data->private_data;
6341         struct event_trigger_data *test;
6342
6343         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6344                                 lockdep_is_held(&event_mutex)) {
6345                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6346                         if (enable_data->enable)
6347                                 test->paused = false;
6348                         else
6349                                 test->paused = true;
6350                 }
6351         }
6352 }
6353
6354 static void
6355 hist_enable_count_trigger(struct event_trigger_data *data,
6356                           struct trace_buffer *buffer,  void *rec,
6357                           struct ring_buffer_event *event)
6358 {
6359         if (!data->count)
6360                 return;
6361
6362         if (data->count != -1)
6363                 (data->count)--;
6364
6365         hist_enable_trigger(data, buffer, rec, event);
6366 }
6367
6368 static struct event_trigger_ops hist_enable_trigger_ops = {
6369         .func                   = hist_enable_trigger,
6370         .print                  = event_enable_trigger_print,
6371         .init                   = event_trigger_init,
6372         .free                   = event_enable_trigger_free,
6373 };
6374
6375 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6376         .func                   = hist_enable_count_trigger,
6377         .print                  = event_enable_trigger_print,
6378         .init                   = event_trigger_init,
6379         .free                   = event_enable_trigger_free,
6380 };
6381
6382 static struct event_trigger_ops hist_disable_trigger_ops = {
6383         .func                   = hist_enable_trigger,
6384         .print                  = event_enable_trigger_print,
6385         .init                   = event_trigger_init,
6386         .free                   = event_enable_trigger_free,
6387 };
6388
6389 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6390         .func                   = hist_enable_count_trigger,
6391         .print                  = event_enable_trigger_print,
6392         .init                   = event_trigger_init,
6393         .free                   = event_enable_trigger_free,
6394 };
6395
6396 static struct event_trigger_ops *
6397 hist_enable_get_trigger_ops(char *cmd, char *param)
6398 {
6399         struct event_trigger_ops *ops;
6400         bool enable;
6401
6402         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6403
6404         if (enable)
6405                 ops = param ? &hist_enable_count_trigger_ops :
6406                         &hist_enable_trigger_ops;
6407         else
6408                 ops = param ? &hist_disable_count_trigger_ops :
6409                         &hist_disable_trigger_ops;
6410
6411         return ops;
6412 }
6413
6414 static void hist_enable_unreg_all(struct trace_event_file *file)
6415 {
6416         struct event_trigger_data *test, *n;
6417
6418         list_for_each_entry_safe(test, n, &file->triggers, list) {
6419                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6420                         list_del_rcu(&test->list);
6421                         update_cond_flag(file);
6422                         trace_event_trigger_enable_disable(file, 0);
6423                         if (test->ops->free)
6424                                 test->ops->free(test->ops, test);
6425                 }
6426         }
6427 }
6428
6429 static struct event_command trigger_hist_enable_cmd = {
6430         .name                   = ENABLE_HIST_STR,
6431         .trigger_type           = ETT_HIST_ENABLE,
6432         .func                   = event_enable_trigger_func,
6433         .reg                    = event_enable_register_trigger,
6434         .unreg                  = event_enable_unregister_trigger,
6435         .unreg_all              = hist_enable_unreg_all,
6436         .get_trigger_ops        = hist_enable_get_trigger_ops,
6437         .set_filter             = set_trigger_filter,
6438 };
6439
6440 static struct event_command trigger_hist_disable_cmd = {
6441         .name                   = DISABLE_HIST_STR,
6442         .trigger_type           = ETT_HIST_ENABLE,
6443         .func                   = event_enable_trigger_func,
6444         .reg                    = event_enable_register_trigger,
6445         .unreg                  = event_enable_unregister_trigger,
6446         .unreg_all              = hist_enable_unreg_all,
6447         .get_trigger_ops        = hist_enable_get_trigger_ops,
6448         .set_filter             = set_trigger_filter,
6449 };
6450
6451 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6452 {
6453         unregister_event_command(&trigger_hist_enable_cmd);
6454         unregister_event_command(&trigger_hist_disable_cmd);
6455 }
6456
6457 __init int register_trigger_hist_enable_disable_cmds(void)
6458 {
6459         int ret;
6460
6461         ret = register_event_command(&trigger_hist_enable_cmd);
6462         if (WARN_ON(ret < 0))
6463                 return ret;
6464         ret = register_event_command(&trigger_hist_disable_cmd);
6465         if (WARN_ON(ret < 0))
6466                 unregister_trigger_hist_enable_disable_cmds();
6467
6468         return ret;
6469 }