kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),                    \
  73         C(NEED_NOHC_VAL,        "Non-hitcount value is required for 'nohitcount'"),
  74
  75 #undef C
  76 #define C(a, b)         HIST_ERR_##a
  77
  78 enum { ERRORS };
  79
  80 #undef C
  81 #define C(a, b)         b
  82
  83 static const char *err_text[] = { ERRORS };
  84
  85 struct hist_field;
  86
  87 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88                                 struct tracing_map_elt *elt,
  89                                 struct trace_buffer *buffer,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96 #define HIST_CONST_DIGITS_MAX   21
  97 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  98
  99 enum field_op_id {
 100         FIELD_OP_NONE,
 101         FIELD_OP_PLUS,
 102         FIELD_OP_MINUS,
 103         FIELD_OP_UNARY_MINUS,
 104         FIELD_OP_DIV,
 105         FIELD_OP_MULT,
 106 };
 107
 108 enum hist_field_fn {
 109         HIST_FIELD_FN_NOP,
 110         HIST_FIELD_FN_VAR_REF,
 111         HIST_FIELD_FN_COUNTER,
 112         HIST_FIELD_FN_CONST,
 113         HIST_FIELD_FN_LOG2,
 114         HIST_FIELD_FN_BUCKET,
 115         HIST_FIELD_FN_TIMESTAMP,
 116         HIST_FIELD_FN_CPU,
 117         HIST_FIELD_FN_STRING,
 118         HIST_FIELD_FN_DYNSTRING,
 119         HIST_FIELD_FN_RELDYNSTRING,
 120         HIST_FIELD_FN_PSTRING,
 121         HIST_FIELD_FN_S64,
 122         HIST_FIELD_FN_U64,
 123         HIST_FIELD_FN_S32,
 124         HIST_FIELD_FN_U32,
 125         HIST_FIELD_FN_S16,
 126         HIST_FIELD_FN_U16,
 127         HIST_FIELD_FN_S8,
 128         HIST_FIELD_FN_U8,
 129         HIST_FIELD_FN_UMINUS,
 130         HIST_FIELD_FN_MINUS,
 131         HIST_FIELD_FN_PLUS,
 132         HIST_FIELD_FN_DIV,
 133         HIST_FIELD_FN_MULT,
 134         HIST_FIELD_FN_DIV_POWER2,
 135         HIST_FIELD_FN_DIV_NOT_POWER2,
 136         HIST_FIELD_FN_DIV_MULT_SHIFT,
 137         HIST_FIELD_FN_EXECNAME,
 138 };
 139
 140 /*
 141  * A hist_var (histogram variable) contains variable information for
 142  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 143  * flag set.  A hist_var has a variable name e.g. ts0, and is
 144  * associated with a given histogram trigger, as specified by
 145  * hist_data.  The hist_var idx is the unique index assigned to the
 146  * variable by the hist trigger's tracing_map.  The idx is what is
 147  * used to set a variable's value and, by a variable reference, to
 148  * retrieve it.
 149  */
 150 struct hist_var {
 151         char                            *name;
 152         struct hist_trigger_data        *hist_data;
 153         unsigned int                    idx;
 154 };
 155
 156 struct hist_field {
 157         struct ftrace_event_field       *field;
 158         unsigned long                   flags;
 159         unsigned long                   buckets;
 160         const char                      *type;
 161         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 162         struct hist_trigger_data        *hist_data;
 163         enum hist_field_fn              fn_num;
 164         unsigned int                    ref;
 165         unsigned int                    size;
 166         unsigned int                    offset;
 167         unsigned int                    is_signed;
 168
 169         /*
 170          * Variable fields contain variable-specific info in var.
 171          */
 172         struct hist_var                 var;
 173         enum field_op_id                operator;
 174         char                            *system;
 175         char                            *event_name;
 176
 177         /*
 178          * The name field is used for EXPR and VAR_REF fields.  VAR
 179          * fields contain the variable name in var.name.
 180          */
 181         char                            *name;
 182
 183         /*
 184          * When a histogram trigger is hit, if it has any references
 185          * to variables, the values of those variables are collected
 186          * into a var_ref_vals array by resolve_var_refs().  The
 187          * current value of each variable is read from the tracing_map
 188          * using the hist field's hist_var.idx and entered into the
 189          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 190          */
 191         unsigned int                    var_ref_idx;
 192         bool                            read_once;
 193
 194         unsigned int                    var_str_idx;
 195
 196         /* Numeric literals are represented as u64 */
 197         u64                             constant;
 198         /* Used to optimize division by constants */
 199         u64                             div_multiplier;
 200 };
 201
 202 static u64 hist_fn_call(struct hist_field *hist_field,
 203                         struct tracing_map_elt *elt,
 204                         struct trace_buffer *buffer,
 205                         struct ring_buffer_event *rbe,
 206                         void *event);
 207
 208 static u64 hist_field_const(struct hist_field *field,
 209                            struct tracing_map_elt *elt,
 210                            struct trace_buffer *buffer,
 211                            struct ring_buffer_event *rbe,
 212                            void *event)
 213 {
 214         return field->constant;
 215 }
 216
 217 static u64 hist_field_counter(struct hist_field *field,
 218                               struct tracing_map_elt *elt,
 219                               struct trace_buffer *buffer,
 220                               struct ring_buffer_event *rbe,
 221                               void *event)
 222 {
 223         return 1;
 224 }
 225
 226 static u64 hist_field_string(struct hist_field *hist_field,
 227                              struct tracing_map_elt *elt,
 228                              struct trace_buffer *buffer,
 229                              struct ring_buffer_event *rbe,
 230                              void *event)
 231 {
 232         char *addr = (char *)(event + hist_field->field->offset);
 233
 234         return (u64)(unsigned long)addr;
 235 }
 236
 237 static u64 hist_field_dynstring(struct hist_field *hist_field,
 238                                 struct tracing_map_elt *elt,
 239                                 struct trace_buffer *buffer,
 240                                 struct ring_buffer_event *rbe,
 241                                 void *event)
 242 {
 243         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 244         int str_loc = str_item & 0xffff;
 245         char *addr = (char *)(event + str_loc);
 246
 247         return (u64)(unsigned long)addr;
 248 }
 249
 250 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 251                                    struct tracing_map_elt *elt,
 252                                    struct trace_buffer *buffer,
 253                                    struct ring_buffer_event *rbe,
 254                                    void *event)
 255 {
 256         u32 *item = event + hist_field->field->offset;
 257         u32 str_item = *item;
 258         int str_loc = str_item & 0xffff;
 259         char *addr = (char *)&item[1] + str_loc;
 260
 261         return (u64)(unsigned long)addr;
 262 }
 263
 264 static u64 hist_field_pstring(struct hist_field *hist_field,
 265                               struct tracing_map_elt *elt,
 266                               struct trace_buffer *buffer,
 267                               struct ring_buffer_event *rbe,
 268                               void *event)
 269 {
 270         char **addr = (char **)(event + hist_field->field->offset);
 271
 272         return (u64)(unsigned long)*addr;
 273 }
 274
 275 static u64 hist_field_log2(struct hist_field *hist_field,
 276                            struct tracing_map_elt *elt,
 277                            struct trace_buffer *buffer,
 278                            struct ring_buffer_event *rbe,
 279                            void *event)
 280 {
 281         struct hist_field *operand = hist_field->operands[0];
 282
 283         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 284
 285         return (u64) ilog2(roundup_pow_of_two(val));
 286 }
 287
 288 static u64 hist_field_bucket(struct hist_field *hist_field,
 289                              struct tracing_map_elt *elt,
 290                              struct trace_buffer *buffer,
 291                              struct ring_buffer_event *rbe,
 292                              void *event)
 293 {
 294         struct hist_field *operand = hist_field->operands[0];
 295         unsigned long buckets = hist_field->buckets;
 296
 297         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 298
 299         if (WARN_ON_ONCE(!buckets))
 300                 return val;
 301
 302         if (val >= LONG_MAX)
 303                 val = div64_ul(val, buckets);
 304         else
 305                 val = (u64)((unsigned long)val / buckets);
 306         return val * buckets;
 307 }
 308
 309 static u64 hist_field_plus(struct hist_field *hist_field,
 310                            struct tracing_map_elt *elt,
 311                            struct trace_buffer *buffer,
 312                            struct ring_buffer_event *rbe,
 313                            void *event)
 314 {
 315         struct hist_field *operand1 = hist_field->operands[0];
 316         struct hist_field *operand2 = hist_field->operands[1];
 317
 318         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 319         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 320
 321         return val1 + val2;
 322 }
 323
 324 static u64 hist_field_minus(struct hist_field *hist_field,
 325                             struct tracing_map_elt *elt,
 326                             struct trace_buffer *buffer,
 327                             struct ring_buffer_event *rbe,
 328                             void *event)
 329 {
 330         struct hist_field *operand1 = hist_field->operands[0];
 331         struct hist_field *operand2 = hist_field->operands[1];
 332
 333         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 334         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 335
 336         return val1 - val2;
 337 }
 338
 339 static u64 hist_field_div(struct hist_field *hist_field,
 340                            struct tracing_map_elt *elt,
 341                            struct trace_buffer *buffer,
 342                            struct ring_buffer_event *rbe,
 343                            void *event)
 344 {
 345         struct hist_field *operand1 = hist_field->operands[0];
 346         struct hist_field *operand2 = hist_field->operands[1];
 347
 348         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 349         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 350
 351         /* Return -1 for the undefined case */
 352         if (!val2)
 353                 return -1;
 354
 355         /* Use shift if the divisor is a power of 2 */
 356         if (!(val2 & (val2 - 1)))
 357                 return val1 >> __ffs64(val2);
 358
 359         return div64_u64(val1, val2);
 360 }
 361
 362 static u64 div_by_power_of_two(struct hist_field *hist_field,
 363                                 struct tracing_map_elt *elt,
 364                                 struct trace_buffer *buffer,
 365                                 struct ring_buffer_event *rbe,
 366                                 void *event)
 367 {
 368         struct hist_field *operand1 = hist_field->operands[0];
 369         struct hist_field *operand2 = hist_field->operands[1];
 370
 371         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 372
 373         return val1 >> __ffs64(operand2->constant);
 374 }
 375
 376 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 377                                 struct tracing_map_elt *elt,
 378                                 struct trace_buffer *buffer,
 379                                 struct ring_buffer_event *rbe,
 380                                 void *event)
 381 {
 382         struct hist_field *operand1 = hist_field->operands[0];
 383         struct hist_field *operand2 = hist_field->operands[1];
 384
 385         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 386
 387         return div64_u64(val1, operand2->constant);
 388 }
 389
 390 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 391                                 struct tracing_map_elt *elt,
 392                                 struct trace_buffer *buffer,
 393                                 struct ring_buffer_event *rbe,
 394                                 void *event)
 395 {
 396         struct hist_field *operand1 = hist_field->operands[0];
 397         struct hist_field *operand2 = hist_field->operands[1];
 398
 399         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 400
 401         /*
 402          * If the divisor is a constant, do a multiplication and shift instead.
 403          *
 404          * Choose Z = some power of 2. If Y <= Z, then:
 405          *     X / Y = (X * (Z / Y)) / Z
 406          *
 407          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 408          *     X / Y = (X * mult) / Z
 409          *
 410          * The division by Z can be replaced by a shift since Z is a power of 2:
 411          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 412          *
 413          * As long, as X < Z the results will not be off by more than 1.
 414          */
 415         if (val1 < (1 << HIST_DIV_SHIFT)) {
 416                 u64 mult = operand2->div_multiplier;
 417
 418                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 419         }
 420
 421         return div64_u64(val1, operand2->constant);
 422 }
 423
 424 static u64 hist_field_mult(struct hist_field *hist_field,
 425                            struct tracing_map_elt *elt,
 426                            struct trace_buffer *buffer,
 427                            struct ring_buffer_event *rbe,
 428                            void *event)
 429 {
 430         struct hist_field *operand1 = hist_field->operands[0];
 431         struct hist_field *operand2 = hist_field->operands[1];
 432
 433         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 434         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 435
 436         return val1 * val2;
 437 }
 438
 439 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 440                                   struct tracing_map_elt *elt,
 441                                   struct trace_buffer *buffer,
 442                                   struct ring_buffer_event *rbe,
 443                                   void *event)
 444 {
 445         struct hist_field *operand = hist_field->operands[0];
 446
 447         s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 448         u64 val = (u64)-sval;
 449
 450         return val;
 451 }
 452
 453 #define DEFINE_HIST_FIELD_FN(type)                                      \
 454         static u64 hist_field_##type(struct hist_field *hist_field,     \
 455                                      struct tracing_map_elt *elt,       \
 456                                      struct trace_buffer *buffer,       \
 457                                      struct ring_buffer_event *rbe,     \
 458                                      void *event)                       \
 459 {                                                                       \
 460         type *addr = (type *)(event + hist_field->field->offset);       \
 461                                                                         \
 462         return (u64)(unsigned long)*addr;                               \
 463 }
 464
 465 DEFINE_HIST_FIELD_FN(s64);
 466 DEFINE_HIST_FIELD_FN(u64);
 467 DEFINE_HIST_FIELD_FN(s32);
 468 DEFINE_HIST_FIELD_FN(u32);
 469 DEFINE_HIST_FIELD_FN(s16);
 470 DEFINE_HIST_FIELD_FN(u16);
 471 DEFINE_HIST_FIELD_FN(s8);
 472 DEFINE_HIST_FIELD_FN(u8);
 473
 474 #define for_each_hist_field(i, hist_data)       \
 475         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 476
 477 #define for_each_hist_val_field(i, hist_data)   \
 478         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 479
 480 #define for_each_hist_key_field(i, hist_data)   \
 481         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 482
 483 #define HITCOUNT_IDX            0
 484 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 485
 486 enum hist_field_flags {
 487         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 488         HIST_FIELD_FL_KEY               = 1 << 1,
 489         HIST_FIELD_FL_STRING            = 1 << 2,
 490         HIST_FIELD_FL_HEX               = 1 << 3,
 491         HIST_FIELD_FL_SYM               = 1 << 4,
 492         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 493         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 494         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 495         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 496         HIST_FIELD_FL_LOG2              = 1 << 9,
 497         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 498         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 499         HIST_FIELD_FL_VAR               = 1 << 12,
 500         HIST_FIELD_FL_EXPR              = 1 << 13,
 501         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 502         HIST_FIELD_FL_CPU               = 1 << 15,
 503         HIST_FIELD_FL_ALIAS             = 1 << 16,
 504         HIST_FIELD_FL_BUCKET            = 1 << 17,
 505         HIST_FIELD_FL_CONST             = 1 << 18,
 506         HIST_FIELD_FL_PERCENT           = 1 << 19,
 507         HIST_FIELD_FL_GRAPH             = 1 << 20,
 508 };
 509
 510 struct var_defs {
 511         unsigned int    n_vars;
 512         char            *name[TRACING_MAP_VARS_MAX];
 513         char            *expr[TRACING_MAP_VARS_MAX];
 514 };
 515
 516 struct hist_trigger_attrs {
 517         char            *keys_str;
 518         char            *vals_str;
 519         char            *sort_key_str;
 520         char            *name;
 521         char            *clock;
 522         bool            pause;
 523         bool            cont;
 524         bool            clear;
 525         bool            ts_in_usecs;
 526         bool            no_hitcount;
 527         unsigned int    map_bits;
 528
 529         char            *assignment_str[TRACING_MAP_VARS_MAX];
 530         unsigned int    n_assignments;
 531
 532         char            *action_str[HIST_ACTIONS_MAX];
 533         unsigned int    n_actions;
 534
 535         struct var_defs var_defs;
 536 };
 537
 538 struct field_var {
 539         struct hist_field       *var;
 540         struct hist_field       *val;
 541 };
 542
 543 struct field_var_hist {
 544         struct hist_trigger_data        *hist_data;
 545         char                            *cmd;
 546 };
 547
 548 struct hist_trigger_data {
 549         struct hist_field               *fields[HIST_FIELDS_MAX];
 550         unsigned int                    n_vals;
 551         unsigned int                    n_keys;
 552         unsigned int                    n_fields;
 553         unsigned int                    n_vars;
 554         unsigned int                    n_var_str;
 555         unsigned int                    key_size;
 556         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 557         unsigned int                    n_sort_keys;
 558         struct trace_event_file         *event_file;
 559         struct hist_trigger_attrs       *attrs;
 560         struct tracing_map              *map;
 561         bool                            enable_timestamps;
 562         bool                            remove;
 563         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 564         unsigned int                    n_var_refs;
 565
 566         struct action_data              *actions[HIST_ACTIONS_MAX];
 567         unsigned int                    n_actions;
 568
 569         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 570         unsigned int                    n_field_vars;
 571         unsigned int                    n_field_var_str;
 572         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 573         unsigned int                    n_field_var_hists;
 574
 575         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 576         unsigned int                    n_save_vars;
 577         unsigned int                    n_save_var_str;
 578 };
 579
 580 struct action_data;
 581
 582 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 583                              struct tracing_map_elt *elt,
 584                              struct trace_buffer *buffer, void *rec,
 585                              struct ring_buffer_event *rbe, void *key,
 586                              struct action_data *data, u64 *var_ref_vals);
 587
 588 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 589
 590 enum handler_id {
 591         HANDLER_ONMATCH = 1,
 592         HANDLER_ONMAX,
 593         HANDLER_ONCHANGE,
 594 };
 595
 596 enum action_id {
 597         ACTION_SAVE = 1,
 598         ACTION_TRACE,
 599         ACTION_SNAPSHOT,
 600 };
 601
 602 struct action_data {
 603         enum handler_id         handler;
 604         enum action_id          action;
 605         char                    *action_name;
 606         action_fn_t             fn;
 607
 608         unsigned int            n_params;
 609         char                    *params[SYNTH_FIELDS_MAX];
 610
 611         /*
 612          * When a histogram trigger is hit, the values of any
 613          * references to variables, including variables being passed
 614          * as parameters to synthetic events, are collected into a
 615          * var_ref_vals array.  This var_ref_idx array is an array of
 616          * indices into the var_ref_vals array, one for each synthetic
 617          * event param, and is passed to the synthetic event
 618          * invocation.
 619          */
 620         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 621         struct synth_event      *synth_event;
 622         bool                    use_trace_keyword;
 623         char                    *synth_event_name;
 624
 625         union {
 626                 struct {
 627                         char                    *event;
 628                         char                    *event_system;
 629                 } match_data;
 630
 631                 struct {
 632                         /*
 633                          * var_str contains the $-unstripped variable
 634                          * name referenced by var_ref, and used when
 635                          * printing the action.  Because var_ref
 636                          * creation is deferred to create_actions(),
 637                          * we need a per-action way to save it until
 638                          * then, thus var_str.
 639                          */
 640                         char                    *var_str;
 641
 642                         /*
 643                          * var_ref refers to the variable being
 644                          * tracked e.g onmax($var).
 645                          */
 646                         struct hist_field       *var_ref;
 647
 648                         /*
 649                          * track_var contains the 'invisible' tracking
 650                          * variable created to keep the current
 651                          * e.g. max value.
 652                          */
 653                         struct hist_field       *track_var;
 654
 655                         check_track_val_fn_t    check_val;
 656                         action_fn_t             save_data;
 657                 } track_data;
 658         };
 659 };
 660
 661 struct track_data {
 662         u64                             track_val;
 663         bool                            updated;
 664
 665         unsigned int                    key_len;
 666         void                            *key;
 667         struct tracing_map_elt          elt;
 668
 669         struct action_data              *action_data;
 670         struct hist_trigger_data        *hist_data;
 671 };
 672
 673 struct hist_elt_data {
 674         char *comm;
 675         u64 *var_ref_vals;
 676         char **field_var_str;
 677         int n_field_var_str;
 678 };
 679
 680 struct snapshot_context {
 681         struct tracing_map_elt  *elt;
 682         void                    *key;
 683 };
 684
 685 /*
 686  * Returns the specific division function to use if the divisor
 687  * is constant. This avoids extra branches when the trigger is hit.
 688  */
 689 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 690 {
 691         u64 div = divisor->constant;
 692
 693         if (!(div & (div - 1)))
 694                 return HIST_FIELD_FN_DIV_POWER2;
 695
 696         /* If the divisor is too large, do a regular division */
 697         if (div > (1 << HIST_DIV_SHIFT))
 698                 return HIST_FIELD_FN_DIV_NOT_POWER2;
 699
 700         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 701         return HIST_FIELD_FN_DIV_MULT_SHIFT;
 702 }
 703
 704 static void track_data_free(struct track_data *track_data)
 705 {
 706         struct hist_elt_data *elt_data;
 707
 708         if (!track_data)
 709                 return;
 710
 711         kfree(track_data->key);
 712
 713         elt_data = track_data->elt.private_data;
 714         if (elt_data) {
 715                 kfree(elt_data->comm);
 716                 kfree(elt_data);
 717         }
 718
 719         kfree(track_data);
 720 }
 721
 722 static struct track_data *track_data_alloc(unsigned int key_len,
 723                                            struct action_data *action_data,
 724                                            struct hist_trigger_data *hist_data)
 725 {
 726         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 727         struct hist_elt_data *elt_data;
 728
 729         if (!data)
 730                 return ERR_PTR(-ENOMEM);
 731
 732         data->key = kzalloc(key_len, GFP_KERNEL);
 733         if (!data->key) {
 734                 track_data_free(data);
 735                 return ERR_PTR(-ENOMEM);
 736         }
 737
 738         data->key_len = key_len;
 739         data->action_data = action_data;
 740         data->hist_data = hist_data;
 741
 742         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 743         if (!elt_data) {
 744                 track_data_free(data);
 745                 return ERR_PTR(-ENOMEM);
 746         }
 747
 748         data->elt.private_data = elt_data;
 749
 750         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 751         if (!elt_data->comm) {
 752                 track_data_free(data);
 753                 return ERR_PTR(-ENOMEM);
 754         }
 755
 756         return data;
 757 }
 758
 759 #define HIST_PREFIX "hist:"
 760
 761 static char *last_cmd;
 762 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 763
 764 static int errpos(char *str)
 765 {
 766         if (!str || !last_cmd)
 767                 return 0;
 768
 769         return err_pos(last_cmd, str);
 770 }
 771
 772 static void last_cmd_set(struct trace_event_file *file, char *str)
 773 {
 774         const char *system = NULL, *name = NULL;
 775         struct trace_event_call *call;
 776         int len;
 777
 778         if (!str)
 779                 return;
 780
 781         /* sizeof() contains the nul byte */
 782         len = sizeof(HIST_PREFIX) + strlen(str);
 783         kfree(last_cmd);
 784         last_cmd = kzalloc(len, GFP_KERNEL);
 785         if (!last_cmd)
 786                 return;
 787
 788         strcpy(last_cmd, HIST_PREFIX);
 789         /* Again, sizeof() contains the nul byte */
 790         len -= sizeof(HIST_PREFIX);
 791         strncat(last_cmd, str, len);
 792
 793         if (file) {
 794                 call = file->event_call;
 795                 system = call->class->system;
 796                 if (system) {
 797                         name = trace_event_name(call);
 798                         if (!name)
 799                                 system = NULL;
 800                 }
 801         }
 802
 803         if (system)
 804                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 805 }
 806
 807 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 808 {
 809         if (!last_cmd)
 810                 return;
 811
 812         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 813                         err_type, err_pos);
 814 }
 815
 816 static void hist_err_clear(void)
 817 {
 818         if (last_cmd)
 819                 last_cmd[0] = '\0';
 820         last_cmd_loc[0] = '\0';
 821 }
 822
 823 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 824                                     unsigned int *var_ref_idx);
 825
 826 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 827                                unsigned int *var_ref_idx)
 828 {
 829         struct tracepoint *tp = event->tp;
 830
 831         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 832                 struct tracepoint_func *probe_func_ptr;
 833                 synth_probe_func_t probe_func;
 834                 void *__data;
 835
 836                 if (!(cpu_online(raw_smp_processor_id())))
 837                         return;
 838
 839                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 840                 if (probe_func_ptr) {
 841                         do {
 842                                 probe_func = probe_func_ptr->func;
 843                                 __data = probe_func_ptr->data;
 844                                 probe_func(__data, var_ref_vals, var_ref_idx);
 845                         } while ((++probe_func_ptr)->func);
 846                 }
 847         }
 848 }
 849
 850 static void action_trace(struct hist_trigger_data *hist_data,
 851                          struct tracing_map_elt *elt,
 852                          struct trace_buffer *buffer, void *rec,
 853                          struct ring_buffer_event *rbe, void *key,
 854                          struct action_data *data, u64 *var_ref_vals)
 855 {
 856         struct synth_event *event = data->synth_event;
 857
 858         trace_synth(event, var_ref_vals, data->var_ref_idx);
 859 }
 860
 861 struct hist_var_data {
 862         struct list_head list;
 863         struct hist_trigger_data *hist_data;
 864 };
 865
 866 static u64 hist_field_timestamp(struct hist_field *hist_field,
 867                                 struct tracing_map_elt *elt,
 868                                 struct trace_buffer *buffer,
 869                                 struct ring_buffer_event *rbe,
 870                                 void *event)
 871 {
 872         struct hist_trigger_data *hist_data = hist_field->hist_data;
 873         struct trace_array *tr = hist_data->event_file->tr;
 874
 875         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 876
 877         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 878                 ts = ns2usecs(ts);
 879
 880         return ts;
 881 }
 882
 883 static u64 hist_field_cpu(struct hist_field *hist_field,
 884                           struct tracing_map_elt *elt,
 885                           struct trace_buffer *buffer,
 886                           struct ring_buffer_event *rbe,
 887                           void *event)
 888 {
 889         int cpu = smp_processor_id();
 890
 891         return cpu;
 892 }
 893
 894 /**
 895  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 896  * @hist_field: The VAR_REF field to check
 897  * @var_data: The hist trigger that owns the variable
 898  * @var_idx: The trigger variable identifier
 899  *
 900  * Check the given VAR_REF field to see whether or not it references
 901  * the given variable associated with the given trigger.
 902  *
 903  * Return: The VAR_REF field if it does reference the variable, NULL if not
 904  */
 905 static struct hist_field *
 906 check_field_for_var_ref(struct hist_field *hist_field,
 907                         struct hist_trigger_data *var_data,
 908                         unsigned int var_idx)
 909 {
 910         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 911
 912         if (hist_field && hist_field->var.idx == var_idx &&
 913             hist_field->var.hist_data == var_data)
 914                 return hist_field;
 915
 916         return NULL;
 917 }
 918
 919 /**
 920  * find_var_ref - Check if a trigger has a reference to a trigger variable
 921  * @hist_data: The hist trigger that might have a reference to the variable
 922  * @var_data: The hist trigger that owns the variable
 923  * @var_idx: The trigger variable identifier
 924  *
 925  * Check the list of var_refs[] on the first hist trigger to see
 926  * whether any of them are references to the variable on the second
 927  * trigger.
 928  *
 929  * Return: The VAR_REF field referencing the variable if so, NULL if not
 930  */
 931 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 932                                        struct hist_trigger_data *var_data,
 933                                        unsigned int var_idx)
 934 {
 935         struct hist_field *hist_field;
 936         unsigned int i;
 937
 938         for (i = 0; i < hist_data->n_var_refs; i++) {
 939                 hist_field = hist_data->var_refs[i];
 940                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 941                         return hist_field;
 942         }
 943
 944         return NULL;
 945 }
 946
 947 /**
 948  * find_any_var_ref - Check if there is a reference to a given trigger variable
 949  * @hist_data: The hist trigger
 950  * @var_idx: The trigger variable identifier
 951  *
 952  * Check to see whether the given variable is currently referenced by
 953  * any other trigger.
 954  *
 955  * The trigger the variable is defined on is explicitly excluded - the
 956  * assumption being that a self-reference doesn't prevent a trigger
 957  * from being removed.
 958  *
 959  * Return: The VAR_REF field referencing the variable if so, NULL if not
 960  */
 961 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 962                                            unsigned int var_idx)
 963 {
 964         struct trace_array *tr = hist_data->event_file->tr;
 965         struct hist_field *found = NULL;
 966         struct hist_var_data *var_data;
 967
 968         list_for_each_entry(var_data, &tr->hist_vars, list) {
 969                 if (var_data->hist_data == hist_data)
 970                         continue;
 971                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 972                 if (found)
 973                         break;
 974         }
 975
 976         return found;
 977 }
 978
 979 /**
 980  * check_var_refs - Check if there is a reference to any of trigger's variables
 981  * @hist_data: The hist trigger
 982  *
 983  * A trigger can define one or more variables.  If any one of them is
 984  * currently referenced by any other trigger, this function will
 985  * determine that.
 986  *
 987  * Typically used to determine whether or not a trigger can be removed
 988  * - if there are any references to a trigger's variables, it cannot.
 989  *
 990  * Return: True if there is a reference to any of trigger's variables
 991  */
 992 static bool check_var_refs(struct hist_trigger_data *hist_data)
 993 {
 994         struct hist_field *field;
 995         bool found = false;
 996         int i;
 997
 998         for_each_hist_field(i, hist_data) {
 999                 field = hist_data->fields[i];
1000                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1001                         if (find_any_var_ref(hist_data, field->var.idx)) {
1002                                 found = true;
1003                                 break;
1004                         }
1005                 }
1006         }
1007
1008         return found;
1009 }
1010
1011 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1012 {
1013         struct trace_array *tr = hist_data->event_file->tr;
1014         struct hist_var_data *var_data, *found = NULL;
1015
1016         list_for_each_entry(var_data, &tr->hist_vars, list) {
1017                 if (var_data->hist_data == hist_data) {
1018                         found = var_data;
1019                         break;
1020                 }
1021         }
1022
1023         return found;
1024 }
1025
1026 static bool field_has_hist_vars(struct hist_field *hist_field,
1027                                 unsigned int level)
1028 {
1029         int i;
1030
1031         if (level > 3)
1032                 return false;
1033
1034         if (!hist_field)
1035                 return false;
1036
1037         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1038             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1039                 return true;
1040
1041         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1042                 struct hist_field *operand;
1043
1044                 operand = hist_field->operands[i];
1045                 if (field_has_hist_vars(operand, level + 1))
1046                         return true;
1047         }
1048
1049         return false;
1050 }
1051
1052 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1053 {
1054         struct hist_field *hist_field;
1055         int i;
1056
1057         for_each_hist_field(i, hist_data) {
1058                 hist_field = hist_data->fields[i];
1059                 if (field_has_hist_vars(hist_field, 0))
1060                         return true;
1061         }
1062
1063         return false;
1064 }
1065
1066 static int save_hist_vars(struct hist_trigger_data *hist_data)
1067 {
1068         struct trace_array *tr = hist_data->event_file->tr;
1069         struct hist_var_data *var_data;
1070
1071         var_data = find_hist_vars(hist_data);
1072         if (var_data)
1073                 return 0;
1074
1075         if (tracing_check_open_get_tr(tr))
1076                 return -ENODEV;
1077
1078         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1079         if (!var_data) {
1080                 trace_array_put(tr);
1081                 return -ENOMEM;
1082         }
1083
1084         var_data->hist_data = hist_data;
1085         list_add(&var_data->list, &tr->hist_vars);
1086
1087         return 0;
1088 }
1089
1090 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1091 {
1092         struct trace_array *tr = hist_data->event_file->tr;
1093         struct hist_var_data *var_data;
1094
1095         var_data = find_hist_vars(hist_data);
1096         if (!var_data)
1097                 return;
1098
1099         if (WARN_ON(check_var_refs(hist_data)))
1100                 return;
1101
1102         list_del(&var_data->list);
1103
1104         kfree(var_data);
1105
1106         trace_array_put(tr);
1107 }
1108
1109 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1110                                          const char *var_name)
1111 {
1112         struct hist_field *hist_field, *found = NULL;
1113         int i;
1114
1115         for_each_hist_field(i, hist_data) {
1116                 hist_field = hist_data->fields[i];
1117                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1118                     strcmp(hist_field->var.name, var_name) == 0) {
1119                         found = hist_field;
1120                         break;
1121                 }
1122         }
1123
1124         return found;
1125 }
1126
1127 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1128                                    struct trace_event_file *file,
1129                                    const char *var_name)
1130 {
1131         struct hist_trigger_data *test_data;
1132         struct event_trigger_data *test;
1133         struct hist_field *hist_field;
1134
1135         lockdep_assert_held(&event_mutex);
1136
1137         hist_field = find_var_field(hist_data, var_name);
1138         if (hist_field)
1139                 return hist_field;
1140
1141         list_for_each_entry(test, &file->triggers, list) {
1142                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1143                         test_data = test->private_data;
1144                         hist_field = find_var_field(test_data, var_name);
1145                         if (hist_field)
1146                                 return hist_field;
1147                 }
1148         }
1149
1150         return NULL;
1151 }
1152
1153 static struct trace_event_file *find_var_file(struct trace_array *tr,
1154                                               char *system,
1155                                               char *event_name,
1156                                               char *var_name)
1157 {
1158         struct hist_trigger_data *var_hist_data;
1159         struct hist_var_data *var_data;
1160         struct trace_event_file *file, *found = NULL;
1161
1162         if (system)
1163                 return find_event_file(tr, system, event_name);
1164
1165         list_for_each_entry(var_data, &tr->hist_vars, list) {
1166                 var_hist_data = var_data->hist_data;
1167                 file = var_hist_data->event_file;
1168                 if (file == found)
1169                         continue;
1170
1171                 if (find_var_field(var_hist_data, var_name)) {
1172                         if (found) {
1173                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1174                                 return NULL;
1175                         }
1176
1177                         found = file;
1178                 }
1179         }
1180
1181         return found;
1182 }
1183
1184 static struct hist_field *find_file_var(struct trace_event_file *file,
1185                                         const char *var_name)
1186 {
1187         struct hist_trigger_data *test_data;
1188         struct event_trigger_data *test;
1189         struct hist_field *hist_field;
1190
1191         lockdep_assert_held(&event_mutex);
1192
1193         list_for_each_entry(test, &file->triggers, list) {
1194                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1195                         test_data = test->private_data;
1196                         hist_field = find_var_field(test_data, var_name);
1197                         if (hist_field)
1198                                 return hist_field;
1199                 }
1200         }
1201
1202         return NULL;
1203 }
1204
1205 static struct hist_field *
1206 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1207 {
1208         struct trace_array *tr = hist_data->event_file->tr;
1209         struct hist_field *hist_field, *found = NULL;
1210         struct trace_event_file *file;
1211         unsigned int i;
1212
1213         for (i = 0; i < hist_data->n_actions; i++) {
1214                 struct action_data *data = hist_data->actions[i];
1215
1216                 if (data->handler == HANDLER_ONMATCH) {
1217                         char *system = data->match_data.event_system;
1218                         char *event_name = data->match_data.event;
1219
1220                         file = find_var_file(tr, system, event_name, var_name);
1221                         if (!file)
1222                                 continue;
1223                         hist_field = find_file_var(file, var_name);
1224                         if (hist_field) {
1225                                 if (found) {
1226                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1227                                                  errpos(var_name));
1228                                         return ERR_PTR(-EINVAL);
1229                                 }
1230
1231                                 found = hist_field;
1232                         }
1233                 }
1234         }
1235         return found;
1236 }
1237
1238 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1239                                          char *system,
1240                                          char *event_name,
1241                                          char *var_name)
1242 {
1243         struct trace_array *tr = hist_data->event_file->tr;
1244         struct hist_field *hist_field = NULL;
1245         struct trace_event_file *file;
1246
1247         if (!system || !event_name) {
1248                 hist_field = find_match_var(hist_data, var_name);
1249                 if (IS_ERR(hist_field))
1250                         return NULL;
1251                 if (hist_field)
1252                         return hist_field;
1253         }
1254
1255         file = find_var_file(tr, system, event_name, var_name);
1256         if (!file)
1257                 return NULL;
1258
1259         hist_field = find_file_var(file, var_name);
1260
1261         return hist_field;
1262 }
1263
1264 static u64 hist_field_var_ref(struct hist_field *hist_field,
1265                               struct tracing_map_elt *elt,
1266                               struct trace_buffer *buffer,
1267                               struct ring_buffer_event *rbe,
1268                               void *event)
1269 {
1270         struct hist_elt_data *elt_data;
1271         u64 var_val = 0;
1272
1273         if (WARN_ON_ONCE(!elt))
1274                 return var_val;
1275
1276         elt_data = elt->private_data;
1277         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1278
1279         return var_val;
1280 }
1281
1282 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1283                              u64 *var_ref_vals, bool self)
1284 {
1285         struct hist_trigger_data *var_data;
1286         struct tracing_map_elt *var_elt;
1287         struct hist_field *hist_field;
1288         unsigned int i, var_idx;
1289         bool resolved = true;
1290         u64 var_val = 0;
1291
1292         for (i = 0; i < hist_data->n_var_refs; i++) {
1293                 hist_field = hist_data->var_refs[i];
1294                 var_idx = hist_field->var.idx;
1295                 var_data = hist_field->var.hist_data;
1296
1297                 if (var_data == NULL) {
1298                         resolved = false;
1299                         break;
1300                 }
1301
1302                 if ((self && var_data != hist_data) ||
1303                     (!self && var_data == hist_data))
1304                         continue;
1305
1306                 var_elt = tracing_map_lookup(var_data->map, key);
1307                 if (!var_elt) {
1308                         resolved = false;
1309                         break;
1310                 }
1311
1312                 if (!tracing_map_var_set(var_elt, var_idx)) {
1313                         resolved = false;
1314                         break;
1315                 }
1316
1317                 if (self || !hist_field->read_once)
1318                         var_val = tracing_map_read_var(var_elt, var_idx);
1319                 else
1320                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1321
1322                 var_ref_vals[i] = var_val;
1323         }
1324
1325         return resolved;
1326 }
1327
1328 static const char *hist_field_name(struct hist_field *field,
1329                                    unsigned int level)
1330 {
1331         const char *field_name = "";
1332
1333         if (level > 1)
1334                 return field_name;
1335
1336         if (field->field)
1337                 field_name = field->field->name;
1338         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1339                  field->flags & HIST_FIELD_FL_ALIAS ||
1340                  field->flags & HIST_FIELD_FL_BUCKET)
1341                 field_name = hist_field_name(field->operands[0], ++level);
1342         else if (field->flags & HIST_FIELD_FL_CPU)
1343                 field_name = "common_cpu";
1344         else if (field->flags & HIST_FIELD_FL_EXPR ||
1345                  field->flags & HIST_FIELD_FL_VAR_REF) {
1346                 if (field->system) {
1347                         static char full_name[MAX_FILTER_STR_VAL];
1348
1349                         strcat(full_name, field->system);
1350                         strcat(full_name, ".");
1351                         strcat(full_name, field->event_name);
1352                         strcat(full_name, ".");
1353                         strcat(full_name, field->name);
1354                         field_name = full_name;
1355                 } else
1356                         field_name = field->name;
1357         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1358                 field_name = "common_timestamp";
1359         else if (field->flags & HIST_FIELD_FL_STACKTRACE)
1360                 field_name = "stacktrace";
1361         else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1362                 field_name = "hitcount";
1363
1364         if (field_name == NULL)
1365                 field_name = "";
1366
1367         return field_name;
1368 }
1369
1370 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1371 {
1372         switch (field_size) {
1373         case 8:
1374                 if (field_is_signed)
1375                         return HIST_FIELD_FN_S64;
1376                 else
1377                         return HIST_FIELD_FN_U64;
1378         case 4:
1379                 if (field_is_signed)
1380                         return HIST_FIELD_FN_S32;
1381                 else
1382                         return HIST_FIELD_FN_U32;
1383         case 2:
1384                 if (field_is_signed)
1385                         return HIST_FIELD_FN_S16;
1386                 else
1387                         return HIST_FIELD_FN_U16;
1388         case 1:
1389                 if (field_is_signed)
1390                         return HIST_FIELD_FN_S8;
1391                 else
1392                         return HIST_FIELD_FN_U8;
1393         }
1394
1395         return HIST_FIELD_FN_NOP;
1396 }
1397
1398 static int parse_map_size(char *str)
1399 {
1400         unsigned long size, map_bits;
1401         int ret;
1402
1403         ret = kstrtoul(str, 0, &size);
1404         if (ret)
1405                 goto out;
1406
1407         map_bits = ilog2(roundup_pow_of_two(size));
1408         if (map_bits < TRACING_MAP_BITS_MIN ||
1409             map_bits > TRACING_MAP_BITS_MAX)
1410                 ret = -EINVAL;
1411         else
1412                 ret = map_bits;
1413  out:
1414         return ret;
1415 }
1416
1417 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1418 {
1419         unsigned int i;
1420
1421         if (!attrs)
1422                 return;
1423
1424         for (i = 0; i < attrs->n_assignments; i++)
1425                 kfree(attrs->assignment_str[i]);
1426
1427         for (i = 0; i < attrs->n_actions; i++)
1428                 kfree(attrs->action_str[i]);
1429
1430         kfree(attrs->name);
1431         kfree(attrs->sort_key_str);
1432         kfree(attrs->keys_str);
1433         kfree(attrs->vals_str);
1434         kfree(attrs->clock);
1435         kfree(attrs);
1436 }
1437
1438 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1439 {
1440         int ret = -EINVAL;
1441
1442         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1443                 return ret;
1444
1445         if ((str_has_prefix(str, "onmatch(")) ||
1446             (str_has_prefix(str, "onmax(")) ||
1447             (str_has_prefix(str, "onchange("))) {
1448                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1449                 if (!attrs->action_str[attrs->n_actions]) {
1450                         ret = -ENOMEM;
1451                         return ret;
1452                 }
1453                 attrs->n_actions++;
1454                 ret = 0;
1455         }
1456         return ret;
1457 }
1458
1459 static int parse_assignment(struct trace_array *tr,
1460                             char *str, struct hist_trigger_attrs *attrs)
1461 {
1462         int len, ret = 0;
1463
1464         if ((len = str_has_prefix(str, "key=")) ||
1465             (len = str_has_prefix(str, "keys="))) {
1466                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1467                 if (!attrs->keys_str) {
1468                         ret = -ENOMEM;
1469                         goto out;
1470                 }
1471         } else if ((len = str_has_prefix(str, "val=")) ||
1472                    (len = str_has_prefix(str, "vals=")) ||
1473                    (len = str_has_prefix(str, "values="))) {
1474                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1475                 if (!attrs->vals_str) {
1476                         ret = -ENOMEM;
1477                         goto out;
1478                 }
1479         } else if ((len = str_has_prefix(str, "sort="))) {
1480                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1481                 if (!attrs->sort_key_str) {
1482                         ret = -ENOMEM;
1483                         goto out;
1484                 }
1485         } else if (str_has_prefix(str, "name=")) {
1486                 attrs->name = kstrdup(str, GFP_KERNEL);
1487                 if (!attrs->name) {
1488                         ret = -ENOMEM;
1489                         goto out;
1490                 }
1491         } else if ((len = str_has_prefix(str, "clock="))) {
1492                 str += len;
1493
1494                 str = strstrip(str);
1495                 attrs->clock = kstrdup(str, GFP_KERNEL);
1496                 if (!attrs->clock) {
1497                         ret = -ENOMEM;
1498                         goto out;
1499                 }
1500         } else if ((len = str_has_prefix(str, "size="))) {
1501                 int map_bits = parse_map_size(str + len);
1502
1503                 if (map_bits < 0) {
1504                         ret = map_bits;
1505                         goto out;
1506                 }
1507                 attrs->map_bits = map_bits;
1508         } else {
1509                 char *assignment;
1510
1511                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1512                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1513                         ret = -EINVAL;
1514                         goto out;
1515                 }
1516
1517                 assignment = kstrdup(str, GFP_KERNEL);
1518                 if (!assignment) {
1519                         ret = -ENOMEM;
1520                         goto out;
1521                 }
1522
1523                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1524         }
1525  out:
1526         return ret;
1527 }
1528
1529 static struct hist_trigger_attrs *
1530 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1531 {
1532         struct hist_trigger_attrs *attrs;
1533         int ret = 0;
1534
1535         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1536         if (!attrs)
1537                 return ERR_PTR(-ENOMEM);
1538
1539         while (trigger_str) {
1540                 char *str = strsep(&trigger_str, ":");
1541                 char *rhs;
1542
1543                 rhs = strchr(str, '=');
1544                 if (rhs) {
1545                         if (!strlen(++rhs)) {
1546                                 ret = -EINVAL;
1547                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1548                                 goto free;
1549                         }
1550                         ret = parse_assignment(tr, str, attrs);
1551                         if (ret)
1552                                 goto free;
1553                 } else if (strcmp(str, "nohitcount") == 0 ||
1554                            strcmp(str, "NOHC") == 0)
1555                         attrs->no_hitcount = true;
1556                 else if (strcmp(str, "pause") == 0)
1557                         attrs->pause = true;
1558                 else if ((strcmp(str, "cont") == 0) ||
1559                          (strcmp(str, "continue") == 0))
1560                         attrs->cont = true;
1561                 else if (strcmp(str, "clear") == 0)
1562                         attrs->clear = true;
1563                 else {
1564                         ret = parse_action(str, attrs);
1565                         if (ret)
1566                                 goto free;
1567                 }
1568         }
1569
1570         if (!attrs->keys_str) {
1571                 ret = -EINVAL;
1572                 goto free;
1573         }
1574
1575         if (!attrs->clock) {
1576                 attrs->clock = kstrdup("global", GFP_KERNEL);
1577                 if (!attrs->clock) {
1578                         ret = -ENOMEM;
1579                         goto free;
1580                 }
1581         }
1582
1583         return attrs;
1584  free:
1585         destroy_hist_trigger_attrs(attrs);
1586
1587         return ERR_PTR(ret);
1588 }
1589
1590 static inline void save_comm(char *comm, struct task_struct *task)
1591 {
1592         if (!task->pid) {
1593                 strcpy(comm, "<idle>");
1594                 return;
1595         }
1596
1597         if (WARN_ON_ONCE(task->pid < 0)) {
1598                 strcpy(comm, "<XXX>");
1599                 return;
1600         }
1601
1602         strncpy(comm, task->comm, TASK_COMM_LEN);
1603 }
1604
1605 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1606 {
1607         unsigned int i;
1608
1609         for (i = 0; i < elt_data->n_field_var_str; i++)
1610                 kfree(elt_data->field_var_str[i]);
1611
1612         kfree(elt_data->field_var_str);
1613
1614         kfree(elt_data->comm);
1615         kfree(elt_data);
1616 }
1617
1618 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1619 {
1620         struct hist_elt_data *elt_data = elt->private_data;
1621
1622         hist_elt_data_free(elt_data);
1623 }
1624
1625 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1626 {
1627         struct hist_trigger_data *hist_data = elt->map->private_data;
1628         unsigned int size = TASK_COMM_LEN;
1629         struct hist_elt_data *elt_data;
1630         struct hist_field *hist_field;
1631         unsigned int i, n_str;
1632
1633         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1634         if (!elt_data)
1635                 return -ENOMEM;
1636
1637         for_each_hist_field(i, hist_data) {
1638                 hist_field = hist_data->fields[i];
1639
1640                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1641                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1642                         if (!elt_data->comm) {
1643                                 kfree(elt_data);
1644                                 return -ENOMEM;
1645                         }
1646                         break;
1647                 }
1648         }
1649
1650         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1651                 hist_data->n_var_str;
1652         if (n_str > SYNTH_FIELDS_MAX) {
1653                 hist_elt_data_free(elt_data);
1654                 return -EINVAL;
1655         }
1656
1657         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1658
1659         size = STR_VAR_LEN_MAX;
1660
1661         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1662         if (!elt_data->field_var_str) {
1663                 hist_elt_data_free(elt_data);
1664                 return -EINVAL;
1665         }
1666         elt_data->n_field_var_str = n_str;
1667
1668         for (i = 0; i < n_str; i++) {
1669                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1670                 if (!elt_data->field_var_str[i]) {
1671                         hist_elt_data_free(elt_data);
1672                         return -ENOMEM;
1673                 }
1674         }
1675
1676         elt->private_data = elt_data;
1677
1678         return 0;
1679 }
1680
1681 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1682 {
1683         struct hist_elt_data *elt_data = elt->private_data;
1684
1685         if (elt_data->comm)
1686                 save_comm(elt_data->comm, current);
1687 }
1688
1689 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1690         .elt_alloc      = hist_trigger_elt_data_alloc,
1691         .elt_free       = hist_trigger_elt_data_free,
1692         .elt_init       = hist_trigger_elt_data_init,
1693 };
1694
1695 static const char *get_hist_field_flags(struct hist_field *hist_field)
1696 {
1697         const char *flags_str = NULL;
1698
1699         if (hist_field->flags & HIST_FIELD_FL_HEX)
1700                 flags_str = "hex";
1701         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1702                 flags_str = "sym";
1703         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1704                 flags_str = "sym-offset";
1705         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1706                 flags_str = "execname";
1707         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1708                 flags_str = "syscall";
1709         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1710                 flags_str = "log2";
1711         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1712                 flags_str = "buckets";
1713         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1714                 flags_str = "usecs";
1715         else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1716                 flags_str = "percent";
1717         else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1718                 flags_str = "graph";
1719         else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1720                 flags_str = "stacktrace";
1721
1722         return flags_str;
1723 }
1724
1725 static void expr_field_str(struct hist_field *field, char *expr)
1726 {
1727         if (field->flags & HIST_FIELD_FL_VAR_REF)
1728                 strcat(expr, "$");
1729         else if (field->flags & HIST_FIELD_FL_CONST) {
1730                 char str[HIST_CONST_DIGITS_MAX];
1731
1732                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1733                 strcat(expr, str);
1734         }
1735
1736         strcat(expr, hist_field_name(field, 0));
1737
1738         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1739                 const char *flags_str = get_hist_field_flags(field);
1740
1741                 if (flags_str) {
1742                         strcat(expr, ".");
1743                         strcat(expr, flags_str);
1744                 }
1745         }
1746 }
1747
1748 static char *expr_str(struct hist_field *field, unsigned int level)
1749 {
1750         char *expr;
1751
1752         if (level > 1)
1753                 return NULL;
1754
1755         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1756         if (!expr)
1757                 return NULL;
1758
1759         if (!field->operands[0]) {
1760                 expr_field_str(field, expr);
1761                 return expr;
1762         }
1763
1764         if (field->operator == FIELD_OP_UNARY_MINUS) {
1765                 char *subexpr;
1766
1767                 strcat(expr, "-(");
1768                 subexpr = expr_str(field->operands[0], ++level);
1769                 if (!subexpr) {
1770                         kfree(expr);
1771                         return NULL;
1772                 }
1773                 strcat(expr, subexpr);
1774                 strcat(expr, ")");
1775
1776                 kfree(subexpr);
1777
1778                 return expr;
1779         }
1780
1781         expr_field_str(field->operands[0], expr);
1782
1783         switch (field->operator) {
1784         case FIELD_OP_MINUS:
1785                 strcat(expr, "-");
1786                 break;
1787         case FIELD_OP_PLUS:
1788                 strcat(expr, "+");
1789                 break;
1790         case FIELD_OP_DIV:
1791                 strcat(expr, "/");
1792                 break;
1793         case FIELD_OP_MULT:
1794                 strcat(expr, "*");
1795                 break;
1796         default:
1797                 kfree(expr);
1798                 return NULL;
1799         }
1800
1801         expr_field_str(field->operands[1], expr);
1802
1803         return expr;
1804 }
1805
1806 /*
1807  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1808  * of the expression tree to be evaluated.
1809  */
1810 static int contains_operator(char *str, char **sep)
1811 {
1812         enum field_op_id field_op = FIELD_OP_NONE;
1813         char *minus_op, *plus_op, *div_op, *mult_op;
1814
1815
1816         /*
1817          * Report the last occurrence of the operators first, so that the
1818          * expression is evaluated left to right. This is important since
1819          * subtraction and division are not associative.
1820          *
1821          *      e.g
1822          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1823          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1824          */
1825
1826         /*
1827          * First, find lower precedence addition and subtraction
1828          * since the expression will be evaluated recursively.
1829          */
1830         minus_op = strrchr(str, '-');
1831         if (minus_op) {
1832                 /*
1833                  * Unary minus is not supported in sub-expressions. If
1834                  * present, it is always the next root operator.
1835                  */
1836                 if (minus_op == str) {
1837                         field_op = FIELD_OP_UNARY_MINUS;
1838                         goto out;
1839                 }
1840
1841                 field_op = FIELD_OP_MINUS;
1842         }
1843
1844         plus_op = strrchr(str, '+');
1845         if (plus_op || minus_op) {
1846                 /*
1847                  * For operators of the same precedence use to rightmost as the
1848                  * root, so that the expression is evaluated left to right.
1849                  */
1850                 if (plus_op > minus_op)
1851                         field_op = FIELD_OP_PLUS;
1852                 goto out;
1853         }
1854
1855         /*
1856          * Multiplication and division have higher precedence than addition and
1857          * subtraction.
1858          */
1859         div_op = strrchr(str, '/');
1860         if (div_op)
1861                 field_op = FIELD_OP_DIV;
1862
1863         mult_op = strrchr(str, '*');
1864         /*
1865          * For operators of the same precedence use to rightmost as the
1866          * root, so that the expression is evaluated left to right.
1867          */
1868         if (mult_op > div_op)
1869                 field_op = FIELD_OP_MULT;
1870
1871 out:
1872         if (sep) {
1873                 switch (field_op) {
1874                 case FIELD_OP_UNARY_MINUS:
1875                 case FIELD_OP_MINUS:
1876                         *sep = minus_op;
1877                         break;
1878                 case FIELD_OP_PLUS:
1879                         *sep = plus_op;
1880                         break;
1881                 case FIELD_OP_DIV:
1882                         *sep = div_op;
1883                         break;
1884                 case FIELD_OP_MULT:
1885                         *sep = mult_op;
1886                         break;
1887                 case FIELD_OP_NONE:
1888                 default:
1889                         *sep = NULL;
1890                         break;
1891                 }
1892         }
1893
1894         return field_op;
1895 }
1896
1897 static void get_hist_field(struct hist_field *hist_field)
1898 {
1899         hist_field->ref++;
1900 }
1901
1902 static void __destroy_hist_field(struct hist_field *hist_field)
1903 {
1904         if (--hist_field->ref > 1)
1905                 return;
1906
1907         kfree(hist_field->var.name);
1908         kfree(hist_field->name);
1909
1910         /* Can likely be a const */
1911         kfree_const(hist_field->type);
1912
1913         kfree(hist_field->system);
1914         kfree(hist_field->event_name);
1915
1916         kfree(hist_field);
1917 }
1918
1919 static void destroy_hist_field(struct hist_field *hist_field,
1920                                unsigned int level)
1921 {
1922         unsigned int i;
1923
1924         if (level > 3)
1925                 return;
1926
1927         if (!hist_field)
1928                 return;
1929
1930         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1931                 return; /* var refs will be destroyed separately */
1932
1933         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1934                 destroy_hist_field(hist_field->operands[i], level + 1);
1935
1936         __destroy_hist_field(hist_field);
1937 }
1938
1939 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1940                                             struct ftrace_event_field *field,
1941                                             unsigned long flags,
1942                                             char *var_name)
1943 {
1944         struct hist_field *hist_field;
1945
1946         if (field && is_function_field(field))
1947                 return NULL;
1948
1949         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1950         if (!hist_field)
1951                 return NULL;
1952
1953         hist_field->ref = 1;
1954
1955         hist_field->hist_data = hist_data;
1956
1957         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1958                 goto out; /* caller will populate */
1959
1960         if (flags & HIST_FIELD_FL_VAR_REF) {
1961                 hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1962                 goto out;
1963         }
1964
1965         if (flags & HIST_FIELD_FL_HITCOUNT) {
1966                 hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1967                 hist_field->size = sizeof(u64);
1968                 hist_field->type = "u64";
1969                 goto out;
1970         }
1971
1972         if (flags & HIST_FIELD_FL_CONST) {
1973                 hist_field->fn_num = HIST_FIELD_FN_CONST;
1974                 hist_field->size = sizeof(u64);
1975                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1976                 if (!hist_field->type)
1977                         goto free;
1978                 goto out;
1979         }
1980
1981         if (flags & HIST_FIELD_FL_STACKTRACE) {
1982                 hist_field->fn_num = HIST_FIELD_FN_NOP;
1983                 hist_field->size = HIST_STACKTRACE_SIZE;
1984                 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1985                 if (!hist_field->type)
1986                         goto free;
1987                 goto out;
1988         }
1989
1990         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1991                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1992                 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1993                         HIST_FIELD_FN_BUCKET;
1994                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1995                 if (!hist_field->operands[0])
1996                         goto free;
1997                 hist_field->size = hist_field->operands[0]->size;
1998                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1999                 if (!hist_field->type)
2000                         goto free;
2001                 goto out;
2002         }
2003
2004         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2005                 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2006                 hist_field->size = sizeof(u64);
2007                 hist_field->type = "u64";
2008                 goto out;
2009         }
2010
2011         if (flags & HIST_FIELD_FL_CPU) {
2012                 hist_field->fn_num = HIST_FIELD_FN_CPU;
2013                 hist_field->size = sizeof(int);
2014                 hist_field->type = "unsigned int";
2015                 goto out;
2016         }
2017
2018         if (WARN_ON_ONCE(!field))
2019                 goto out;
2020
2021         /* Pointers to strings are just pointers and dangerous to dereference */
2022         if (is_string_field(field) &&
2023             (field->filter_type != FILTER_PTR_STRING)) {
2024                 flags |= HIST_FIELD_FL_STRING;
2025
2026                 hist_field->size = MAX_FILTER_STR_VAL;
2027                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2028                 if (!hist_field->type)
2029                         goto free;
2030
2031                 if (field->filter_type == FILTER_STATIC_STRING) {
2032                         hist_field->fn_num = HIST_FIELD_FN_STRING;
2033                         hist_field->size = field->size;
2034                 } else if (field->filter_type == FILTER_DYN_STRING) {
2035                         hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2036                 } else if (field->filter_type == FILTER_RDYN_STRING)
2037                         hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2038                 else
2039                         hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2040         } else {
2041                 hist_field->size = field->size;
2042                 hist_field->is_signed = field->is_signed;
2043                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2044                 if (!hist_field->type)
2045                         goto free;
2046
2047                 hist_field->fn_num = select_value_fn(field->size,
2048                                                      field->is_signed);
2049                 if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2050                         destroy_hist_field(hist_field, 0);
2051                         return NULL;
2052                 }
2053         }
2054  out:
2055         hist_field->field = field;
2056         hist_field->flags = flags;
2057
2058         if (var_name) {
2059                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2060                 if (!hist_field->var.name)
2061                         goto free;
2062         }
2063
2064         return hist_field;
2065  free:
2066         destroy_hist_field(hist_field, 0);
2067         return NULL;
2068 }
2069
2070 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2071 {
2072         unsigned int i;
2073
2074         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2075                 if (hist_data->fields[i]) {
2076                         destroy_hist_field(hist_data->fields[i], 0);
2077                         hist_data->fields[i] = NULL;
2078                 }
2079         }
2080
2081         for (i = 0; i < hist_data->n_var_refs; i++) {
2082                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2083                 __destroy_hist_field(hist_data->var_refs[i]);
2084                 hist_data->var_refs[i] = NULL;
2085         }
2086 }
2087
2088 static int init_var_ref(struct hist_field *ref_field,
2089                         struct hist_field *var_field,
2090                         char *system, char *event_name)
2091 {
2092         int err = 0;
2093
2094         ref_field->var.idx = var_field->var.idx;
2095         ref_field->var.hist_data = var_field->hist_data;
2096         ref_field->size = var_field->size;
2097         ref_field->is_signed = var_field->is_signed;
2098         ref_field->flags |= var_field->flags &
2099                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2100
2101         if (system) {
2102                 ref_field->system = kstrdup(system, GFP_KERNEL);
2103                 if (!ref_field->system)
2104                         return -ENOMEM;
2105         }
2106
2107         if (event_name) {
2108                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2109                 if (!ref_field->event_name) {
2110                         err = -ENOMEM;
2111                         goto free;
2112                 }
2113         }
2114
2115         if (var_field->var.name) {
2116                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2117                 if (!ref_field->name) {
2118                         err = -ENOMEM;
2119                         goto free;
2120                 }
2121         } else if (var_field->name) {
2122                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2123                 if (!ref_field->name) {
2124                         err = -ENOMEM;
2125                         goto free;
2126                 }
2127         }
2128
2129         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2130         if (!ref_field->type) {
2131                 err = -ENOMEM;
2132                 goto free;
2133         }
2134  out:
2135         return err;
2136  free:
2137         kfree(ref_field->system);
2138         ref_field->system = NULL;
2139         kfree(ref_field->event_name);
2140         ref_field->event_name = NULL;
2141         kfree(ref_field->name);
2142         ref_field->name = NULL;
2143
2144         goto out;
2145 }
2146
2147 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2148                             struct hist_field *var_field)
2149 {
2150         struct hist_field *ref_field;
2151         int i;
2152
2153         for (i = 0; i < hist_data->n_var_refs; i++) {
2154                 ref_field = hist_data->var_refs[i];
2155                 if (ref_field->var.idx == var_field->var.idx &&
2156                     ref_field->var.hist_data == var_field->hist_data)
2157                         return i;
2158         }
2159
2160         return -ENOENT;
2161 }
2162
2163 /**
2164  * create_var_ref - Create a variable reference and attach it to trigger
2165  * @hist_data: The trigger that will be referencing the variable
2166  * @var_field: The VAR field to create a reference to
2167  * @system: The optional system string
2168  * @event_name: The optional event_name string
2169  *
2170  * Given a variable hist_field, create a VAR_REF hist_field that
2171  * represents a reference to it.
2172  *
2173  * This function also adds the reference to the trigger that
2174  * now references the variable.
2175  *
2176  * Return: The VAR_REF field if successful, NULL if not
2177  */
2178 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2179                                          struct hist_field *var_field,
2180                                          char *system, char *event_name)
2181 {
2182         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2183         struct hist_field *ref_field;
2184         int i;
2185
2186         /* Check if the variable already exists */
2187         for (i = 0; i < hist_data->n_var_refs; i++) {
2188                 ref_field = hist_data->var_refs[i];
2189                 if (ref_field->var.idx == var_field->var.idx &&
2190                     ref_field->var.hist_data == var_field->hist_data) {
2191                         get_hist_field(ref_field);
2192                         return ref_field;
2193                 }
2194         }
2195         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2196         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2197                 return NULL;
2198         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2199         if (ref_field) {
2200                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2201                         destroy_hist_field(ref_field, 0);
2202                         return NULL;
2203                 }
2204
2205                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2206                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2207         }
2208
2209         return ref_field;
2210 }
2211
2212 static bool is_var_ref(char *var_name)
2213 {
2214         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2215                 return false;
2216
2217         return true;
2218 }
2219
2220 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2221                                  char *var_name)
2222 {
2223         char *name, *field;
2224         unsigned int i;
2225
2226         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2227                 name = hist_data->attrs->var_defs.name[i];
2228
2229                 if (strcmp(var_name, name) == 0) {
2230                         field = hist_data->attrs->var_defs.expr[i];
2231                         if (contains_operator(field, NULL) || is_var_ref(field))
2232                                 continue;
2233                         return field;
2234                 }
2235         }
2236
2237         return NULL;
2238 }
2239
2240 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2241                                  char *system, char *event_name,
2242                                  char *var_name)
2243 {
2244         struct trace_event_call *call;
2245
2246         if (system && event_name) {
2247                 call = hist_data->event_file->event_call;
2248
2249                 if (strcmp(system, call->class->system) != 0)
2250                         return NULL;
2251
2252                 if (strcmp(event_name, trace_event_name(call)) != 0)
2253                         return NULL;
2254         }
2255
2256         if (!!system != !!event_name)
2257                 return NULL;
2258
2259         if (!is_var_ref(var_name))
2260                 return NULL;
2261
2262         var_name++;
2263
2264         return field_name_from_var(hist_data, var_name);
2265 }
2266
2267 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2268                                         char *system, char *event_name,
2269                                         char *var_name)
2270 {
2271         struct hist_field *var_field = NULL, *ref_field = NULL;
2272         struct trace_array *tr = hist_data->event_file->tr;
2273
2274         if (!is_var_ref(var_name))
2275                 return NULL;
2276
2277         var_name++;
2278
2279         var_field = find_event_var(hist_data, system, event_name, var_name);
2280         if (var_field)
2281                 ref_field = create_var_ref(hist_data, var_field,
2282                                            system, event_name);
2283
2284         if (!ref_field)
2285                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2286
2287         return ref_field;
2288 }
2289
2290 static struct ftrace_event_field *
2291 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2292             char *field_str, unsigned long *flags, unsigned long *buckets)
2293 {
2294         struct ftrace_event_field *field = NULL;
2295         char *field_name, *modifier, *str;
2296         struct trace_array *tr = file->tr;
2297
2298         modifier = str = kstrdup(field_str, GFP_KERNEL);
2299         if (!modifier)
2300                 return ERR_PTR(-ENOMEM);
2301
2302         field_name = strsep(&modifier, ".");
2303         if (modifier) {
2304                 if (strcmp(modifier, "hex") == 0)
2305                         *flags |= HIST_FIELD_FL_HEX;
2306                 else if (strcmp(modifier, "sym") == 0)
2307                         *flags |= HIST_FIELD_FL_SYM;
2308                 /*
2309                  * 'sym-offset' occurrences in the trigger string are modified
2310                  * to 'symXoffset' to simplify arithmetic expression parsing.
2311                  */
2312                 else if (strcmp(modifier, "symXoffset") == 0)
2313                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2314                 else if ((strcmp(modifier, "execname") == 0) &&
2315                          (strcmp(field_name, "common_pid") == 0))
2316                         *flags |= HIST_FIELD_FL_EXECNAME;
2317                 else if (strcmp(modifier, "syscall") == 0)
2318                         *flags |= HIST_FIELD_FL_SYSCALL;
2319                 else if (strcmp(modifier, "stacktrace") == 0)
2320                         *flags |= HIST_FIELD_FL_STACKTRACE;
2321                 else if (strcmp(modifier, "log2") == 0)
2322                         *flags |= HIST_FIELD_FL_LOG2;
2323                 else if (strcmp(modifier, "usecs") == 0)
2324                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2325                 else if (strncmp(modifier, "bucket", 6) == 0) {
2326                         int ret;
2327
2328                         modifier += 6;
2329
2330                         if (*modifier == 's')
2331                                 modifier++;
2332                         if (*modifier != '=')
2333                                 goto error;
2334                         modifier++;
2335                         ret = kstrtoul(modifier, 0, buckets);
2336                         if (ret || !(*buckets))
2337                                 goto error;
2338                         *flags |= HIST_FIELD_FL_BUCKET;
2339                 } else if (strncmp(modifier, "percent", 7) == 0) {
2340                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2341                                 goto error;
2342                         *flags |= HIST_FIELD_FL_PERCENT;
2343                 } else if (strncmp(modifier, "graph", 5) == 0) {
2344                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2345                                 goto error;
2346                         *flags |= HIST_FIELD_FL_GRAPH;
2347                 } else {
2348  error:
2349                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2350                         field = ERR_PTR(-EINVAL);
2351                         goto out;
2352                 }
2353         }
2354
2355         if (strcmp(field_name, "common_timestamp") == 0) {
2356                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2357                 hist_data->enable_timestamps = true;
2358                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2359                         hist_data->attrs->ts_in_usecs = true;
2360         } else if (strcmp(field_name, "stacktrace") == 0) {
2361                 *flags |= HIST_FIELD_FL_STACKTRACE;
2362         } else if (strcmp(field_name, "common_cpu") == 0)
2363                 *flags |= HIST_FIELD_FL_CPU;
2364         else if (strcmp(field_name, "hitcount") == 0)
2365                 *flags |= HIST_FIELD_FL_HITCOUNT;
2366         else {
2367                 field = trace_find_event_field(file->event_call, field_name);
2368                 if (!field || !field->size) {
2369                         /*
2370                          * For backward compatibility, if field_name
2371                          * was "cpu", then we treat this the same as
2372                          * common_cpu. This also works for "CPU".
2373                          */
2374                         if (field && field->filter_type == FILTER_CPU) {
2375                                 *flags |= HIST_FIELD_FL_CPU;
2376                         } else {
2377                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2378                                          errpos(field_name));
2379                                 field = ERR_PTR(-EINVAL);
2380                                 goto out;
2381                         }
2382                 }
2383         }
2384  out:
2385         kfree(str);
2386
2387         return field;
2388 }
2389
2390 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2391                                        struct hist_field *var_ref,
2392                                        char *var_name)
2393 {
2394         struct hist_field *alias = NULL;
2395         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2396
2397         alias = create_hist_field(hist_data, NULL, flags, var_name);
2398         if (!alias)
2399                 return NULL;
2400
2401         alias->fn_num = var_ref->fn_num;
2402         alias->operands[0] = var_ref;
2403
2404         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2405                 destroy_hist_field(alias, 0);
2406                 return NULL;
2407         }
2408
2409         alias->var_ref_idx = var_ref->var_ref_idx;
2410
2411         return alias;
2412 }
2413
2414 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2415                                       char *str, char *var_name,
2416                                       unsigned long *flags)
2417 {
2418         struct trace_array *tr = hist_data->event_file->tr;
2419         struct hist_field *field = NULL;
2420         u64 constant;
2421
2422         if (kstrtoull(str, 0, &constant)) {
2423                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2424                 return NULL;
2425         }
2426
2427         *flags |= HIST_FIELD_FL_CONST;
2428         field = create_hist_field(hist_data, NULL, *flags, var_name);
2429         if (!field)
2430                 return NULL;
2431
2432         field->constant = constant;
2433
2434         return field;
2435 }
2436
2437 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2438                                      struct trace_event_file *file, char *str,
2439                                      unsigned long *flags, char *var_name)
2440 {
2441         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2442         struct ftrace_event_field *field = NULL;
2443         struct hist_field *hist_field = NULL;
2444         unsigned long buckets = 0;
2445         int ret = 0;
2446
2447         if (isdigit(str[0])) {
2448                 hist_field = parse_const(hist_data, str, var_name, flags);
2449                 if (!hist_field) {
2450                         ret = -EINVAL;
2451                         goto out;
2452                 }
2453                 return hist_field;
2454         }
2455
2456         s = strchr(str, '.');
2457         if (s) {
2458                 s = strchr(++s, '.');
2459                 if (s) {
2460                         ref_system = strsep(&str, ".");
2461                         if (!str) {
2462                                 ret = -EINVAL;
2463                                 goto out;
2464                         }
2465                         ref_event = strsep(&str, ".");
2466                         if (!str) {
2467                                 ret = -EINVAL;
2468                                 goto out;
2469                         }
2470                         ref_var = str;
2471                 }
2472         }
2473
2474         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2475         if (!s) {
2476                 hist_field = parse_var_ref(hist_data, ref_system,
2477                                            ref_event, ref_var);
2478                 if (hist_field) {
2479                         if (var_name) {
2480                                 hist_field = create_alias(hist_data, hist_field, var_name);
2481                                 if (!hist_field) {
2482                                         ret = -ENOMEM;
2483                                         goto out;
2484                                 }
2485                         }
2486                         return hist_field;
2487                 }
2488         } else
2489                 str = s;
2490
2491         field = parse_field(hist_data, file, str, flags, &buckets);
2492         if (IS_ERR(field)) {
2493                 ret = PTR_ERR(field);
2494                 goto out;
2495         }
2496
2497         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2498         if (!hist_field) {
2499                 ret = -ENOMEM;
2500                 goto out;
2501         }
2502         hist_field->buckets = buckets;
2503
2504         return hist_field;
2505  out:
2506         return ERR_PTR(ret);
2507 }
2508
2509 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2510                                      struct trace_event_file *file,
2511                                      char *str, unsigned long flags,
2512                                      char *var_name, unsigned int *n_subexprs);
2513
2514 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2515                                       struct trace_event_file *file,
2516                                       char *str, unsigned long flags,
2517                                       char *var_name, unsigned int *n_subexprs)
2518 {
2519         struct hist_field *operand1, *expr = NULL;
2520         unsigned long operand_flags;
2521         int ret = 0;
2522         char *s;
2523
2524         /* Unary minus operator, increment n_subexprs */
2525         ++*n_subexprs;
2526
2527         /* we support only -(xxx) i.e. explicit parens required */
2528
2529         if (*n_subexprs > 3) {
2530                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2531                 ret = -EINVAL;
2532                 goto free;
2533         }
2534
2535         str++; /* skip leading '-' */
2536
2537         s = strchr(str, '(');
2538         if (s)
2539                 str++;
2540         else {
2541                 ret = -EINVAL;
2542                 goto free;
2543         }
2544
2545         s = strrchr(str, ')');
2546         if (s) {
2547                  /* unary minus not supported in sub-expressions */
2548                 if (*(s+1) != '\0') {
2549                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2550                                  errpos(str));
2551                         ret = -EINVAL;
2552                         goto free;
2553                 }
2554                 *s = '\0';
2555         }
2556         else {
2557                 ret = -EINVAL; /* no closing ')' */
2558                 goto free;
2559         }
2560
2561         flags |= HIST_FIELD_FL_EXPR;
2562         expr = create_hist_field(hist_data, NULL, flags, var_name);
2563         if (!expr) {
2564                 ret = -ENOMEM;
2565                 goto free;
2566         }
2567
2568         operand_flags = 0;
2569         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2570         if (IS_ERR(operand1)) {
2571                 ret = PTR_ERR(operand1);
2572                 goto free;
2573         }
2574         if (operand1->flags & HIST_FIELD_FL_STRING) {
2575                 /* String type can not be the operand of unary operator. */
2576                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2577                 destroy_hist_field(operand1, 0);
2578                 ret = -EINVAL;
2579                 goto free;
2580         }
2581
2582         expr->flags |= operand1->flags &
2583                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2584         expr->fn_num = HIST_FIELD_FN_UMINUS;
2585         expr->operands[0] = operand1;
2586         expr->size = operand1->size;
2587         expr->is_signed = operand1->is_signed;
2588         expr->operator = FIELD_OP_UNARY_MINUS;
2589         expr->name = expr_str(expr, 0);
2590         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2591         if (!expr->type) {
2592                 ret = -ENOMEM;
2593                 goto free;
2594         }
2595
2596         return expr;
2597  free:
2598         destroy_hist_field(expr, 0);
2599         return ERR_PTR(ret);
2600 }
2601
2602 /*
2603  * If the operands are var refs, return pointers the
2604  * variable(s) referenced in var1 and var2, else NULL.
2605  */
2606 static int check_expr_operands(struct trace_array *tr,
2607                                struct hist_field *operand1,
2608                                struct hist_field *operand2,
2609                                struct hist_field **var1,
2610                                struct hist_field **var2)
2611 {
2612         unsigned long operand1_flags = operand1->flags;
2613         unsigned long operand2_flags = operand2->flags;
2614
2615         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2616             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2617                 struct hist_field *var;
2618
2619                 var = find_var_field(operand1->var.hist_data, operand1->name);
2620                 if (!var)
2621                         return -EINVAL;
2622                 operand1_flags = var->flags;
2623                 *var1 = var;
2624         }
2625
2626         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2627             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2628                 struct hist_field *var;
2629
2630                 var = find_var_field(operand2->var.hist_data, operand2->name);
2631                 if (!var)
2632                         return -EINVAL;
2633                 operand2_flags = var->flags;
2634                 *var2 = var;
2635         }
2636
2637         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2638             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2639                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2640                 return -EINVAL;
2641         }
2642
2643         return 0;
2644 }
2645
2646 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2647                                      struct trace_event_file *file,
2648                                      char *str, unsigned long flags,
2649                                      char *var_name, unsigned int *n_subexprs)
2650 {
2651         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2652         struct hist_field *var1 = NULL, *var2 = NULL;
2653         unsigned long operand_flags, operand2_flags;
2654         int field_op, ret = -EINVAL;
2655         char *sep, *operand1_str;
2656         enum hist_field_fn op_fn;
2657         bool combine_consts;
2658
2659         if (*n_subexprs > 3) {
2660                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2661                 return ERR_PTR(-EINVAL);
2662         }
2663
2664         field_op = contains_operator(str, &sep);
2665
2666         if (field_op == FIELD_OP_NONE)
2667                 return parse_atom(hist_data, file, str, &flags, var_name);
2668
2669         if (field_op == FIELD_OP_UNARY_MINUS)
2670                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2671
2672         /* Binary operator found, increment n_subexprs */
2673         ++*n_subexprs;
2674
2675         /* Split the expression string at the root operator */
2676         if (!sep)
2677                 return ERR_PTR(-EINVAL);
2678
2679         *sep = '\0';
2680         operand1_str = str;
2681         str = sep+1;
2682
2683         /* Binary operator requires both operands */
2684         if (*operand1_str == '\0' || *str == '\0')
2685                 return ERR_PTR(-EINVAL);
2686
2687         operand_flags = 0;
2688
2689         /* LHS of string is an expression e.g. a+b in a+b+c */
2690         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2691         if (IS_ERR(operand1))
2692                 return ERR_CAST(operand1);
2693
2694         if (operand1->flags & HIST_FIELD_FL_STRING) {
2695                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2696                 ret = -EINVAL;
2697                 goto free_op1;
2698         }
2699
2700         /* RHS of string is another expression e.g. c in a+b+c */
2701         operand_flags = 0;
2702         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2703         if (IS_ERR(operand2)) {
2704                 ret = PTR_ERR(operand2);
2705                 goto free_op1;
2706         }
2707         if (operand2->flags & HIST_FIELD_FL_STRING) {
2708                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2709                 ret = -EINVAL;
2710                 goto free_operands;
2711         }
2712
2713         switch (field_op) {
2714         case FIELD_OP_MINUS:
2715                 op_fn = HIST_FIELD_FN_MINUS;
2716                 break;
2717         case FIELD_OP_PLUS:
2718                 op_fn = HIST_FIELD_FN_PLUS;
2719                 break;
2720         case FIELD_OP_DIV:
2721                 op_fn = HIST_FIELD_FN_DIV;
2722                 break;
2723         case FIELD_OP_MULT:
2724                 op_fn = HIST_FIELD_FN_MULT;
2725                 break;
2726         default:
2727                 ret = -EINVAL;
2728                 goto free_operands;
2729         }
2730
2731         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2732         if (ret)
2733                 goto free_operands;
2734
2735         operand_flags = var1 ? var1->flags : operand1->flags;
2736         operand2_flags = var2 ? var2->flags : operand2->flags;
2737
2738         /*
2739          * If both operands are constant, the expression can be
2740          * collapsed to a single constant.
2741          */
2742         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2743
2744         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2745
2746         flags |= operand1->flags &
2747                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2748
2749         expr = create_hist_field(hist_data, NULL, flags, var_name);
2750         if (!expr) {
2751                 ret = -ENOMEM;
2752                 goto free_operands;
2753         }
2754
2755         operand1->read_once = true;
2756         operand2->read_once = true;
2757
2758         /* The operands are now owned and free'd by 'expr' */
2759         expr->operands[0] = operand1;
2760         expr->operands[1] = operand2;
2761
2762         if (field_op == FIELD_OP_DIV &&
2763                         operand2_flags & HIST_FIELD_FL_CONST) {
2764                 u64 divisor = var2 ? var2->constant : operand2->constant;
2765
2766                 if (!divisor) {
2767                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2768                         ret = -EDOM;
2769                         goto free_expr;
2770                 }
2771
2772                 /*
2773                  * Copy the divisor here so we don't have to look it up
2774                  * later if this is a var ref
2775                  */
2776                 operand2->constant = divisor;
2777                 op_fn = hist_field_get_div_fn(operand2);
2778         }
2779
2780         expr->fn_num = op_fn;
2781
2782         if (combine_consts) {
2783                 if (var1)
2784                         expr->operands[0] = var1;
2785                 if (var2)
2786                         expr->operands[1] = var2;
2787
2788                 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2789                 expr->fn_num = HIST_FIELD_FN_CONST;
2790
2791                 expr->operands[0] = NULL;
2792                 expr->operands[1] = NULL;
2793
2794                 /*
2795                  * var refs won't be destroyed immediately
2796                  * See: destroy_hist_field()
2797                  */
2798                 destroy_hist_field(operand2, 0);
2799                 destroy_hist_field(operand1, 0);
2800
2801                 expr->name = expr_str(expr, 0);
2802         } else {
2803                 /* The operand sizes should be the same, so just pick one */
2804                 expr->size = operand1->size;
2805                 expr->is_signed = operand1->is_signed;
2806
2807                 expr->operator = field_op;
2808                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2809                 if (!expr->type) {
2810                         ret = -ENOMEM;
2811                         goto free_expr;
2812                 }
2813
2814                 expr->name = expr_str(expr, 0);
2815         }
2816
2817         return expr;
2818
2819 free_operands:
2820         destroy_hist_field(operand2, 0);
2821 free_op1:
2822         destroy_hist_field(operand1, 0);
2823         return ERR_PTR(ret);
2824
2825 free_expr:
2826         destroy_hist_field(expr, 0);
2827         return ERR_PTR(ret);
2828 }
2829
2830 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2831                                  struct trace_event_file *file)
2832 {
2833         struct event_trigger_data *test;
2834
2835         lockdep_assert_held(&event_mutex);
2836
2837         list_for_each_entry(test, &file->triggers, list) {
2838                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2839                         if (test->private_data == hist_data)
2840                                 return test->filter_str;
2841                 }
2842         }
2843
2844         return NULL;
2845 }
2846
2847 static struct event_command trigger_hist_cmd;
2848 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2849                                     struct trace_event_file *file,
2850                                     char *glob, char *cmd,
2851                                     char *param_and_filter);
2852
2853 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2854                             struct hist_trigger_data *hist_data,
2855                             unsigned int n_keys)
2856 {
2857         struct hist_field *target_hist_field, *hist_field;
2858         unsigned int n, i, j;
2859
2860         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2861                 return false;
2862
2863         i = hist_data->n_vals;
2864         j = target_hist_data->n_vals;
2865
2866         for (n = 0; n < n_keys; n++) {
2867                 hist_field = hist_data->fields[i + n];
2868                 target_hist_field = target_hist_data->fields[j + n];
2869
2870                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2871                         return false;
2872                 if (hist_field->size != target_hist_field->size)
2873                         return false;
2874                 if (hist_field->is_signed != target_hist_field->is_signed)
2875                         return false;
2876         }
2877
2878         return true;
2879 }
2880
2881 static struct hist_trigger_data *
2882 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2883                      struct trace_event_file *file)
2884 {
2885         struct hist_trigger_data *hist_data;
2886         struct event_trigger_data *test;
2887         unsigned int n_keys;
2888
2889         lockdep_assert_held(&event_mutex);
2890
2891         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2892
2893         list_for_each_entry(test, &file->triggers, list) {
2894                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2895                         hist_data = test->private_data;
2896
2897                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2898                                 return hist_data;
2899                 }
2900         }
2901
2902         return NULL;
2903 }
2904
2905 static struct trace_event_file *event_file(struct trace_array *tr,
2906                                            char *system, char *event_name)
2907 {
2908         struct trace_event_file *file;
2909
2910         file = __find_event_file(tr, system, event_name);
2911         if (!file)
2912                 return ERR_PTR(-EINVAL);
2913
2914         return file;
2915 }
2916
2917 static struct hist_field *
2918 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2919                          char *system, char *event_name, char *field_name)
2920 {
2921         struct hist_field *event_var;
2922         char *synthetic_name;
2923
2924         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2925         if (!synthetic_name)
2926                 return ERR_PTR(-ENOMEM);
2927
2928         strcpy(synthetic_name, "synthetic_");
2929         strcat(synthetic_name, field_name);
2930
2931         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2932
2933         kfree(synthetic_name);
2934
2935         return event_var;
2936 }
2937
2938 /**
2939  * create_field_var_hist - Automatically create a histogram and var for a field
2940  * @target_hist_data: The target hist trigger
2941  * @subsys_name: Optional subsystem name
2942  * @event_name: Optional event name
2943  * @field_name: The name of the field (and the resulting variable)
2944  *
2945  * Hist trigger actions fetch data from variables, not directly from
2946  * events.  However, for convenience, users are allowed to directly
2947  * specify an event field in an action, which will be automatically
2948  * converted into a variable on their behalf.
2949  *
2950  * If a user specifies a field on an event that isn't the event the
2951  * histogram currently being defined (the target event histogram), the
2952  * only way that can be accomplished is if a new hist trigger is
2953  * created and the field variable defined on that.
2954  *
2955  * This function creates a new histogram compatible with the target
2956  * event (meaning a histogram with the same key as the target
2957  * histogram), and creates a variable for the specified field, but
2958  * with 'synthetic_' prepended to the variable name in order to avoid
2959  * collision with normal field variables.
2960  *
2961  * Return: The variable created for the field.
2962  */
2963 static struct hist_field *
2964 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2965                       char *subsys_name, char *event_name, char *field_name)
2966 {
2967         struct trace_array *tr = target_hist_data->event_file->tr;
2968         struct hist_trigger_data *hist_data;
2969         unsigned int i, n, first = true;
2970         struct field_var_hist *var_hist;
2971         struct trace_event_file *file;
2972         struct hist_field *key_field;
2973         struct hist_field *event_var;
2974         char *saved_filter;
2975         char *cmd;
2976         int ret;
2977
2978         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2979                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2980                 return ERR_PTR(-EINVAL);
2981         }
2982
2983         file = event_file(tr, subsys_name, event_name);
2984
2985         if (IS_ERR(file)) {
2986                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2987                 ret = PTR_ERR(file);
2988                 return ERR_PTR(ret);
2989         }
2990
2991         /*
2992          * Look for a histogram compatible with target.  We'll use the
2993          * found histogram specification to create a new matching
2994          * histogram with our variable on it.  target_hist_data is not
2995          * yet a registered histogram so we can't use that.
2996          */
2997         hist_data = find_compatible_hist(target_hist_data, file);
2998         if (!hist_data) {
2999                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3000                 return ERR_PTR(-EINVAL);
3001         }
3002
3003         /* See if a synthetic field variable has already been created */
3004         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3005                                              event_name, field_name);
3006         if (!IS_ERR_OR_NULL(event_var))
3007                 return event_var;
3008
3009         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3010         if (!var_hist)
3011                 return ERR_PTR(-ENOMEM);
3012
3013         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3014         if (!cmd) {
3015                 kfree(var_hist);
3016                 return ERR_PTR(-ENOMEM);
3017         }
3018
3019         /* Use the same keys as the compatible histogram */
3020         strcat(cmd, "keys=");
3021
3022         for_each_hist_key_field(i, hist_data) {
3023                 key_field = hist_data->fields[i];
3024                 if (!first)
3025                         strcat(cmd, ",");
3026                 strcat(cmd, key_field->field->name);
3027                 first = false;
3028         }
3029
3030         /* Create the synthetic field variable specification */
3031         strcat(cmd, ":synthetic_");
3032         strcat(cmd, field_name);
3033         strcat(cmd, "=");
3034         strcat(cmd, field_name);
3035
3036         /* Use the same filter as the compatible histogram */
3037         saved_filter = find_trigger_filter(hist_data, file);
3038         if (saved_filter) {
3039                 strcat(cmd, " if ");
3040                 strcat(cmd, saved_filter);
3041         }
3042
3043         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3044         if (!var_hist->cmd) {
3045                 kfree(cmd);
3046                 kfree(var_hist);
3047                 return ERR_PTR(-ENOMEM);
3048         }
3049
3050         /* Save the compatible histogram information */
3051         var_hist->hist_data = hist_data;
3052
3053         /* Create the new histogram with our variable */
3054         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3055                                        "", "hist", cmd);
3056         if (ret) {
3057                 kfree(cmd);
3058                 kfree(var_hist->cmd);
3059                 kfree(var_hist);
3060                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3061                 return ERR_PTR(ret);
3062         }
3063
3064         kfree(cmd);
3065
3066         /* If we can't find the variable, something went wrong */
3067         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3068                                              event_name, field_name);
3069         if (IS_ERR_OR_NULL(event_var)) {
3070                 kfree(var_hist->cmd);
3071                 kfree(var_hist);
3072                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3073                 return ERR_PTR(-EINVAL);
3074         }
3075
3076         n = target_hist_data->n_field_var_hists;
3077         target_hist_data->field_var_hists[n] = var_hist;
3078         target_hist_data->n_field_var_hists++;
3079
3080         return event_var;
3081 }
3082
3083 static struct hist_field *
3084 find_target_event_var(struct hist_trigger_data *hist_data,
3085                       char *subsys_name, char *event_name, char *var_name)
3086 {
3087         struct trace_event_file *file = hist_data->event_file;
3088         struct hist_field *hist_field = NULL;
3089
3090         if (subsys_name) {
3091                 struct trace_event_call *call;
3092
3093                 if (!event_name)
3094                         return NULL;
3095
3096                 call = file->event_call;
3097
3098                 if (strcmp(subsys_name, call->class->system) != 0)
3099                         return NULL;
3100
3101                 if (strcmp(event_name, trace_event_name(call)) != 0)
3102                         return NULL;
3103         }
3104
3105         hist_field = find_var_field(hist_data, var_name);
3106
3107         return hist_field;
3108 }
3109
3110 static inline void __update_field_vars(struct tracing_map_elt *elt,
3111                                        struct trace_buffer *buffer,
3112                                        struct ring_buffer_event *rbe,
3113                                        void *rec,
3114                                        struct field_var **field_vars,
3115                                        unsigned int n_field_vars,
3116                                        unsigned int field_var_str_start)
3117 {
3118         struct hist_elt_data *elt_data = elt->private_data;
3119         unsigned int i, j, var_idx;
3120         u64 var_val;
3121
3122         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3123                 struct field_var *field_var = field_vars[i];
3124                 struct hist_field *var = field_var->var;
3125                 struct hist_field *val = field_var->val;
3126
3127                 var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3128                 var_idx = var->var.idx;
3129
3130                 if (val->flags & (HIST_FIELD_FL_STRING |
3131                                   HIST_FIELD_FL_STACKTRACE)) {
3132                         char *str = elt_data->field_var_str[j++];
3133                         char *val_str = (char *)(uintptr_t)var_val;
3134                         unsigned int size;
3135
3136                         if (val->flags & HIST_FIELD_FL_STRING) {
3137                                 size = min(val->size, STR_VAR_LEN_MAX);
3138                                 strscpy(str, val_str, size);
3139                         } else {
3140                                 char *stack_start = str + sizeof(unsigned long);
3141                                 int e;
3142
3143                                 e = stack_trace_save((void *)stack_start,
3144                                                      HIST_STACKTRACE_DEPTH,
3145                                                      HIST_STACKTRACE_SKIP);
3146                                 if (e < HIST_STACKTRACE_DEPTH - 1)
3147                                         ((unsigned long *)stack_start)[e] = 0;
3148                                 *((unsigned long *)str) = e;
3149                         }
3150                         var_val = (u64)(uintptr_t)str;
3151                 }
3152                 tracing_map_set_var(elt, var_idx, var_val);
3153         }
3154 }
3155
3156 static void update_field_vars(struct hist_trigger_data *hist_data,
3157                               struct tracing_map_elt *elt,
3158                               struct trace_buffer *buffer,
3159                               struct ring_buffer_event *rbe,
3160                               void *rec)
3161 {
3162         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3163                             hist_data->n_field_vars, 0);
3164 }
3165
3166 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3167                                  struct tracing_map_elt *elt,
3168                                  struct trace_buffer *buffer,  void *rec,
3169                                  struct ring_buffer_event *rbe, void *key,
3170                                  struct action_data *data, u64 *var_ref_vals)
3171 {
3172         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3173                             hist_data->n_save_vars, hist_data->n_field_var_str);
3174 }
3175
3176 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3177                                      struct trace_event_file *file,
3178                                      char *name, int size, const char *type)
3179 {
3180         struct hist_field *var;
3181         int idx;
3182
3183         if (find_var(hist_data, file, name) && !hist_data->remove) {
3184                 var = ERR_PTR(-EINVAL);
3185                 goto out;
3186         }
3187
3188         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3189         if (!var) {
3190                 var = ERR_PTR(-ENOMEM);
3191                 goto out;
3192         }
3193
3194         idx = tracing_map_add_var(hist_data->map);
3195         if (idx < 0) {
3196                 kfree(var);
3197                 var = ERR_PTR(-EINVAL);
3198                 goto out;
3199         }
3200
3201         var->ref = 1;
3202         var->flags = HIST_FIELD_FL_VAR;
3203         var->var.idx = idx;
3204         var->var.hist_data = var->hist_data = hist_data;
3205         var->size = size;
3206         var->var.name = kstrdup(name, GFP_KERNEL);
3207         var->type = kstrdup_const(type, GFP_KERNEL);
3208         if (!var->var.name || !var->type) {
3209                 kfree_const(var->type);
3210                 kfree(var->var.name);
3211                 kfree(var);
3212                 var = ERR_PTR(-ENOMEM);
3213         }
3214  out:
3215         return var;
3216 }
3217
3218 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3219                                           struct trace_event_file *file,
3220                                           char *field_name)
3221 {
3222         struct hist_field *val = NULL, *var = NULL;
3223         unsigned long flags = HIST_FIELD_FL_VAR;
3224         struct trace_array *tr = file->tr;
3225         struct field_var *field_var;
3226         int ret = 0;
3227
3228         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3229                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3230                 ret = -EINVAL;
3231                 goto err;
3232         }
3233
3234         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3235         if (IS_ERR(val)) {
3236                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3237                 ret = PTR_ERR(val);
3238                 goto err;
3239         }
3240
3241         var = create_var(hist_data, file, field_name, val->size, val->type);
3242         if (IS_ERR(var)) {
3243                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3244                 kfree(val);
3245                 ret = PTR_ERR(var);
3246                 goto err;
3247         }
3248
3249         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3250         if (!field_var) {
3251                 kfree(val);
3252                 kfree(var);
3253                 ret =  -ENOMEM;
3254                 goto err;
3255         }
3256
3257         field_var->var = var;
3258         field_var->val = val;
3259  out:
3260         return field_var;
3261  err:
3262         field_var = ERR_PTR(ret);
3263         goto out;
3264 }
3265
3266 /**
3267  * create_target_field_var - Automatically create a variable for a field
3268  * @target_hist_data: The target hist trigger
3269  * @subsys_name: Optional subsystem name
3270  * @event_name: Optional event name
3271  * @var_name: The name of the field (and the resulting variable)
3272  *
3273  * Hist trigger actions fetch data from variables, not directly from
3274  * events.  However, for convenience, users are allowed to directly
3275  * specify an event field in an action, which will be automatically
3276  * converted into a variable on their behalf.
3277  *
3278  * This function creates a field variable with the name var_name on
3279  * the hist trigger currently being defined on the target event.  If
3280  * subsys_name and event_name are specified, this function simply
3281  * verifies that they do in fact match the target event subsystem and
3282  * event name.
3283  *
3284  * Return: The variable created for the field.
3285  */
3286 static struct field_var *
3287 create_target_field_var(struct hist_trigger_data *target_hist_data,
3288                         char *subsys_name, char *event_name, char *var_name)
3289 {
3290         struct trace_event_file *file = target_hist_data->event_file;
3291
3292         if (subsys_name) {
3293                 struct trace_event_call *call;
3294
3295                 if (!event_name)
3296                         return NULL;
3297
3298                 call = file->event_call;
3299
3300                 if (strcmp(subsys_name, call->class->system) != 0)
3301                         return NULL;
3302
3303                 if (strcmp(event_name, trace_event_name(call)) != 0)
3304                         return NULL;
3305         }
3306
3307         return create_field_var(target_hist_data, file, var_name);
3308 }
3309
3310 static bool check_track_val_max(u64 track_val, u64 var_val)
3311 {
3312         if (var_val <= track_val)
3313                 return false;
3314
3315         return true;
3316 }
3317
3318 static bool check_track_val_changed(u64 track_val, u64 var_val)
3319 {
3320         if (var_val == track_val)
3321                 return false;
3322
3323         return true;
3324 }
3325
3326 static u64 get_track_val(struct hist_trigger_data *hist_data,
3327                          struct tracing_map_elt *elt,
3328                          struct action_data *data)
3329 {
3330         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3331         u64 track_val;
3332
3333         track_val = tracing_map_read_var(elt, track_var_idx);
3334
3335         return track_val;
3336 }
3337
3338 static void save_track_val(struct hist_trigger_data *hist_data,
3339                            struct tracing_map_elt *elt,
3340                            struct action_data *data, u64 var_val)
3341 {
3342         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3343
3344         tracing_map_set_var(elt, track_var_idx, var_val);
3345 }
3346
3347 static void save_track_data(struct hist_trigger_data *hist_data,
3348                             struct tracing_map_elt *elt,
3349                             struct trace_buffer *buffer, void *rec,
3350                             struct ring_buffer_event *rbe, void *key,
3351                             struct action_data *data, u64 *var_ref_vals)
3352 {
3353         if (data->track_data.save_data)
3354                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3355                                            key, data, var_ref_vals);
3356 }
3357
3358 static bool check_track_val(struct tracing_map_elt *elt,
3359                             struct action_data *data,
3360                             u64 var_val)
3361 {
3362         struct hist_trigger_data *hist_data;
3363         u64 track_val;
3364
3365         hist_data = data->track_data.track_var->hist_data;
3366         track_val = get_track_val(hist_data, elt, data);
3367
3368         return data->track_data.check_val(track_val, var_val);
3369 }
3370
3371 #ifdef CONFIG_TRACER_SNAPSHOT
3372 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3373 {
3374         /* called with tr->max_lock held */
3375         struct track_data *track_data = tr->cond_snapshot->cond_data;
3376         struct hist_elt_data *elt_data, *track_elt_data;
3377         struct snapshot_context *context = cond_data;
3378         struct action_data *action;
3379         u64 track_val;
3380
3381         if (!track_data)
3382                 return false;
3383
3384         action = track_data->action_data;
3385
3386         track_val = get_track_val(track_data->hist_data, context->elt,
3387                                   track_data->action_data);
3388
3389         if (!action->track_data.check_val(track_data->track_val, track_val))
3390                 return false;
3391
3392         track_data->track_val = track_val;
3393         memcpy(track_data->key, context->key, track_data->key_len);
3394
3395         elt_data = context->elt->private_data;
3396         track_elt_data = track_data->elt.private_data;
3397         if (elt_data->comm)
3398                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3399
3400         track_data->updated = true;
3401
3402         return true;
3403 }
3404
3405 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3406                                      struct tracing_map_elt *elt,
3407                                      struct trace_buffer *buffer, void *rec,
3408                                      struct ring_buffer_event *rbe, void *key,
3409                                      struct action_data *data,
3410                                      u64 *var_ref_vals)
3411 {
3412         struct trace_event_file *file = hist_data->event_file;
3413         struct snapshot_context context;
3414
3415         context.elt = elt;
3416         context.key = key;
3417
3418         tracing_snapshot_cond(file->tr, &context);
3419 }
3420
3421 static void hist_trigger_print_key(struct seq_file *m,
3422                                    struct hist_trigger_data *hist_data,
3423                                    void *key,
3424                                    struct tracing_map_elt *elt);
3425
3426 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3427 {
3428         unsigned int i;
3429
3430         if (!hist_data->n_actions)
3431                 return NULL;
3432
3433         for (i = 0; i < hist_data->n_actions; i++) {
3434                 struct action_data *data = hist_data->actions[i];
3435
3436                 if (data->action == ACTION_SNAPSHOT)
3437                         return data;
3438         }
3439
3440         return NULL;
3441 }
3442
3443 static void track_data_snapshot_print(struct seq_file *m,
3444                                       struct hist_trigger_data *hist_data)
3445 {
3446         struct trace_event_file *file = hist_data->event_file;
3447         struct track_data *track_data;
3448         struct action_data *action;
3449
3450         track_data = tracing_cond_snapshot_data(file->tr);
3451         if (!track_data)
3452                 return;
3453
3454         if (!track_data->updated)
3455                 return;
3456
3457         action = snapshot_action(hist_data);
3458         if (!action)
3459                 return;
3460
3461         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3462         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3463                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3464                    action->track_data.var_str, track_data->track_val);
3465
3466         seq_puts(m, "\ttriggered by event with key: ");
3467         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3468         seq_putc(m, '\n');
3469 }
3470 #else
3471 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3472 {
3473         return false;
3474 }
3475 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3476                                      struct tracing_map_elt *elt,
3477                                      struct trace_buffer *buffer, void *rec,
3478                                      struct ring_buffer_event *rbe, void *key,
3479                                      struct action_data *data,
3480                                      u64 *var_ref_vals) {}
3481 static void track_data_snapshot_print(struct seq_file *m,
3482                                       struct hist_trigger_data *hist_data) {}
3483 #endif /* CONFIG_TRACER_SNAPSHOT */
3484
3485 static void track_data_print(struct seq_file *m,
3486                              struct hist_trigger_data *hist_data,
3487                              struct tracing_map_elt *elt,
3488                              struct action_data *data)
3489 {
3490         u64 track_val = get_track_val(hist_data, elt, data);
3491         unsigned int i, save_var_idx;
3492
3493         if (data->handler == HANDLER_ONMAX)
3494                 seq_printf(m, "\n\tmax: %10llu", track_val);
3495         else if (data->handler == HANDLER_ONCHANGE)
3496                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3497
3498         if (data->action == ACTION_SNAPSHOT)
3499                 return;
3500
3501         for (i = 0; i < hist_data->n_save_vars; i++) {
3502                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3503                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3504                 u64 val;
3505
3506                 save_var_idx = save_var->var.idx;
3507
3508                 val = tracing_map_read_var(elt, save_var_idx);
3509
3510                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3511                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3512                                    (char *)(uintptr_t)(val));
3513                 } else
3514                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3515         }
3516 }
3517
3518 static void ontrack_action(struct hist_trigger_data *hist_data,
3519                            struct tracing_map_elt *elt,
3520                            struct trace_buffer *buffer, void *rec,
3521                            struct ring_buffer_event *rbe, void *key,
3522                            struct action_data *data, u64 *var_ref_vals)
3523 {
3524         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3525
3526         if (check_track_val(elt, data, var_val)) {
3527                 save_track_val(hist_data, elt, data, var_val);
3528                 save_track_data(hist_data, elt, buffer, rec, rbe,
3529                                 key, data, var_ref_vals);
3530         }
3531 }
3532
3533 static void action_data_destroy(struct action_data *data)
3534 {
3535         unsigned int i;
3536
3537         lockdep_assert_held(&event_mutex);
3538
3539         kfree(data->action_name);
3540
3541         for (i = 0; i < data->n_params; i++)
3542                 kfree(data->params[i]);
3543
3544         if (data->synth_event)
3545                 data->synth_event->ref--;
3546
3547         kfree(data->synth_event_name);
3548
3549         kfree(data);
3550 }
3551
3552 static void track_data_destroy(struct hist_trigger_data *hist_data,
3553                                struct action_data *data)
3554 {
3555         struct trace_event_file *file = hist_data->event_file;
3556
3557         destroy_hist_field(data->track_data.track_var, 0);
3558
3559         if (data->action == ACTION_SNAPSHOT) {
3560                 struct track_data *track_data;
3561
3562                 track_data = tracing_cond_snapshot_data(file->tr);
3563                 if (track_data && track_data->hist_data == hist_data) {
3564                         tracing_snapshot_cond_disable(file->tr);
3565                         track_data_free(track_data);
3566                 }
3567         }
3568
3569         kfree(data->track_data.var_str);
3570
3571         action_data_destroy(data);
3572 }
3573
3574 static int action_create(struct hist_trigger_data *hist_data,
3575                          struct action_data *data);
3576
3577 static int track_data_create(struct hist_trigger_data *hist_data,
3578                              struct action_data *data)
3579 {
3580         struct hist_field *var_field, *ref_field, *track_var = NULL;
3581         struct trace_event_file *file = hist_data->event_file;
3582         struct trace_array *tr = file->tr;
3583         char *track_data_var_str;
3584         int ret = 0;
3585
3586         track_data_var_str = data->track_data.var_str;
3587         if (track_data_var_str[0] != '$') {
3588                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3589                 return -EINVAL;
3590         }
3591         track_data_var_str++;
3592
3593         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3594         if (!var_field) {
3595                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3596                 return -EINVAL;
3597         }
3598
3599         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3600         if (!ref_field)
3601                 return -ENOMEM;
3602
3603         data->track_data.var_ref = ref_field;
3604
3605         if (data->handler == HANDLER_ONMAX)
3606                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3607         if (IS_ERR(track_var)) {
3608                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3609                 ret = PTR_ERR(track_var);
3610                 goto out;
3611         }
3612
3613         if (data->handler == HANDLER_ONCHANGE)
3614                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3615         if (IS_ERR(track_var)) {
3616                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3617                 ret = PTR_ERR(track_var);
3618                 goto out;
3619         }
3620         data->track_data.track_var = track_var;
3621
3622         ret = action_create(hist_data, data);
3623  out:
3624         return ret;
3625 }
3626
3627 static int parse_action_params(struct trace_array *tr, char *params,
3628                                struct action_data *data)
3629 {
3630         char *param, *saved_param;
3631         bool first_param = true;
3632         int ret = 0;
3633
3634         while (params) {
3635                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3636                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3637                         ret = -EINVAL;
3638                         goto out;
3639                 }
3640
3641                 param = strsep(&params, ",");
3642                 if (!param) {
3643                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3644                         ret = -EINVAL;
3645                         goto out;
3646                 }
3647
3648                 param = strstrip(param);
3649                 if (strlen(param) < 2) {
3650                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3651                         ret = -EINVAL;
3652                         goto out;
3653                 }
3654
3655                 saved_param = kstrdup(param, GFP_KERNEL);
3656                 if (!saved_param) {
3657                         ret = -ENOMEM;
3658                         goto out;
3659                 }
3660
3661                 if (first_param && data->use_trace_keyword) {
3662                         data->synth_event_name = saved_param;
3663                         first_param = false;
3664                         continue;
3665                 }
3666                 first_param = false;
3667
3668                 data->params[data->n_params++] = saved_param;
3669         }
3670  out:
3671         return ret;
3672 }
3673
3674 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3675                         enum handler_id handler)
3676 {
3677         char *action_name;
3678         int ret = 0;
3679
3680         strsep(&str, ".");
3681         if (!str) {
3682                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3683                 ret = -EINVAL;
3684                 goto out;
3685         }
3686
3687         action_name = strsep(&str, "(");
3688         if (!action_name || !str) {
3689                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3690                 ret = -EINVAL;
3691                 goto out;
3692         }
3693
3694         if (str_has_prefix(action_name, "save")) {
3695                 char *params = strsep(&str, ")");
3696
3697                 if (!params) {
3698                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3699                         ret = -EINVAL;
3700                         goto out;
3701                 }
3702
3703                 ret = parse_action_params(tr, params, data);
3704                 if (ret)
3705                         goto out;
3706
3707                 if (handler == HANDLER_ONMAX)
3708                         data->track_data.check_val = check_track_val_max;
3709                 else if (handler == HANDLER_ONCHANGE)
3710                         data->track_data.check_val = check_track_val_changed;
3711                 else {
3712                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3713                         ret = -EINVAL;
3714                         goto out;
3715                 }
3716
3717                 data->track_data.save_data = save_track_data_vars;
3718                 data->fn = ontrack_action;
3719                 data->action = ACTION_SAVE;
3720         } else if (str_has_prefix(action_name, "snapshot")) {
3721                 char *params = strsep(&str, ")");
3722
3723                 if (!str) {
3724                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3725                         ret = -EINVAL;
3726                         goto out;
3727                 }
3728
3729                 if (handler == HANDLER_ONMAX)
3730                         data->track_data.check_val = check_track_val_max;
3731                 else if (handler == HANDLER_ONCHANGE)
3732                         data->track_data.check_val = check_track_val_changed;
3733                 else {
3734                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3735                         ret = -EINVAL;
3736                         goto out;
3737                 }
3738
3739                 data->track_data.save_data = save_track_data_snapshot;
3740                 data->fn = ontrack_action;
3741                 data->action = ACTION_SNAPSHOT;
3742         } else {
3743                 char *params = strsep(&str, ")");
3744
3745                 if (str_has_prefix(action_name, "trace"))
3746                         data->use_trace_keyword = true;
3747
3748                 if (params) {
3749                         ret = parse_action_params(tr, params, data);
3750                         if (ret)
3751                                 goto out;
3752                 }
3753
3754                 if (handler == HANDLER_ONMAX)
3755                         data->track_data.check_val = check_track_val_max;
3756                 else if (handler == HANDLER_ONCHANGE)
3757                         data->track_data.check_val = check_track_val_changed;
3758
3759                 if (handler != HANDLER_ONMATCH) {
3760                         data->track_data.save_data = action_trace;
3761                         data->fn = ontrack_action;
3762                 } else
3763                         data->fn = action_trace;
3764
3765                 data->action = ACTION_TRACE;
3766         }
3767
3768         data->action_name = kstrdup(action_name, GFP_KERNEL);
3769         if (!data->action_name) {
3770                 ret = -ENOMEM;
3771                 goto out;
3772         }
3773
3774         data->handler = handler;
3775  out:
3776         return ret;
3777 }
3778
3779 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3780                                             char *str, enum handler_id handler)
3781 {
3782         struct action_data *data;
3783         int ret = -EINVAL;
3784         char *var_str;
3785
3786         data = kzalloc(sizeof(*data), GFP_KERNEL);
3787         if (!data)
3788                 return ERR_PTR(-ENOMEM);
3789
3790         var_str = strsep(&str, ")");
3791         if (!var_str || !str) {
3792                 ret = -EINVAL;
3793                 goto free;
3794         }
3795
3796         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3797         if (!data->track_data.var_str) {
3798                 ret = -ENOMEM;
3799                 goto free;
3800         }
3801
3802         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3803         if (ret)
3804                 goto free;
3805  out:
3806         return data;
3807  free:
3808         track_data_destroy(hist_data, data);
3809         data = ERR_PTR(ret);
3810         goto out;
3811 }
3812
3813 static void onmatch_destroy(struct action_data *data)
3814 {
3815         kfree(data->match_data.event);
3816         kfree(data->match_data.event_system);
3817
3818         action_data_destroy(data);
3819 }
3820
3821 static void destroy_field_var(struct field_var *field_var)
3822 {
3823         if (!field_var)
3824                 return;
3825
3826         destroy_hist_field(field_var->var, 0);
3827         destroy_hist_field(field_var->val, 0);
3828
3829         kfree(field_var);
3830 }
3831
3832 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3833 {
3834         unsigned int i;
3835
3836         for (i = 0; i < hist_data->n_field_vars; i++)
3837                 destroy_field_var(hist_data->field_vars[i]);
3838
3839         for (i = 0; i < hist_data->n_save_vars; i++)
3840                 destroy_field_var(hist_data->save_vars[i]);
3841 }
3842
3843 static void save_field_var(struct hist_trigger_data *hist_data,
3844                            struct field_var *field_var)
3845 {
3846         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3847
3848         /* Stack traces are saved in the string storage too */
3849         if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3850                 hist_data->n_field_var_str++;
3851 }
3852
3853
3854 static int check_synth_field(struct synth_event *event,
3855                              struct hist_field *hist_field,
3856                              unsigned int field_pos)
3857 {
3858         struct synth_field *field;
3859
3860         if (field_pos >= event->n_fields)
3861                 return -EINVAL;
3862
3863         field = event->fields[field_pos];
3864
3865         /*
3866          * A dynamic string synth field can accept static or
3867          * dynamic. A static string synth field can only accept a
3868          * same-sized static string, which is checked for later.
3869          */
3870         if (strstr(hist_field->type, "char[") && field->is_string
3871             && field->is_dynamic)
3872                 return 0;
3873
3874         if (strstr(hist_field->type, "long[") && field->is_stack)
3875                 return 0;
3876
3877         if (strcmp(field->type, hist_field->type) != 0) {
3878                 if (field->size != hist_field->size ||
3879                     (!field->is_string && field->is_signed != hist_field->is_signed))
3880                         return -EINVAL;
3881         }
3882
3883         return 0;
3884 }
3885
3886 static struct hist_field *
3887 trace_action_find_var(struct hist_trigger_data *hist_data,
3888                       struct action_data *data,
3889                       char *system, char *event, char *var)
3890 {
3891         struct trace_array *tr = hist_data->event_file->tr;
3892         struct hist_field *hist_field;
3893
3894         var++; /* skip '$' */
3895
3896         hist_field = find_target_event_var(hist_data, system, event, var);
3897         if (!hist_field) {
3898                 if (!system && data->handler == HANDLER_ONMATCH) {
3899                         system = data->match_data.event_system;
3900                         event = data->match_data.event;
3901                 }
3902
3903                 hist_field = find_event_var(hist_data, system, event, var);
3904         }
3905
3906         if (!hist_field)
3907                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3908
3909         return hist_field;
3910 }
3911
3912 static struct hist_field *
3913 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3914                               struct action_data *data, char *system,
3915                               char *event, char *var)
3916 {
3917         struct hist_field *hist_field = NULL;
3918         struct field_var *field_var;
3919
3920         /*
3921          * First try to create a field var on the target event (the
3922          * currently being defined).  This will create a variable for
3923          * unqualified fields on the target event, or if qualified,
3924          * target fields that have qualified names matching the target.
3925          */
3926         field_var = create_target_field_var(hist_data, system, event, var);
3927
3928         if (field_var && !IS_ERR(field_var)) {
3929                 save_field_var(hist_data, field_var);
3930                 hist_field = field_var->var;
3931         } else {
3932                 field_var = NULL;
3933                 /*
3934                  * If no explicit system.event is specified, default to
3935                  * looking for fields on the onmatch(system.event.xxx)
3936                  * event.
3937                  */
3938                 if (!system && data->handler == HANDLER_ONMATCH) {
3939                         system = data->match_data.event_system;
3940                         event = data->match_data.event;
3941                 }
3942
3943                 if (!event)
3944                         goto free;
3945                 /*
3946                  * At this point, we're looking at a field on another
3947                  * event.  Because we can't modify a hist trigger on
3948                  * another event to add a variable for a field, we need
3949                  * to create a new trigger on that event and create the
3950                  * variable at the same time.
3951                  */
3952                 hist_field = create_field_var_hist(hist_data, system, event, var);
3953                 if (IS_ERR(hist_field))
3954                         goto free;
3955         }
3956  out:
3957         return hist_field;
3958  free:
3959         destroy_field_var(field_var);
3960         hist_field = NULL;
3961         goto out;
3962 }
3963
3964 static int trace_action_create(struct hist_trigger_data *hist_data,
3965                                struct action_data *data)
3966 {
3967         struct trace_array *tr = hist_data->event_file->tr;
3968         char *event_name, *param, *system = NULL;
3969         struct hist_field *hist_field, *var_ref;
3970         unsigned int i;
3971         unsigned int field_pos = 0;
3972         struct synth_event *event;
3973         char *synth_event_name;
3974         int var_ref_idx, ret = 0;
3975
3976         lockdep_assert_held(&event_mutex);
3977
3978         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3979         if (data->n_params > SYNTH_FIELDS_MAX)
3980                 return -EINVAL;
3981
3982         if (data->use_trace_keyword)
3983                 synth_event_name = data->synth_event_name;
3984         else
3985                 synth_event_name = data->action_name;
3986
3987         event = find_synth_event(synth_event_name);
3988         if (!event) {
3989                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3990                 return -EINVAL;
3991         }
3992
3993         event->ref++;
3994
3995         for (i = 0; i < data->n_params; i++) {
3996                 char *p;
3997
3998                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3999                 if (!param) {
4000                         ret = -ENOMEM;
4001                         goto err;
4002                 }
4003
4004                 system = strsep(&param, ".");
4005                 if (!param) {
4006                         param = (char *)system;
4007                         system = event_name = NULL;
4008                 } else {
4009                         event_name = strsep(&param, ".");
4010                         if (!param) {
4011                                 kfree(p);
4012                                 ret = -EINVAL;
4013                                 goto err;
4014                         }
4015                 }
4016
4017                 if (param[0] == '$')
4018                         hist_field = trace_action_find_var(hist_data, data,
4019                                                            system, event_name,
4020                                                            param);
4021                 else
4022                         hist_field = trace_action_create_field_var(hist_data,
4023                                                                    data,
4024                                                                    system,
4025                                                                    event_name,
4026                                                                    param);
4027
4028                 if (!hist_field) {
4029                         kfree(p);
4030                         ret = -EINVAL;
4031                         goto err;
4032                 }
4033
4034                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4035                         var_ref = create_var_ref(hist_data, hist_field,
4036                                                  system, event_name);
4037                         if (!var_ref) {
4038                                 kfree(p);
4039                                 ret = -ENOMEM;
4040                                 goto err;
4041                         }
4042
4043                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4044                         if (WARN_ON(var_ref_idx < 0)) {
4045                                 kfree(p);
4046                                 ret = var_ref_idx;
4047                                 goto err;
4048                         }
4049
4050                         data->var_ref_idx[i] = var_ref_idx;
4051
4052                         field_pos++;
4053                         kfree(p);
4054                         continue;
4055                 }
4056
4057                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4058                 kfree(p);
4059                 ret = -EINVAL;
4060                 goto err;
4061         }
4062
4063         if (field_pos != event->n_fields) {
4064                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4065                 ret = -EINVAL;
4066                 goto err;
4067         }
4068
4069         data->synth_event = event;
4070  out:
4071         return ret;
4072  err:
4073         event->ref--;
4074
4075         goto out;
4076 }
4077
4078 static int action_create(struct hist_trigger_data *hist_data,
4079                          struct action_data *data)
4080 {
4081         struct trace_event_file *file = hist_data->event_file;
4082         struct trace_array *tr = file->tr;
4083         struct track_data *track_data;
4084         struct field_var *field_var;
4085         unsigned int i;
4086         char *param;
4087         int ret = 0;
4088
4089         if (data->action == ACTION_TRACE)
4090                 return trace_action_create(hist_data, data);
4091
4092         if (data->action == ACTION_SNAPSHOT) {
4093                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4094                 if (IS_ERR(track_data)) {
4095                         ret = PTR_ERR(track_data);
4096                         goto out;
4097                 }
4098
4099                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4100                                                    cond_snapshot_update);
4101                 if (ret)
4102                         track_data_free(track_data);
4103
4104                 goto out;
4105         }
4106
4107         if (data->action == ACTION_SAVE) {
4108                 if (hist_data->n_save_vars) {
4109                         ret = -EEXIST;
4110                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4111                         goto out;
4112                 }
4113
4114                 for (i = 0; i < data->n_params; i++) {
4115                         param = kstrdup(data->params[i], GFP_KERNEL);
4116                         if (!param) {
4117                                 ret = -ENOMEM;
4118                                 goto out;
4119                         }
4120
4121                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4122                         if (IS_ERR(field_var)) {
4123                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4124                                          errpos(param));
4125                                 ret = PTR_ERR(field_var);
4126                                 kfree(param);
4127                                 goto out;
4128                         }
4129
4130                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4131                         if (field_var->val->flags &
4132                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4133                                 hist_data->n_save_var_str++;
4134                         kfree(param);
4135                 }
4136         }
4137  out:
4138         return ret;
4139 }
4140
4141 static int onmatch_create(struct hist_trigger_data *hist_data,
4142                           struct action_data *data)
4143 {
4144         return action_create(hist_data, data);
4145 }
4146
4147 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4148 {
4149         char *match_event, *match_event_system;
4150         struct action_data *data;
4151         int ret = -EINVAL;
4152
4153         data = kzalloc(sizeof(*data), GFP_KERNEL);
4154         if (!data)
4155                 return ERR_PTR(-ENOMEM);
4156
4157         match_event = strsep(&str, ")");
4158         if (!match_event || !str) {
4159                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4160                 goto free;
4161         }
4162
4163         match_event_system = strsep(&match_event, ".");
4164         if (!match_event) {
4165                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4166                 goto free;
4167         }
4168
4169         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4170                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4171                 goto free;
4172         }
4173
4174         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4175         if (!data->match_data.event) {
4176                 ret = -ENOMEM;
4177                 goto free;
4178         }
4179
4180         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4181         if (!data->match_data.event_system) {
4182                 ret = -ENOMEM;
4183                 goto free;
4184         }
4185
4186         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4187         if (ret)
4188                 goto free;
4189  out:
4190         return data;
4191  free:
4192         onmatch_destroy(data);
4193         data = ERR_PTR(ret);
4194         goto out;
4195 }
4196
4197 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4198 {
4199         hist_data->fields[HITCOUNT_IDX] =
4200                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4201         if (!hist_data->fields[HITCOUNT_IDX])
4202                 return -ENOMEM;
4203
4204         hist_data->n_vals++;
4205         hist_data->n_fields++;
4206
4207         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4208                 return -EINVAL;
4209
4210         return 0;
4211 }
4212
4213 static int __create_val_field(struct hist_trigger_data *hist_data,
4214                               unsigned int val_idx,
4215                               struct trace_event_file *file,
4216                               char *var_name, char *field_str,
4217                               unsigned long flags)
4218 {
4219         struct hist_field *hist_field;
4220         int ret = 0, n_subexprs = 0;
4221
4222         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4223         if (IS_ERR(hist_field)) {
4224                 ret = PTR_ERR(hist_field);
4225                 goto out;
4226         }
4227
4228         hist_data->fields[val_idx] = hist_field;
4229
4230         ++hist_data->n_vals;
4231         ++hist_data->n_fields;
4232
4233         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4234                 ret = -EINVAL;
4235  out:
4236         return ret;
4237 }
4238
4239 static int create_val_field(struct hist_trigger_data *hist_data,
4240                             unsigned int val_idx,
4241                             struct trace_event_file *file,
4242                             char *field_str)
4243 {
4244         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4245                 return -EINVAL;
4246
4247         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4248 }
4249
4250 static const char no_comm[] = "(no comm)";
4251
4252 static u64 hist_field_execname(struct hist_field *hist_field,
4253                                struct tracing_map_elt *elt,
4254                                struct trace_buffer *buffer,
4255                                struct ring_buffer_event *rbe,
4256                                void *event)
4257 {
4258         struct hist_elt_data *elt_data;
4259
4260         if (WARN_ON_ONCE(!elt))
4261                 return (u64)(unsigned long)no_comm;
4262
4263         elt_data = elt->private_data;
4264
4265         if (WARN_ON_ONCE(!elt_data->comm))
4266                 return (u64)(unsigned long)no_comm;
4267
4268         return (u64)(unsigned long)(elt_data->comm);
4269 }
4270
4271 static u64 hist_fn_call(struct hist_field *hist_field,
4272                         struct tracing_map_elt *elt,
4273                         struct trace_buffer *buffer,
4274                         struct ring_buffer_event *rbe,
4275                         void *event)
4276 {
4277         switch (hist_field->fn_num) {
4278         case HIST_FIELD_FN_VAR_REF:
4279                 return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4280         case HIST_FIELD_FN_COUNTER:
4281                 return hist_field_counter(hist_field, elt, buffer, rbe, event);
4282         case HIST_FIELD_FN_CONST:
4283                 return hist_field_const(hist_field, elt, buffer, rbe, event);
4284         case HIST_FIELD_FN_LOG2:
4285                 return hist_field_log2(hist_field, elt, buffer, rbe, event);
4286         case HIST_FIELD_FN_BUCKET:
4287                 return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4288         case HIST_FIELD_FN_TIMESTAMP:
4289                 return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4290         case HIST_FIELD_FN_CPU:
4291                 return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4292         case HIST_FIELD_FN_STRING:
4293                 return hist_field_string(hist_field, elt, buffer, rbe, event);
4294         case HIST_FIELD_FN_DYNSTRING:
4295                 return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4296         case HIST_FIELD_FN_RELDYNSTRING:
4297                 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4298         case HIST_FIELD_FN_PSTRING:
4299                 return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4300         case HIST_FIELD_FN_S64:
4301                 return hist_field_s64(hist_field, elt, buffer, rbe, event);
4302         case HIST_FIELD_FN_U64:
4303                 return hist_field_u64(hist_field, elt, buffer, rbe, event);
4304         case HIST_FIELD_FN_S32:
4305                 return hist_field_s32(hist_field, elt, buffer, rbe, event);
4306         case HIST_FIELD_FN_U32:
4307                 return hist_field_u32(hist_field, elt, buffer, rbe, event);
4308         case HIST_FIELD_FN_S16:
4309                 return hist_field_s16(hist_field, elt, buffer, rbe, event);
4310         case HIST_FIELD_FN_U16:
4311                 return hist_field_u16(hist_field, elt, buffer, rbe, event);
4312         case HIST_FIELD_FN_S8:
4313                 return hist_field_s8(hist_field, elt, buffer, rbe, event);
4314         case HIST_FIELD_FN_U8:
4315                 return hist_field_u8(hist_field, elt, buffer, rbe, event);
4316         case HIST_FIELD_FN_UMINUS:
4317                 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4318         case HIST_FIELD_FN_MINUS:
4319                 return hist_field_minus(hist_field, elt, buffer, rbe, event);
4320         case HIST_FIELD_FN_PLUS:
4321                 return hist_field_plus(hist_field, elt, buffer, rbe, event);
4322         case HIST_FIELD_FN_DIV:
4323                 return hist_field_div(hist_field, elt, buffer, rbe, event);
4324         case HIST_FIELD_FN_MULT:
4325                 return hist_field_mult(hist_field, elt, buffer, rbe, event);
4326         case HIST_FIELD_FN_DIV_POWER2:
4327                 return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4328         case HIST_FIELD_FN_DIV_NOT_POWER2:
4329                 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4330         case HIST_FIELD_FN_DIV_MULT_SHIFT:
4331                 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4332         case HIST_FIELD_FN_EXECNAME:
4333                 return hist_field_execname(hist_field, elt, buffer, rbe, event);
4334         default:
4335                 return 0;
4336         }
4337 }
4338
4339 /* Convert a var that points to common_pid.execname to a string */
4340 static void update_var_execname(struct hist_field *hist_field)
4341 {
4342         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4343                 HIST_FIELD_FL_EXECNAME;
4344         hist_field->size = MAX_FILTER_STR_VAL;
4345         hist_field->is_signed = 0;
4346
4347         kfree_const(hist_field->type);
4348         hist_field->type = "char[]";
4349
4350         hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4351 }
4352
4353 static int create_var_field(struct hist_trigger_data *hist_data,
4354                             unsigned int val_idx,
4355                             struct trace_event_file *file,
4356                             char *var_name, char *expr_str)
4357 {
4358         struct trace_array *tr = hist_data->event_file->tr;
4359         unsigned long flags = 0;
4360         int ret;
4361
4362         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4363                 return -EINVAL;
4364
4365         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4366                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4367                 return -EINVAL;
4368         }
4369
4370         flags |= HIST_FIELD_FL_VAR;
4371         hist_data->n_vars++;
4372         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4373                 return -EINVAL;
4374
4375         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4376
4377         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4378                 update_var_execname(hist_data->fields[val_idx]);
4379
4380         if (!ret && hist_data->fields[val_idx]->flags &
4381             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4382                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4383
4384         return ret;
4385 }
4386
4387 static int create_val_fields(struct hist_trigger_data *hist_data,
4388                              struct trace_event_file *file)
4389 {
4390         unsigned int i, j = 1, n_hitcount = 0;
4391         char *fields_str, *field_str;
4392         int ret;
4393
4394         ret = create_hitcount_val(hist_data);
4395         if (ret)
4396                 goto out;
4397
4398         fields_str = hist_data->attrs->vals_str;
4399         if (!fields_str)
4400                 goto out;
4401
4402         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4403                      j < TRACING_MAP_VALS_MAX; i++) {
4404                 field_str = strsep(&fields_str, ",");
4405                 if (!field_str)
4406                         break;
4407
4408                 if (strcmp(field_str, "hitcount") == 0) {
4409                         if (!n_hitcount++)
4410                                 continue;
4411                 }
4412
4413                 ret = create_val_field(hist_data, j++, file, field_str);
4414                 if (ret)
4415                         goto out;
4416         }
4417
4418         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4419                 ret = -EINVAL;
4420  out:
4421         /* There is only raw hitcount but nohitcount suppresses it. */
4422         if (j == 1 && hist_data->attrs->no_hitcount) {
4423                 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4424                 ret = -ENOENT;
4425         }
4426
4427         return ret;
4428 }
4429
4430 static int create_key_field(struct hist_trigger_data *hist_data,
4431                             unsigned int key_idx,
4432                             unsigned int key_offset,
4433                             struct trace_event_file *file,
4434                             char *field_str)
4435 {
4436         struct trace_array *tr = hist_data->event_file->tr;
4437         struct hist_field *hist_field = NULL;
4438         unsigned long flags = 0;
4439         unsigned int key_size;
4440         int ret = 0, n_subexprs = 0;
4441
4442         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4443                 return -EINVAL;
4444
4445         flags |= HIST_FIELD_FL_KEY;
4446
4447         if (strcmp(field_str, "stacktrace") == 0) {
4448                 flags |= HIST_FIELD_FL_STACKTRACE;
4449                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4450                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4451         } else {
4452                 hist_field = parse_expr(hist_data, file, field_str, flags,
4453                                         NULL, &n_subexprs);
4454                 if (IS_ERR(hist_field)) {
4455                         ret = PTR_ERR(hist_field);
4456                         goto out;
4457                 }
4458
4459                 if (field_has_hist_vars(hist_field, 0)) {
4460                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4461                         destroy_hist_field(hist_field, 0);
4462                         ret = -EINVAL;
4463                         goto out;
4464                 }
4465
4466                 key_size = hist_field->size;
4467         }
4468
4469         hist_data->fields[key_idx] = hist_field;
4470
4471         key_size = ALIGN(key_size, sizeof(u64));
4472         hist_data->fields[key_idx]->size = key_size;
4473         hist_data->fields[key_idx]->offset = key_offset;
4474
4475         hist_data->key_size += key_size;
4476
4477         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4478                 ret = -EINVAL;
4479                 goto out;
4480         }
4481
4482         hist_data->n_keys++;
4483         hist_data->n_fields++;
4484
4485         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4486                 return -EINVAL;
4487
4488         ret = key_size;
4489  out:
4490         return ret;
4491 }
4492
4493 static int create_key_fields(struct hist_trigger_data *hist_data,
4494                              struct trace_event_file *file)
4495 {
4496         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4497         char *fields_str, *field_str;
4498         int ret = -EINVAL;
4499
4500         fields_str = hist_data->attrs->keys_str;
4501         if (!fields_str)
4502                 goto out;
4503
4504         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4505                 field_str = strsep(&fields_str, ",");
4506                 if (!field_str)
4507                         break;
4508                 ret = create_key_field(hist_data, i, key_offset,
4509                                        file, field_str);
4510                 if (ret < 0)
4511                         goto out;
4512                 key_offset += ret;
4513         }
4514         if (fields_str) {
4515                 ret = -EINVAL;
4516                 goto out;
4517         }
4518         ret = 0;
4519  out:
4520         return ret;
4521 }
4522
4523 static int create_var_fields(struct hist_trigger_data *hist_data,
4524                              struct trace_event_file *file)
4525 {
4526         unsigned int i, j = hist_data->n_vals;
4527         int ret = 0;
4528
4529         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4530
4531         for (i = 0; i < n_vars; i++) {
4532                 char *var_name = hist_data->attrs->var_defs.name[i];
4533                 char *expr = hist_data->attrs->var_defs.expr[i];
4534
4535                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4536                 if (ret)
4537                         goto out;
4538         }
4539  out:
4540         return ret;
4541 }
4542
4543 static void free_var_defs(struct hist_trigger_data *hist_data)
4544 {
4545         unsigned int i;
4546
4547         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4548                 kfree(hist_data->attrs->var_defs.name[i]);
4549                 kfree(hist_data->attrs->var_defs.expr[i]);
4550         }
4551
4552         hist_data->attrs->var_defs.n_vars = 0;
4553 }
4554
4555 static int parse_var_defs(struct hist_trigger_data *hist_data)
4556 {
4557         struct trace_array *tr = hist_data->event_file->tr;
4558         char *s, *str, *var_name, *field_str;
4559         unsigned int i, j, n_vars = 0;
4560         int ret = 0;
4561
4562         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4563                 str = hist_data->attrs->assignment_str[i];
4564                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4565                         field_str = strsep(&str, ",");
4566                         if (!field_str)
4567                                 break;
4568
4569                         var_name = strsep(&field_str, "=");
4570                         if (!var_name || !field_str) {
4571                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4572                                          errpos(var_name));
4573                                 ret = -EINVAL;
4574                                 goto free;
4575                         }
4576
4577                         if (n_vars == TRACING_MAP_VARS_MAX) {
4578                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4579                                 ret = -EINVAL;
4580                                 goto free;
4581                         }
4582
4583                         s = kstrdup(var_name, GFP_KERNEL);
4584                         if (!s) {
4585                                 ret = -ENOMEM;
4586                                 goto free;
4587                         }
4588                         hist_data->attrs->var_defs.name[n_vars] = s;
4589
4590                         s = kstrdup(field_str, GFP_KERNEL);
4591                         if (!s) {
4592                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4593                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4594                                 ret = -ENOMEM;
4595                                 goto free;
4596                         }
4597                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4598
4599                         hist_data->attrs->var_defs.n_vars = n_vars;
4600                 }
4601         }
4602
4603         return ret;
4604  free:
4605         free_var_defs(hist_data);
4606
4607         return ret;
4608 }
4609
4610 static int create_hist_fields(struct hist_trigger_data *hist_data,
4611                               struct trace_event_file *file)
4612 {
4613         int ret;
4614
4615         ret = parse_var_defs(hist_data);
4616         if (ret)
4617                 return ret;
4618
4619         ret = create_val_fields(hist_data, file);
4620         if (ret)
4621                 goto out;
4622
4623         ret = create_var_fields(hist_data, file);
4624         if (ret)
4625                 goto out;
4626
4627         ret = create_key_fields(hist_data, file);
4628
4629  out:
4630         free_var_defs(hist_data);
4631
4632         return ret;
4633 }
4634
4635 static int is_descending(struct trace_array *tr, const char *str)
4636 {
4637         if (!str)
4638                 return 0;
4639
4640         if (strcmp(str, "descending") == 0)
4641                 return 1;
4642
4643         if (strcmp(str, "ascending") == 0)
4644                 return 0;
4645
4646         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4647
4648         return -EINVAL;
4649 }
4650
4651 static int create_sort_keys(struct hist_trigger_data *hist_data)
4652 {
4653         struct trace_array *tr = hist_data->event_file->tr;
4654         char *fields_str = hist_data->attrs->sort_key_str;
4655         struct tracing_map_sort_key *sort_key;
4656         int descending, ret = 0;
4657         unsigned int i, j, k;
4658
4659         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4660
4661         if (!fields_str)
4662                 goto out;
4663
4664         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4665                 struct hist_field *hist_field;
4666                 char *field_str, *field_name;
4667                 const char *test_name;
4668
4669                 sort_key = &hist_data->sort_keys[i];
4670
4671                 field_str = strsep(&fields_str, ",");
4672                 if (!field_str)
4673                         break;
4674
4675                 if (!*field_str) {
4676                         ret = -EINVAL;
4677                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4678                         break;
4679                 }
4680
4681                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4682                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4683                         ret = -EINVAL;
4684                         break;
4685                 }
4686
4687                 field_name = strsep(&field_str, ".");
4688                 if (!field_name || !*field_name) {
4689                         ret = -EINVAL;
4690                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4691                         break;
4692                 }
4693
4694                 if (strcmp(field_name, "hitcount") == 0) {
4695                         descending = is_descending(tr, field_str);
4696                         if (descending < 0) {
4697                                 ret = descending;
4698                                 break;
4699                         }
4700                         sort_key->descending = descending;
4701                         continue;
4702                 }
4703
4704                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4705                         unsigned int idx;
4706
4707                         hist_field = hist_data->fields[j];
4708                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4709                                 continue;
4710
4711                         idx = k++;
4712
4713                         test_name = hist_field_name(hist_field, 0);
4714
4715                         if (strcmp(field_name, test_name) == 0) {
4716                                 sort_key->field_idx = idx;
4717                                 descending = is_descending(tr, field_str);
4718                                 if (descending < 0) {
4719                                         ret = descending;
4720                                         goto out;
4721                                 }
4722                                 sort_key->descending = descending;
4723                                 break;
4724                         }
4725                 }
4726                 if (j == hist_data->n_fields) {
4727                         ret = -EINVAL;
4728                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4729                         break;
4730                 }
4731         }
4732
4733         hist_data->n_sort_keys = i;
4734  out:
4735         return ret;
4736 }
4737
4738 static void destroy_actions(struct hist_trigger_data *hist_data)
4739 {
4740         unsigned int i;
4741
4742         for (i = 0; i < hist_data->n_actions; i++) {
4743                 struct action_data *data = hist_data->actions[i];
4744
4745                 if (data->handler == HANDLER_ONMATCH)
4746                         onmatch_destroy(data);
4747                 else if (data->handler == HANDLER_ONMAX ||
4748                          data->handler == HANDLER_ONCHANGE)
4749                         track_data_destroy(hist_data, data);
4750                 else
4751                         kfree(data);
4752         }
4753 }
4754
4755 static int parse_actions(struct hist_trigger_data *hist_data)
4756 {
4757         struct trace_array *tr = hist_data->event_file->tr;
4758         struct action_data *data;
4759         unsigned int i;
4760         int ret = 0;
4761         char *str;
4762         int len;
4763
4764         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4765                 str = hist_data->attrs->action_str[i];
4766
4767                 if ((len = str_has_prefix(str, "onmatch("))) {
4768                         char *action_str = str + len;
4769
4770                         data = onmatch_parse(tr, action_str);
4771                         if (IS_ERR(data)) {
4772                                 ret = PTR_ERR(data);
4773                                 break;
4774                         }
4775                 } else if ((len = str_has_prefix(str, "onmax("))) {
4776                         char *action_str = str + len;
4777
4778                         data = track_data_parse(hist_data, action_str,
4779                                                 HANDLER_ONMAX);
4780                         if (IS_ERR(data)) {
4781                                 ret = PTR_ERR(data);
4782                                 break;
4783                         }
4784                 } else if ((len = str_has_prefix(str, "onchange("))) {
4785                         char *action_str = str + len;
4786
4787                         data = track_data_parse(hist_data, action_str,
4788                                                 HANDLER_ONCHANGE);
4789                         if (IS_ERR(data)) {
4790                                 ret = PTR_ERR(data);
4791                                 break;
4792                         }
4793                 } else {
4794                         ret = -EINVAL;
4795                         break;
4796                 }
4797
4798                 hist_data->actions[hist_data->n_actions++] = data;
4799         }
4800
4801         return ret;
4802 }
4803
4804 static int create_actions(struct hist_trigger_data *hist_data)
4805 {
4806         struct action_data *data;
4807         unsigned int i;
4808         int ret = 0;
4809
4810         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4811                 data = hist_data->actions[i];
4812
4813                 if (data->handler == HANDLER_ONMATCH) {
4814                         ret = onmatch_create(hist_data, data);
4815                         if (ret)
4816                                 break;
4817                 } else if (data->handler == HANDLER_ONMAX ||
4818                            data->handler == HANDLER_ONCHANGE) {
4819                         ret = track_data_create(hist_data, data);
4820                         if (ret)
4821                                 break;
4822                 } else {
4823                         ret = -EINVAL;
4824                         break;
4825                 }
4826         }
4827
4828         return ret;
4829 }
4830
4831 static void print_actions(struct seq_file *m,
4832                           struct hist_trigger_data *hist_data,
4833                           struct tracing_map_elt *elt)
4834 {
4835         unsigned int i;
4836
4837         for (i = 0; i < hist_data->n_actions; i++) {
4838                 struct action_data *data = hist_data->actions[i];
4839
4840                 if (data->action == ACTION_SNAPSHOT)
4841                         continue;
4842
4843                 if (data->handler == HANDLER_ONMAX ||
4844                     data->handler == HANDLER_ONCHANGE)
4845                         track_data_print(m, hist_data, elt, data);
4846         }
4847 }
4848
4849 static void print_action_spec(struct seq_file *m,
4850                               struct hist_trigger_data *hist_data,
4851                               struct action_data *data)
4852 {
4853         unsigned int i;
4854
4855         if (data->action == ACTION_SAVE) {
4856                 for (i = 0; i < hist_data->n_save_vars; i++) {
4857                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4858                         if (i < hist_data->n_save_vars - 1)
4859                                 seq_puts(m, ",");
4860                 }
4861         } else if (data->action == ACTION_TRACE) {
4862                 if (data->use_trace_keyword)
4863                         seq_printf(m, "%s", data->synth_event_name);
4864                 for (i = 0; i < data->n_params; i++) {
4865                         if (i || data->use_trace_keyword)
4866                                 seq_puts(m, ",");
4867                         seq_printf(m, "%s", data->params[i]);
4868                 }
4869         }
4870 }
4871
4872 static void print_track_data_spec(struct seq_file *m,
4873                                   struct hist_trigger_data *hist_data,
4874                                   struct action_data *data)
4875 {
4876         if (data->handler == HANDLER_ONMAX)
4877                 seq_puts(m, ":onmax(");
4878         else if (data->handler == HANDLER_ONCHANGE)
4879                 seq_puts(m, ":onchange(");
4880         seq_printf(m, "%s", data->track_data.var_str);
4881         seq_printf(m, ").%s(", data->action_name);
4882
4883         print_action_spec(m, hist_data, data);
4884
4885         seq_puts(m, ")");
4886 }
4887
4888 static void print_onmatch_spec(struct seq_file *m,
4889                                struct hist_trigger_data *hist_data,
4890                                struct action_data *data)
4891 {
4892         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4893                    data->match_data.event);
4894
4895         seq_printf(m, "%s(", data->action_name);
4896
4897         print_action_spec(m, hist_data, data);
4898
4899         seq_puts(m, ")");
4900 }
4901
4902 static bool actions_match(struct hist_trigger_data *hist_data,
4903                           struct hist_trigger_data *hist_data_test)
4904 {
4905         unsigned int i, j;
4906
4907         if (hist_data->n_actions != hist_data_test->n_actions)
4908                 return false;
4909
4910         for (i = 0; i < hist_data->n_actions; i++) {
4911                 struct action_data *data = hist_data->actions[i];
4912                 struct action_data *data_test = hist_data_test->actions[i];
4913                 char *action_name, *action_name_test;
4914
4915                 if (data->handler != data_test->handler)
4916                         return false;
4917                 if (data->action != data_test->action)
4918                         return false;
4919
4920                 if (data->n_params != data_test->n_params)
4921                         return false;
4922
4923                 for (j = 0; j < data->n_params; j++) {
4924                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4925                                 return false;
4926                 }
4927
4928                 if (data->use_trace_keyword)
4929                         action_name = data->synth_event_name;
4930                 else
4931                         action_name = data->action_name;
4932
4933                 if (data_test->use_trace_keyword)
4934                         action_name_test = data_test->synth_event_name;
4935                 else
4936                         action_name_test = data_test->action_name;
4937
4938                 if (strcmp(action_name, action_name_test) != 0)
4939                         return false;
4940
4941                 if (data->handler == HANDLER_ONMATCH) {
4942                         if (strcmp(data->match_data.event_system,
4943                                    data_test->match_data.event_system) != 0)
4944                                 return false;
4945                         if (strcmp(data->match_data.event,
4946                                    data_test->match_data.event) != 0)
4947                                 return false;
4948                 } else if (data->handler == HANDLER_ONMAX ||
4949                            data->handler == HANDLER_ONCHANGE) {
4950                         if (strcmp(data->track_data.var_str,
4951                                    data_test->track_data.var_str) != 0)
4952                                 return false;
4953                 }
4954         }
4955
4956         return true;
4957 }
4958
4959
4960 static void print_actions_spec(struct seq_file *m,
4961                                struct hist_trigger_data *hist_data)
4962 {
4963         unsigned int i;
4964
4965         for (i = 0; i < hist_data->n_actions; i++) {
4966                 struct action_data *data = hist_data->actions[i];
4967
4968                 if (data->handler == HANDLER_ONMATCH)
4969                         print_onmatch_spec(m, hist_data, data);
4970                 else if (data->handler == HANDLER_ONMAX ||
4971                          data->handler == HANDLER_ONCHANGE)
4972                         print_track_data_spec(m, hist_data, data);
4973         }
4974 }
4975
4976 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4977 {
4978         unsigned int i;
4979
4980         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4981                 kfree(hist_data->field_var_hists[i]->cmd);
4982                 kfree(hist_data->field_var_hists[i]);
4983         }
4984 }
4985
4986 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4987 {
4988         if (!hist_data)
4989                 return;
4990
4991         destroy_hist_trigger_attrs(hist_data->attrs);
4992         destroy_hist_fields(hist_data);
4993         tracing_map_destroy(hist_data->map);
4994
4995         destroy_actions(hist_data);
4996         destroy_field_vars(hist_data);
4997         destroy_field_var_hists(hist_data);
4998
4999         kfree(hist_data);
5000 }
5001
5002 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5003 {
5004         struct tracing_map *map = hist_data->map;
5005         struct ftrace_event_field *field;
5006         struct hist_field *hist_field;
5007         int i, idx = 0;
5008
5009         for_each_hist_field(i, hist_data) {
5010                 hist_field = hist_data->fields[i];
5011                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5012                         tracing_map_cmp_fn_t cmp_fn;
5013
5014                         field = hist_field->field;
5015
5016                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5017                                 cmp_fn = tracing_map_cmp_none;
5018                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5019                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5020                                                              hist_field->is_signed);
5021                         else if (is_string_field(field))
5022                                 cmp_fn = tracing_map_cmp_string;
5023                         else
5024                                 cmp_fn = tracing_map_cmp_num(field->size,
5025                                                              field->is_signed);
5026                         idx = tracing_map_add_key_field(map,
5027                                                         hist_field->offset,
5028                                                         cmp_fn);
5029                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5030                         idx = tracing_map_add_sum_field(map);
5031
5032                 if (idx < 0)
5033                         return idx;
5034
5035                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5036                         idx = tracing_map_add_var(map);
5037                         if (idx < 0)
5038                                 return idx;
5039                         hist_field->var.idx = idx;
5040                         hist_field->var.hist_data = hist_data;
5041                 }
5042         }
5043
5044         return 0;
5045 }
5046
5047 static struct hist_trigger_data *
5048 create_hist_data(unsigned int map_bits,
5049                  struct hist_trigger_attrs *attrs,
5050                  struct trace_event_file *file,
5051                  bool remove)
5052 {
5053         const struct tracing_map_ops *map_ops = NULL;
5054         struct hist_trigger_data *hist_data;
5055         int ret = 0;
5056
5057         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5058         if (!hist_data)
5059                 return ERR_PTR(-ENOMEM);
5060
5061         hist_data->attrs = attrs;
5062         hist_data->remove = remove;
5063         hist_data->event_file = file;
5064
5065         ret = parse_actions(hist_data);
5066         if (ret)
5067                 goto free;
5068
5069         ret = create_hist_fields(hist_data, file);
5070         if (ret)
5071                 goto free;
5072
5073         ret = create_sort_keys(hist_data);
5074         if (ret)
5075                 goto free;
5076
5077         map_ops = &hist_trigger_elt_data_ops;
5078
5079         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5080                                             map_ops, hist_data);
5081         if (IS_ERR(hist_data->map)) {
5082                 ret = PTR_ERR(hist_data->map);
5083                 hist_data->map = NULL;
5084                 goto free;
5085         }
5086
5087         ret = create_tracing_map_fields(hist_data);
5088         if (ret)
5089                 goto free;
5090  out:
5091         return hist_data;
5092  free:
5093         hist_data->attrs = NULL;
5094
5095         destroy_hist_data(hist_data);
5096
5097         hist_data = ERR_PTR(ret);
5098
5099         goto out;
5100 }
5101
5102 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5103                                     struct tracing_map_elt *elt,
5104                                     struct trace_buffer *buffer, void *rec,
5105                                     struct ring_buffer_event *rbe,
5106                                     u64 *var_ref_vals)
5107 {
5108         struct hist_elt_data *elt_data;
5109         struct hist_field *hist_field;
5110         unsigned int i, var_idx;
5111         u64 hist_val;
5112
5113         elt_data = elt->private_data;
5114         elt_data->var_ref_vals = var_ref_vals;
5115
5116         for_each_hist_val_field(i, hist_data) {
5117                 hist_field = hist_data->fields[i];
5118                 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5119                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5120                         var_idx = hist_field->var.idx;
5121
5122                         if (hist_field->flags &
5123                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5124                                 unsigned int str_start, var_str_idx, idx;
5125                                 char *str, *val_str;
5126                                 unsigned int size;
5127
5128                                 str_start = hist_data->n_field_var_str +
5129                                         hist_data->n_save_var_str;
5130                                 var_str_idx = hist_field->var_str_idx;
5131                                 idx = str_start + var_str_idx;
5132
5133                                 str = elt_data->field_var_str[idx];
5134                                 val_str = (char *)(uintptr_t)hist_val;
5135
5136                                 if (hist_field->flags & HIST_FIELD_FL_STRING) {
5137                                         size = min(hist_field->size, STR_VAR_LEN_MAX);
5138                                         strscpy(str, val_str, size);
5139                                 } else {
5140                                         char *stack_start = str + sizeof(unsigned long);
5141                                         int e;
5142
5143                                         e = stack_trace_save((void *)stack_start,
5144                                                              HIST_STACKTRACE_DEPTH,
5145                                                              HIST_STACKTRACE_SKIP);
5146                                         if (e < HIST_STACKTRACE_DEPTH - 1)
5147                                                 ((unsigned long *)stack_start)[e] = 0;
5148                                         *((unsigned long *)str) = e;
5149                                 }
5150                                 hist_val = (u64)(uintptr_t)str;
5151                         }
5152                         tracing_map_set_var(elt, var_idx, hist_val);
5153                         continue;
5154                 }
5155                 tracing_map_update_sum(elt, i, hist_val);
5156         }
5157
5158         for_each_hist_key_field(i, hist_data) {
5159                 hist_field = hist_data->fields[i];
5160                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5161                         hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5162                         var_idx = hist_field->var.idx;
5163                         tracing_map_set_var(elt, var_idx, hist_val);
5164                 }
5165         }
5166
5167         update_field_vars(hist_data, elt, buffer, rbe, rec);
5168 }
5169
5170 static inline void add_to_key(char *compound_key, void *key,
5171                               struct hist_field *key_field, void *rec)
5172 {
5173         size_t size = key_field->size;
5174
5175         if (key_field->flags & HIST_FIELD_FL_STRING) {
5176                 struct ftrace_event_field *field;
5177
5178                 field = key_field->field;
5179                 if (field->filter_type == FILTER_DYN_STRING ||
5180                     field->filter_type == FILTER_RDYN_STRING)
5181                         size = *(u32 *)(rec + field->offset) >> 16;
5182                 else if (field->filter_type == FILTER_STATIC_STRING)
5183                         size = field->size;
5184
5185                 /* ensure NULL-termination */
5186                 if (size > key_field->size - 1)
5187                         size = key_field->size - 1;
5188
5189                 strncpy(compound_key + key_field->offset, (char *)key, size);
5190         } else
5191                 memcpy(compound_key + key_field->offset, key, size);
5192 }
5193
5194 static void
5195 hist_trigger_actions(struct hist_trigger_data *hist_data,
5196                      struct tracing_map_elt *elt,
5197                      struct trace_buffer *buffer, void *rec,
5198                      struct ring_buffer_event *rbe, void *key,
5199                      u64 *var_ref_vals)
5200 {
5201         struct action_data *data;
5202         unsigned int i;
5203
5204         for (i = 0; i < hist_data->n_actions; i++) {
5205                 data = hist_data->actions[i];
5206                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5207         }
5208 }
5209
5210 static void event_hist_trigger(struct event_trigger_data *data,
5211                                struct trace_buffer *buffer, void *rec,
5212                                struct ring_buffer_event *rbe)
5213 {
5214         struct hist_trigger_data *hist_data = data->private_data;
5215         bool use_compound_key = (hist_data->n_keys > 1);
5216         unsigned long entries[HIST_STACKTRACE_DEPTH];
5217         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5218         char compound_key[HIST_KEY_SIZE_MAX];
5219         struct tracing_map_elt *elt = NULL;
5220         struct hist_field *key_field;
5221         u64 field_contents;
5222         void *key = NULL;
5223         unsigned int i;
5224
5225         if (unlikely(!rbe))
5226                 return;
5227
5228         memset(compound_key, 0, hist_data->key_size);
5229
5230         for_each_hist_key_field(i, hist_data) {
5231                 key_field = hist_data->fields[i];
5232
5233                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5234                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5235                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5236                                          HIST_STACKTRACE_SKIP);
5237                         key = entries;
5238                 } else {
5239                         field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5240                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5241                                 key = (void *)(unsigned long)field_contents;
5242                                 use_compound_key = true;
5243                         } else
5244                                 key = (void *)&field_contents;
5245                 }
5246
5247                 if (use_compound_key)
5248                         add_to_key(compound_key, key, key_field, rec);
5249         }
5250
5251         if (use_compound_key)
5252                 key = compound_key;
5253
5254         if (hist_data->n_var_refs &&
5255             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5256                 return;
5257
5258         elt = tracing_map_insert(hist_data->map, key);
5259         if (!elt)
5260                 return;
5261
5262         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5263
5264         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5265                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5266 }
5267
5268 static void hist_trigger_stacktrace_print(struct seq_file *m,
5269                                           unsigned long *stacktrace_entries,
5270                                           unsigned int max_entries)
5271 {
5272         unsigned int spaces = 8;
5273         unsigned int i;
5274
5275         for (i = 0; i < max_entries; i++) {
5276                 if (!stacktrace_entries[i])
5277                         return;
5278
5279                 seq_printf(m, "%*c", 1 + spaces, ' ');
5280                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5281         }
5282 }
5283
5284 static void hist_trigger_print_key(struct seq_file *m,
5285                                    struct hist_trigger_data *hist_data,
5286                                    void *key,
5287                                    struct tracing_map_elt *elt)
5288 {
5289         struct hist_field *key_field;
5290         bool multiline = false;
5291         const char *field_name;
5292         unsigned int i;
5293         u64 uval;
5294
5295         seq_puts(m, "{ ");
5296
5297         for_each_hist_key_field(i, hist_data) {
5298                 key_field = hist_data->fields[i];
5299
5300                 if (i > hist_data->n_vals)
5301                         seq_puts(m, ", ");
5302
5303                 field_name = hist_field_name(key_field, 0);
5304
5305                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5306                         uval = *(u64 *)(key + key_field->offset);
5307                         seq_printf(m, "%s: %llx", field_name, uval);
5308                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5309                         uval = *(u64 *)(key + key_field->offset);
5310                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5311                                    uval, (void *)(uintptr_t)uval);
5312                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5313                         uval = *(u64 *)(key + key_field->offset);
5314                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5315                                    uval, (void *)(uintptr_t)uval);
5316                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5317                         struct hist_elt_data *elt_data = elt->private_data;
5318                         char *comm;
5319
5320                         if (WARN_ON_ONCE(!elt_data))
5321                                 return;
5322
5323                         comm = elt_data->comm;
5324
5325                         uval = *(u64 *)(key + key_field->offset);
5326                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5327                                    comm, uval);
5328                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5329                         const char *syscall_name;
5330
5331                         uval = *(u64 *)(key + key_field->offset);
5332                         syscall_name = get_syscall_name(uval);
5333                         if (!syscall_name)
5334                                 syscall_name = "unknown_syscall";
5335
5336                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5337                                    syscall_name, uval);
5338                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5339                         seq_puts(m, "stacktrace:\n");
5340                         hist_trigger_stacktrace_print(m,
5341                                                       key + key_field->offset,
5342                                                       HIST_STACKTRACE_DEPTH);
5343                         multiline = true;
5344                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5345                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5346                                    *(u64 *)(key + key_field->offset));
5347                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5348                         unsigned long buckets = key_field->buckets;
5349                         uval = *(u64 *)(key + key_field->offset);
5350                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5351                                    uval, uval + buckets -1);
5352                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5353                         seq_printf(m, "%s: %-50s", field_name,
5354                                    (char *)(key + key_field->offset));
5355                 } else {
5356                         uval = *(u64 *)(key + key_field->offset);
5357                         seq_printf(m, "%s: %10llu", field_name, uval);
5358                 }
5359         }
5360
5361         if (!multiline)
5362                 seq_puts(m, " ");
5363
5364         seq_puts(m, "}");
5365 }
5366
5367 /* Get the 100 times of the percentage of @val in @total */
5368 static inline unsigned int __get_percentage(u64 val, u64 total)
5369 {
5370         if (!total)
5371                 goto div0;
5372
5373         if (val < (U64_MAX / 10000))
5374                 return (unsigned int)div64_ul(val * 10000, total);
5375
5376         total = div64_u64(total, 10000);
5377         if (!total)
5378                 goto div0;
5379
5380         return (unsigned int)div64_ul(val, total);
5381 div0:
5382         return val ? UINT_MAX : 0;
5383 }
5384
5385 #define BAR_CHAR '#'
5386
5387 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5388 {
5389         unsigned int len = __get_percentage(val, max);
5390         int i;
5391
5392         if (len == UINT_MAX) {
5393                 snprintf(buf, size, "[ERROR]");
5394                 return buf;
5395         }
5396
5397         len = len * size / 10000;
5398         for (i = 0; i < len && i < size; i++)
5399                 buf[i] = BAR_CHAR;
5400         while (i < size)
5401                 buf[i++] = ' ';
5402         buf[size] = '\0';
5403
5404         return buf;
5405 }
5406
5407 struct hist_val_stat {
5408         u64 max;
5409         u64 total;
5410 };
5411
5412 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5413                                    const char *field_name, unsigned long flags,
5414                                    struct hist_val_stat *stats,
5415                                    struct tracing_map_elt *elt)
5416 {
5417         u64 val = tracing_map_read_sum(elt, idx);
5418         unsigned int pc;
5419         char bar[21];
5420
5421         if (flags & HIST_FIELD_FL_PERCENT) {
5422                 pc = __get_percentage(val, stats[idx].total);
5423                 if (pc == UINT_MAX)
5424                         seq_printf(m, " %s (%%):[ERROR]", field_name);
5425                 else
5426                         seq_printf(m, " %s (%%): %3u.%02u", field_name,
5427                                         pc / 100, pc % 100);
5428         } else if (flags & HIST_FIELD_FL_GRAPH) {
5429                 seq_printf(m, " %s: %20s", field_name,
5430                            __fill_bar_str(bar, 20, val, stats[idx].max));
5431         } else if (flags & HIST_FIELD_FL_HEX) {
5432                 seq_printf(m, " %s: %10llx", field_name, val);
5433         } else {
5434                 seq_printf(m, " %s: %10llu", field_name, val);
5435         }
5436 }
5437
5438 static void hist_trigger_entry_print(struct seq_file *m,
5439                                      struct hist_trigger_data *hist_data,
5440                                      struct hist_val_stat *stats,
5441                                      void *key,
5442                                      struct tracing_map_elt *elt)
5443 {
5444         const char *field_name;
5445         unsigned int i = HITCOUNT_IDX;
5446         unsigned long flags;
5447
5448         hist_trigger_print_key(m, hist_data, key, elt);
5449
5450         /* At first, show the raw hitcount if !nohitcount */
5451         if (!hist_data->attrs->no_hitcount)
5452                 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5453
5454         for (i = 1; i < hist_data->n_vals; i++) {
5455                 field_name = hist_field_name(hist_data->fields[i], 0);
5456                 flags = hist_data->fields[i]->flags;
5457                 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5458                         continue;
5459
5460                 seq_puts(m, " ");
5461                 hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5462         }
5463
5464         print_actions(m, hist_data, elt);
5465
5466         seq_puts(m, "\n");
5467 }
5468
5469 static int print_entries(struct seq_file *m,
5470                          struct hist_trigger_data *hist_data)
5471 {
5472         struct tracing_map_sort_entry **sort_entries = NULL;
5473         struct tracing_map *map = hist_data->map;
5474         int i, j, n_entries;
5475         struct hist_val_stat *stats = NULL;
5476         u64 val;
5477
5478         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5479                                              hist_data->n_sort_keys,
5480                                              &sort_entries);
5481         if (n_entries < 0)
5482                 return n_entries;
5483
5484         /* Calculate the max and the total for each field if needed. */
5485         for (j = 0; j < hist_data->n_vals; j++) {
5486                 if (!(hist_data->fields[j]->flags &
5487                         (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5488                         continue;
5489                 if (!stats) {
5490                         stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5491                                        GFP_KERNEL);
5492                         if (!stats) {
5493                                 n_entries = -ENOMEM;
5494                                 goto out;
5495                         }
5496                 }
5497                 for (i = 0; i < n_entries; i++) {
5498                         val = tracing_map_read_sum(sort_entries[i]->elt, j);
5499                         stats[j].total += val;
5500                         if (stats[j].max < val)
5501                                 stats[j].max = val;
5502                 }
5503         }
5504
5505         for (i = 0; i < n_entries; i++)
5506                 hist_trigger_entry_print(m, hist_data, stats,
5507                                          sort_entries[i]->key,
5508                                          sort_entries[i]->elt);
5509
5510         kfree(stats);
5511 out:
5512         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5513
5514         return n_entries;
5515 }
5516
5517 static void hist_trigger_show(struct seq_file *m,
5518                               struct event_trigger_data *data, int n)
5519 {
5520         struct hist_trigger_data *hist_data;
5521         int n_entries;
5522
5523         if (n > 0)
5524                 seq_puts(m, "\n\n");
5525
5526         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5527         data->ops->print(m, data);
5528         seq_puts(m, "#\n\n");
5529
5530         hist_data = data->private_data;
5531         n_entries = print_entries(m, hist_data);
5532         if (n_entries < 0)
5533                 n_entries = 0;
5534
5535         track_data_snapshot_print(m, hist_data);
5536
5537         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5538                    (u64)atomic64_read(&hist_data->map->hits),
5539                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5540 }
5541
5542 static int hist_show(struct seq_file *m, void *v)
5543 {
5544         struct event_trigger_data *data;
5545         struct trace_event_file *event_file;
5546         int n = 0, ret = 0;
5547
5548         mutex_lock(&event_mutex);
5549
5550         event_file = event_file_data(m->private);
5551         if (unlikely(!event_file)) {
5552                 ret = -ENODEV;
5553                 goto out_unlock;
5554         }
5555
5556         list_for_each_entry(data, &event_file->triggers, list) {
5557                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5558                         hist_trigger_show(m, data, n++);
5559         }
5560
5561  out_unlock:
5562         mutex_unlock(&event_mutex);
5563
5564         return ret;
5565 }
5566
5567 static int event_hist_open(struct inode *inode, struct file *file)
5568 {
5569         int ret;
5570
5571         ret = security_locked_down(LOCKDOWN_TRACEFS);
5572         if (ret)
5573                 return ret;
5574
5575         return single_open(file, hist_show, file);
5576 }
5577
5578 const struct file_operations event_hist_fops = {
5579         .open = event_hist_open,
5580         .read = seq_read,
5581         .llseek = seq_lseek,
5582         .release = single_release,
5583 };
5584
5585 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5586 static void hist_field_debug_show_flags(struct seq_file *m,
5587                                         unsigned long flags)
5588 {
5589         seq_puts(m, "      flags:\n");
5590
5591         if (flags & HIST_FIELD_FL_KEY)
5592                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5593         else if (flags & HIST_FIELD_FL_HITCOUNT)
5594                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5595         else if (flags & HIST_FIELD_FL_VAR)
5596                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5597         else if (flags & HIST_FIELD_FL_VAR_REF)
5598                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5599         else
5600                 seq_puts(m, "        VAL: normal u64 value\n");
5601
5602         if (flags & HIST_FIELD_FL_ALIAS)
5603                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5604         else if (flags & HIST_FIELD_FL_CONST)
5605                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5606 }
5607
5608 static int hist_field_debug_show(struct seq_file *m,
5609                                  struct hist_field *field, unsigned long flags)
5610 {
5611         if ((field->flags & flags) != flags) {
5612                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5613                 return -EINVAL;
5614         }
5615
5616         hist_field_debug_show_flags(m, field->flags);
5617         if (field->field)
5618                 seq_printf(m, "      ftrace_event_field name: %s\n",
5619                            field->field->name);
5620
5621         if (field->flags & HIST_FIELD_FL_VAR) {
5622                 seq_printf(m, "      var.name: %s\n", field->var.name);
5623                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5624                            field->var.idx);
5625         }
5626
5627         if (field->flags & HIST_FIELD_FL_CONST)
5628                 seq_printf(m, "      constant: %llu\n", field->constant);
5629
5630         if (field->flags & HIST_FIELD_FL_ALIAS)
5631                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5632                            field->var_ref_idx);
5633
5634         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5635                 seq_printf(m, "      name: %s\n", field->name);
5636                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5637                            field->var.idx);
5638                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5639                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5640                            field->var_ref_idx);
5641                 if (field->system)
5642                         seq_printf(m, "      system: %s\n", field->system);
5643                 if (field->event_name)
5644                         seq_printf(m, "      event_name: %s\n", field->event_name);
5645         }
5646
5647         seq_printf(m, "      type: %s\n", field->type);
5648         seq_printf(m, "      size: %u\n", field->size);
5649         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5650
5651         return 0;
5652 }
5653
5654 static int field_var_debug_show(struct seq_file *m,
5655                                 struct field_var *field_var, unsigned int i,
5656                                 bool save_vars)
5657 {
5658         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5659         struct hist_field *field;
5660         int ret = 0;
5661
5662         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5663
5664         field = field_var->var;
5665
5666         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5667
5668         hist_field_debug_show_flags(m, field->flags);
5669         seq_printf(m, "      var.name: %s\n", field->var.name);
5670         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5671                    field->var.idx);
5672
5673         field = field_var->val;
5674
5675         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5676         if (field->field)
5677                 seq_printf(m, "      ftrace_event_field name: %s\n",
5678                            field->field->name);
5679         else {
5680                 ret = -EINVAL;
5681                 goto out;
5682         }
5683
5684         seq_printf(m, "      type: %s\n", field->type);
5685         seq_printf(m, "      size: %u\n", field->size);
5686         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5687 out:
5688         return ret;
5689 }
5690
5691 static int hist_action_debug_show(struct seq_file *m,
5692                                   struct action_data *data, int i)
5693 {
5694         int ret = 0;
5695
5696         if (data->handler == HANDLER_ONMAX ||
5697             data->handler == HANDLER_ONCHANGE) {
5698                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5699                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5700                                             HIST_FIELD_FL_VAR_REF);
5701                 if (ret)
5702                         goto out;
5703
5704                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5705                 ret = hist_field_debug_show(m, data->track_data.track_var,
5706                                             HIST_FIELD_FL_VAR);
5707                 if (ret)
5708                         goto out;
5709         }
5710
5711         if (data->handler == HANDLER_ONMATCH) {
5712                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5713                            i, data->match_data.event_system);
5714                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5715                            i, data->match_data.event);
5716         }
5717 out:
5718         return ret;
5719 }
5720
5721 static int hist_actions_debug_show(struct seq_file *m,
5722                                    struct hist_trigger_data *hist_data)
5723 {
5724         int i, ret = 0;
5725
5726         if (hist_data->n_actions)
5727                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5728
5729         for (i = 0; i < hist_data->n_actions; i++) {
5730                 struct action_data *action = hist_data->actions[i];
5731
5732                 ret = hist_action_debug_show(m, action, i);
5733                 if (ret)
5734                         goto out;
5735         }
5736
5737         if (hist_data->n_save_vars)
5738                 seq_puts(m, "\n  save action variables (save() params):\n");
5739
5740         for (i = 0; i < hist_data->n_save_vars; i++) {
5741                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5742                 if (ret)
5743                         goto out;
5744         }
5745 out:
5746         return ret;
5747 }
5748
5749 static void hist_trigger_debug_show(struct seq_file *m,
5750                                     struct event_trigger_data *data, int n)
5751 {
5752         struct hist_trigger_data *hist_data;
5753         int i, ret;
5754
5755         if (n > 0)
5756                 seq_puts(m, "\n\n");
5757
5758         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5759         data->ops->print(m, data);
5760         seq_puts(m, "#\n\n");
5761
5762         hist_data = data->private_data;
5763
5764         seq_printf(m, "hist_data: %p\n\n", hist_data);
5765         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5766         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5767         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5768
5769         seq_puts(m, "\n  val fields:\n\n");
5770
5771         seq_puts(m, "    hist_data->fields[0]:\n");
5772         ret = hist_field_debug_show(m, hist_data->fields[0],
5773                                     HIST_FIELD_FL_HITCOUNT);
5774         if (ret)
5775                 return;
5776
5777         for (i = 1; i < hist_data->n_vals; i++) {
5778                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5779                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5780                 if (ret)
5781                         return;
5782         }
5783
5784         seq_puts(m, "\n  key fields:\n");
5785
5786         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5787                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5788                 ret = hist_field_debug_show(m, hist_data->fields[i],
5789                                             HIST_FIELD_FL_KEY);
5790                 if (ret)
5791                         return;
5792         }
5793
5794         if (hist_data->n_var_refs)
5795                 seq_puts(m, "\n  variable reference fields:\n");
5796
5797         for (i = 0; i < hist_data->n_var_refs; i++) {
5798                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5799                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5800                                             HIST_FIELD_FL_VAR_REF);
5801                 if (ret)
5802                         return;
5803         }
5804
5805         if (hist_data->n_field_vars)
5806                 seq_puts(m, "\n  field variables:\n");
5807
5808         for (i = 0; i < hist_data->n_field_vars; i++) {
5809                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5810                 if (ret)
5811                         return;
5812         }
5813
5814         ret = hist_actions_debug_show(m, hist_data);
5815         if (ret)
5816                 return;
5817 }
5818
5819 static int hist_debug_show(struct seq_file *m, void *v)
5820 {
5821         struct event_trigger_data *data;
5822         struct trace_event_file *event_file;
5823         int n = 0, ret = 0;
5824
5825         mutex_lock(&event_mutex);
5826
5827         event_file = event_file_data(m->private);
5828         if (unlikely(!event_file)) {
5829                 ret = -ENODEV;
5830                 goto out_unlock;
5831         }
5832
5833         list_for_each_entry(data, &event_file->triggers, list) {
5834                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5835                         hist_trigger_debug_show(m, data, n++);
5836         }
5837
5838  out_unlock:
5839         mutex_unlock(&event_mutex);
5840
5841         return ret;
5842 }
5843
5844 static int event_hist_debug_open(struct inode *inode, struct file *file)
5845 {
5846         int ret;
5847
5848         ret = security_locked_down(LOCKDOWN_TRACEFS);
5849         if (ret)
5850                 return ret;
5851
5852         return single_open(file, hist_debug_show, file);
5853 }
5854
5855 const struct file_operations event_hist_debug_fops = {
5856         .open = event_hist_debug_open,
5857         .read = seq_read,
5858         .llseek = seq_lseek,
5859         .release = single_release,
5860 };
5861 #endif
5862
5863 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5864 {
5865         const char *field_name = hist_field_name(hist_field, 0);
5866
5867         if (hist_field->var.name)
5868                 seq_printf(m, "%s=", hist_field->var.name);
5869
5870         if (hist_field->flags & HIST_FIELD_FL_CPU)
5871                 seq_puts(m, "common_cpu");
5872         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5873                 seq_printf(m, "%llu", hist_field->constant);
5874         else if (field_name) {
5875                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5876                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5877                         seq_putc(m, '$');
5878                 seq_printf(m, "%s", field_name);
5879         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5880                 seq_puts(m, "common_timestamp");
5881
5882         if (hist_field->flags) {
5883                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5884                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5885                         const char *flags = get_hist_field_flags(hist_field);
5886
5887                         if (flags)
5888                                 seq_printf(m, ".%s", flags);
5889                 }
5890         }
5891         if (hist_field->buckets)
5892                 seq_printf(m, "=%ld", hist_field->buckets);
5893 }
5894
5895 static int event_hist_trigger_print(struct seq_file *m,
5896                                     struct event_trigger_data *data)
5897 {
5898         struct hist_trigger_data *hist_data = data->private_data;
5899         struct hist_field *field;
5900         bool have_var = false;
5901         bool show_val = false;
5902         unsigned int i;
5903
5904         seq_puts(m, HIST_PREFIX);
5905
5906         if (data->name)
5907                 seq_printf(m, "%s:", data->name);
5908
5909         seq_puts(m, "keys=");
5910
5911         for_each_hist_key_field(i, hist_data) {
5912                 field = hist_data->fields[i];
5913
5914                 if (i > hist_data->n_vals)
5915                         seq_puts(m, ",");
5916
5917                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5918                         seq_puts(m, "stacktrace");
5919                 else
5920                         hist_field_print(m, field);
5921         }
5922
5923         seq_puts(m, ":vals=");
5924
5925         for_each_hist_val_field(i, hist_data) {
5926                 field = hist_data->fields[i];
5927                 if (field->flags & HIST_FIELD_FL_VAR) {
5928                         have_var = true;
5929                         continue;
5930                 }
5931
5932                 if (i == HITCOUNT_IDX) {
5933                         if (hist_data->attrs->no_hitcount)
5934                                 continue;
5935                         seq_puts(m, "hitcount");
5936                 } else {
5937                         if (show_val)
5938                                 seq_puts(m, ",");
5939                         hist_field_print(m, field);
5940                 }
5941                 show_val = true;
5942         }
5943
5944         if (have_var) {
5945                 unsigned int n = 0;
5946
5947                 seq_puts(m, ":");
5948
5949                 for_each_hist_val_field(i, hist_data) {
5950                         field = hist_data->fields[i];
5951
5952                         if (field->flags & HIST_FIELD_FL_VAR) {
5953                                 if (n++)
5954                                         seq_puts(m, ",");
5955                                 hist_field_print(m, field);
5956                         }
5957                 }
5958         }
5959
5960         seq_puts(m, ":sort=");
5961
5962         for (i = 0; i < hist_data->n_sort_keys; i++) {
5963                 struct tracing_map_sort_key *sort_key;
5964                 unsigned int idx, first_key_idx;
5965
5966                 /* skip VAR vals */
5967                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5968
5969                 sort_key = &hist_data->sort_keys[i];
5970                 idx = sort_key->field_idx;
5971
5972                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5973                         return -EINVAL;
5974
5975                 if (i > 0)
5976                         seq_puts(m, ",");
5977
5978                 if (idx == HITCOUNT_IDX)
5979                         seq_puts(m, "hitcount");
5980                 else {
5981                         if (idx >= first_key_idx)
5982                                 idx += hist_data->n_vars;
5983                         hist_field_print(m, hist_data->fields[idx]);
5984                 }
5985
5986                 if (sort_key->descending)
5987                         seq_puts(m, ".descending");
5988         }
5989         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5990         if (hist_data->enable_timestamps)
5991                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5992         if (hist_data->attrs->no_hitcount)
5993                 seq_puts(m, ":nohitcount");
5994
5995         print_actions_spec(m, hist_data);
5996
5997         if (data->filter_str)
5998                 seq_printf(m, " if %s", data->filter_str);
5999
6000         if (data->paused)
6001                 seq_puts(m, " [paused]");
6002         else
6003                 seq_puts(m, " [active]");
6004
6005         seq_putc(m, '\n');
6006
6007         return 0;
6008 }
6009
6010 static int event_hist_trigger_init(struct event_trigger_data *data)
6011 {
6012         struct hist_trigger_data *hist_data = data->private_data;
6013
6014         if (!data->ref && hist_data->attrs->name)
6015                 save_named_trigger(hist_data->attrs->name, data);
6016
6017         data->ref++;
6018
6019         return 0;
6020 }
6021
6022 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6023 {
6024         struct trace_event_file *file;
6025         unsigned int i;
6026         char *cmd;
6027         int ret;
6028
6029         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6030                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6031                 cmd = hist_data->field_var_hists[i]->cmd;
6032                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6033                                                "!hist", "hist", cmd);
6034                 WARN_ON_ONCE(ret < 0);
6035         }
6036 }
6037
6038 static void event_hist_trigger_free(struct event_trigger_data *data)
6039 {
6040         struct hist_trigger_data *hist_data = data->private_data;
6041
6042         if (WARN_ON_ONCE(data->ref <= 0))
6043                 return;
6044
6045         data->ref--;
6046         if (!data->ref) {
6047                 if (data->name)
6048                         del_named_trigger(data);
6049
6050                 trigger_data_free(data);
6051
6052                 remove_hist_vars(hist_data);
6053
6054                 unregister_field_var_hists(hist_data);
6055
6056                 destroy_hist_data(hist_data);
6057         }
6058 }
6059
6060 static struct event_trigger_ops event_hist_trigger_ops = {
6061         .trigger                = event_hist_trigger,
6062         .print                  = event_hist_trigger_print,
6063         .init                   = event_hist_trigger_init,
6064         .free                   = event_hist_trigger_free,
6065 };
6066
6067 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6068 {
6069         data->ref++;
6070
6071         save_named_trigger(data->named_data->name, data);
6072
6073         event_hist_trigger_init(data->named_data);
6074
6075         return 0;
6076 }
6077
6078 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6079 {
6080         if (WARN_ON_ONCE(data->ref <= 0))
6081                 return;
6082
6083         event_hist_trigger_free(data->named_data);
6084
6085         data->ref--;
6086         if (!data->ref) {
6087                 del_named_trigger(data);
6088                 trigger_data_free(data);
6089         }
6090 }
6091
6092 static struct event_trigger_ops event_hist_trigger_named_ops = {
6093         .trigger                = event_hist_trigger,
6094         .print                  = event_hist_trigger_print,
6095         .init                   = event_hist_trigger_named_init,
6096         .free                   = event_hist_trigger_named_free,
6097 };
6098
6099 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6100                                                             char *param)
6101 {
6102         return &event_hist_trigger_ops;
6103 }
6104
6105 static void hist_clear(struct event_trigger_data *data)
6106 {
6107         struct hist_trigger_data *hist_data = data->private_data;
6108
6109         if (data->name)
6110                 pause_named_trigger(data);
6111
6112         tracepoint_synchronize_unregister();
6113
6114         tracing_map_clear(hist_data->map);
6115
6116         if (data->name)
6117                 unpause_named_trigger(data);
6118 }
6119
6120 static bool compatible_field(struct ftrace_event_field *field,
6121                              struct ftrace_event_field *test_field)
6122 {
6123         if (field == test_field)
6124                 return true;
6125         if (field == NULL || test_field == NULL)
6126                 return false;
6127         if (strcmp(field->name, test_field->name) != 0)
6128                 return false;
6129         if (strcmp(field->type, test_field->type) != 0)
6130                 return false;
6131         if (field->size != test_field->size)
6132                 return false;
6133         if (field->is_signed != test_field->is_signed)
6134                 return false;
6135
6136         return true;
6137 }
6138
6139 static bool hist_trigger_match(struct event_trigger_data *data,
6140                                struct event_trigger_data *data_test,
6141                                struct event_trigger_data *named_data,
6142                                bool ignore_filter)
6143 {
6144         struct tracing_map_sort_key *sort_key, *sort_key_test;
6145         struct hist_trigger_data *hist_data, *hist_data_test;
6146         struct hist_field *key_field, *key_field_test;
6147         unsigned int i;
6148
6149         if (named_data && (named_data != data_test) &&
6150             (named_data != data_test->named_data))
6151                 return false;
6152
6153         if (!named_data && is_named_trigger(data_test))
6154                 return false;
6155
6156         hist_data = data->private_data;
6157         hist_data_test = data_test->private_data;
6158
6159         if (hist_data->n_vals != hist_data_test->n_vals ||
6160             hist_data->n_fields != hist_data_test->n_fields ||
6161             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6162                 return false;
6163
6164         if (!ignore_filter) {
6165                 if ((data->filter_str && !data_test->filter_str) ||
6166                    (!data->filter_str && data_test->filter_str))
6167                         return false;
6168         }
6169
6170         for_each_hist_field(i, hist_data) {
6171                 key_field = hist_data->fields[i];
6172                 key_field_test = hist_data_test->fields[i];
6173
6174                 if (key_field->flags != key_field_test->flags)
6175                         return false;
6176                 if (!compatible_field(key_field->field, key_field_test->field))
6177                         return false;
6178                 if (key_field->offset != key_field_test->offset)
6179                         return false;
6180                 if (key_field->size != key_field_test->size)
6181                         return false;
6182                 if (key_field->is_signed != key_field_test->is_signed)
6183                         return false;
6184                 if (!!key_field->var.name != !!key_field_test->var.name)
6185                         return false;
6186                 if (key_field->var.name &&
6187                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6188                         return false;
6189         }
6190
6191         for (i = 0; i < hist_data->n_sort_keys; i++) {
6192                 sort_key = &hist_data->sort_keys[i];
6193                 sort_key_test = &hist_data_test->sort_keys[i];
6194
6195                 if (sort_key->field_idx != sort_key_test->field_idx ||
6196                     sort_key->descending != sort_key_test->descending)
6197                         return false;
6198         }
6199
6200         if (!ignore_filter && data->filter_str &&
6201             (strcmp(data->filter_str, data_test->filter_str) != 0))
6202                 return false;
6203
6204         if (!actions_match(hist_data, hist_data_test))
6205                 return false;
6206
6207         return true;
6208 }
6209
6210 static bool existing_hist_update_only(char *glob,
6211                                       struct event_trigger_data *data,
6212                                       struct trace_event_file *file)
6213 {
6214         struct hist_trigger_data *hist_data = data->private_data;
6215         struct event_trigger_data *test, *named_data = NULL;
6216         bool updated = false;
6217
6218         if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6219             !hist_data->attrs->clear)
6220                 goto out;
6221
6222         if (hist_data->attrs->name) {
6223                 named_data = find_named_trigger(hist_data->attrs->name);
6224                 if (named_data) {
6225                         if (!hist_trigger_match(data, named_data, named_data,
6226                                                 true))
6227                                 goto out;
6228                 }
6229         }
6230
6231         if (hist_data->attrs->name && !named_data)
6232                 goto out;
6233
6234         list_for_each_entry(test, &file->triggers, list) {
6235                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6236                         if (!hist_trigger_match(data, test, named_data, false))
6237                                 continue;
6238                         if (hist_data->attrs->pause)
6239                                 test->paused = true;
6240                         else if (hist_data->attrs->cont)
6241                                 test->paused = false;
6242                         else if (hist_data->attrs->clear)
6243                                 hist_clear(test);
6244                         updated = true;
6245                         goto out;
6246                 }
6247         }
6248  out:
6249         return updated;
6250 }
6251
6252 static int hist_register_trigger(char *glob,
6253                                  struct event_trigger_data *data,
6254                                  struct trace_event_file *file)
6255 {
6256         struct hist_trigger_data *hist_data = data->private_data;
6257         struct event_trigger_data *test, *named_data = NULL;
6258         struct trace_array *tr = file->tr;
6259         int ret = 0;
6260
6261         if (hist_data->attrs->name) {
6262                 named_data = find_named_trigger(hist_data->attrs->name);
6263                 if (named_data) {
6264                         if (!hist_trigger_match(data, named_data, named_data,
6265                                                 true)) {
6266                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6267                                 ret = -EINVAL;
6268                                 goto out;
6269                         }
6270                 }
6271         }
6272
6273         if (hist_data->attrs->name && !named_data)
6274                 goto new;
6275
6276         lockdep_assert_held(&event_mutex);
6277
6278         list_for_each_entry(test, &file->triggers, list) {
6279                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6280                         if (hist_trigger_match(data, test, named_data, false)) {
6281                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6282                                 ret = -EEXIST;
6283                                 goto out;
6284                         }
6285                 }
6286         }
6287  new:
6288         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6289                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6290                 ret = -ENOENT;
6291                 goto out;
6292         }
6293
6294         if (hist_data->attrs->pause)
6295                 data->paused = true;
6296
6297         if (named_data) {
6298                 data->private_data = named_data->private_data;
6299                 set_named_trigger_data(data, named_data);
6300                 data->ops = &event_hist_trigger_named_ops;
6301         }
6302
6303         if (data->ops->init) {
6304                 ret = data->ops->init(data);
6305                 if (ret < 0)
6306                         goto out;
6307         }
6308
6309         if (hist_data->enable_timestamps) {
6310                 char *clock = hist_data->attrs->clock;
6311
6312                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6313                 if (ret) {
6314                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6315                         goto out;
6316                 }
6317
6318                 tracing_set_filter_buffering(file->tr, true);
6319         }
6320
6321         if (named_data)
6322                 destroy_hist_data(hist_data);
6323  out:
6324         return ret;
6325 }
6326
6327 static int hist_trigger_enable(struct event_trigger_data *data,
6328                                struct trace_event_file *file)
6329 {
6330         int ret = 0;
6331
6332         list_add_tail_rcu(&data->list, &file->triggers);
6333
6334         update_cond_flag(file);
6335
6336         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6337                 list_del_rcu(&data->list);
6338                 update_cond_flag(file);
6339                 ret--;
6340         }
6341
6342         return ret;
6343 }
6344
6345 static bool have_hist_trigger_match(struct event_trigger_data *data,
6346                                     struct trace_event_file *file)
6347 {
6348         struct hist_trigger_data *hist_data = data->private_data;
6349         struct event_trigger_data *test, *named_data = NULL;
6350         bool match = false;
6351
6352         lockdep_assert_held(&event_mutex);
6353
6354         if (hist_data->attrs->name)
6355                 named_data = find_named_trigger(hist_data->attrs->name);
6356
6357         list_for_each_entry(test, &file->triggers, list) {
6358                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6359                         if (hist_trigger_match(data, test, named_data, false)) {
6360                                 match = true;
6361                                 break;
6362                         }
6363                 }
6364         }
6365
6366         return match;
6367 }
6368
6369 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6370                                     struct trace_event_file *file)
6371 {
6372         struct hist_trigger_data *hist_data = data->private_data;
6373         struct event_trigger_data *test, *named_data = NULL;
6374
6375         lockdep_assert_held(&event_mutex);
6376
6377         if (hist_data->attrs->name)
6378                 named_data = find_named_trigger(hist_data->attrs->name);
6379
6380         list_for_each_entry(test, &file->triggers, list) {
6381                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6382                         if (!hist_trigger_match(data, test, named_data, false))
6383                                 continue;
6384                         hist_data = test->private_data;
6385                         if (check_var_refs(hist_data))
6386                                 return true;
6387                         break;
6388                 }
6389         }
6390
6391         return false;
6392 }
6393
6394 static void hist_unregister_trigger(char *glob,
6395                                     struct event_trigger_data *data,
6396                                     struct trace_event_file *file)
6397 {
6398         struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6399         struct hist_trigger_data *hist_data = data->private_data;
6400
6401         lockdep_assert_held(&event_mutex);
6402
6403         if (hist_data->attrs->name)
6404                 named_data = find_named_trigger(hist_data->attrs->name);
6405
6406         list_for_each_entry(iter, &file->triggers, list) {
6407                 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6408                         if (!hist_trigger_match(data, iter, named_data, false))
6409                                 continue;
6410                         test = iter;
6411                         list_del_rcu(&test->list);
6412                         trace_event_trigger_enable_disable(file, 0);
6413                         update_cond_flag(file);
6414                         break;
6415                 }
6416         }
6417
6418         if (test && test->ops->free)
6419                 test->ops->free(test);
6420
6421         if (hist_data->enable_timestamps) {
6422                 if (!hist_data->remove || test)
6423                         tracing_set_filter_buffering(file->tr, false);
6424         }
6425 }
6426
6427 static bool hist_file_check_refs(struct trace_event_file *file)
6428 {
6429         struct hist_trigger_data *hist_data;
6430         struct event_trigger_data *test;
6431
6432         lockdep_assert_held(&event_mutex);
6433
6434         list_for_each_entry(test, &file->triggers, list) {
6435                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6436                         hist_data = test->private_data;
6437                         if (check_var_refs(hist_data))
6438                                 return true;
6439                 }
6440         }
6441
6442         return false;
6443 }
6444
6445 static void hist_unreg_all(struct trace_event_file *file)
6446 {
6447         struct event_trigger_data *test, *n;
6448         struct hist_trigger_data *hist_data;
6449         struct synth_event *se;
6450         const char *se_name;
6451
6452         lockdep_assert_held(&event_mutex);
6453
6454         if (hist_file_check_refs(file))
6455                 return;
6456
6457         list_for_each_entry_safe(test, n, &file->triggers, list) {
6458                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6459                         hist_data = test->private_data;
6460                         list_del_rcu(&test->list);
6461                         trace_event_trigger_enable_disable(file, 0);
6462
6463                         se_name = trace_event_name(file->event_call);
6464                         se = find_synth_event(se_name);
6465                         if (se)
6466                                 se->ref--;
6467
6468                         update_cond_flag(file);
6469                         if (hist_data->enable_timestamps)
6470                                 tracing_set_filter_buffering(file->tr, false);
6471                         if (test->ops->free)
6472                                 test->ops->free(test);
6473                 }
6474         }
6475 }
6476
6477 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6478                                     struct trace_event_file *file,
6479                                     char *glob, char *cmd,
6480                                     char *param_and_filter)
6481 {
6482         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6483         struct event_trigger_data *trigger_data;
6484         struct hist_trigger_attrs *attrs;
6485         struct hist_trigger_data *hist_data;
6486         char *param, *filter, *p, *start;
6487         struct synth_event *se;
6488         const char *se_name;
6489         bool remove;
6490         int ret = 0;
6491
6492         lockdep_assert_held(&event_mutex);
6493
6494         if (WARN_ON(!glob))
6495                 return -EINVAL;
6496
6497         if (glob[0]) {
6498                 hist_err_clear();
6499                 last_cmd_set(file, param_and_filter);
6500         }
6501
6502         remove = event_trigger_check_remove(glob);
6503
6504         if (event_trigger_empty_param(param_and_filter))
6505                 return -EINVAL;
6506
6507         /*
6508          * separate the trigger from the filter (k:v [if filter])
6509          * allowing for whitespace in the trigger
6510          */
6511         p = param = param_and_filter;
6512         do {
6513                 p = strstr(p, "if");
6514                 if (!p)
6515                         break;
6516                 if (p == param_and_filter)
6517                         return -EINVAL;
6518                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6519                         p++;
6520                         continue;
6521                 }
6522                 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6523                         return -EINVAL;
6524                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6525                         p++;
6526                         continue;
6527                 }
6528                 break;
6529         } while (1);
6530
6531         if (!p)
6532                 filter = NULL;
6533         else {
6534                 *(p - 1) = '\0';
6535                 filter = strstrip(p);
6536                 param = strstrip(param);
6537         }
6538
6539         /*
6540          * To simplify arithmetic expression parsing, replace occurrences of
6541          * '.sym-offset' modifier with '.symXoffset'
6542          */
6543         start = strstr(param, ".sym-offset");
6544         while (start) {
6545                 *(start + 4) = 'X';
6546                 start = strstr(start + 11, ".sym-offset");
6547         }
6548
6549         attrs = parse_hist_trigger_attrs(file->tr, param);
6550         if (IS_ERR(attrs))
6551                 return PTR_ERR(attrs);
6552
6553         if (attrs->map_bits)
6554                 hist_trigger_bits = attrs->map_bits;
6555
6556         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6557         if (IS_ERR(hist_data)) {
6558                 destroy_hist_trigger_attrs(attrs);
6559                 return PTR_ERR(hist_data);
6560         }
6561
6562         trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6563         if (!trigger_data) {
6564                 ret = -ENOMEM;
6565                 goto out_free;
6566         }
6567
6568         ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6569         if (ret < 0)
6570                 goto out_free;
6571
6572         if (remove) {
6573                 if (!have_hist_trigger_match(trigger_data, file))
6574                         goto out_free;
6575
6576                 if (hist_trigger_check_refs(trigger_data, file)) {
6577                         ret = -EBUSY;
6578                         goto out_free;
6579                 }
6580
6581                 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6582                 se_name = trace_event_name(file->event_call);
6583                 se = find_synth_event(se_name);
6584                 if (se)
6585                         se->ref--;
6586                 ret = 0;
6587                 goto out_free;
6588         }
6589
6590         if (existing_hist_update_only(glob, trigger_data, file))
6591                 goto out_free;
6592
6593         ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6594         if (ret < 0)
6595                 goto out_free;
6596
6597         if (get_named_trigger_data(trigger_data))
6598                 goto enable;
6599
6600         if (has_hist_vars(hist_data))
6601                 save_hist_vars(hist_data);
6602
6603         ret = create_actions(hist_data);
6604         if (ret)
6605                 goto out_unreg;
6606
6607         ret = tracing_map_init(hist_data->map);
6608         if (ret)
6609                 goto out_unreg;
6610 enable:
6611         ret = hist_trigger_enable(trigger_data, file);
6612         if (ret)
6613                 goto out_unreg;
6614
6615         se_name = trace_event_name(file->event_call);
6616         se = find_synth_event(se_name);
6617         if (se)
6618                 se->ref++;
6619  out:
6620         if (ret == 0 && glob[0])
6621                 hist_err_clear();
6622
6623         return ret;
6624  out_unreg:
6625         event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6626  out_free:
6627         event_trigger_reset_filter(cmd_ops, trigger_data);
6628
6629         remove_hist_vars(hist_data);
6630
6631         kfree(trigger_data);
6632
6633         destroy_hist_data(hist_data);
6634         goto out;
6635 }
6636
6637 static struct event_command trigger_hist_cmd = {
6638         .name                   = "hist",
6639         .trigger_type           = ETT_EVENT_HIST,
6640         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6641         .parse                  = event_hist_trigger_parse,
6642         .reg                    = hist_register_trigger,
6643         .unreg                  = hist_unregister_trigger,
6644         .unreg_all              = hist_unreg_all,
6645         .get_trigger_ops        = event_hist_get_trigger_ops,
6646         .set_filter             = set_trigger_filter,
6647 };
6648
6649 __init int register_trigger_hist_cmd(void)
6650 {
6651         int ret;
6652
6653         ret = register_event_command(&trigger_hist_cmd);
6654         WARN_ON(ret < 0);
6655
6656         return ret;
6657 }
6658
6659 static void
6660 hist_enable_trigger(struct event_trigger_data *data,
6661                     struct trace_buffer *buffer,  void *rec,
6662                     struct ring_buffer_event *event)
6663 {
6664         struct enable_trigger_data *enable_data = data->private_data;
6665         struct event_trigger_data *test;
6666
6667         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6668                                 lockdep_is_held(&event_mutex)) {
6669                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6670                         if (enable_data->enable)
6671                                 test->paused = false;
6672                         else
6673                                 test->paused = true;
6674                 }
6675         }
6676 }
6677
6678 static void
6679 hist_enable_count_trigger(struct event_trigger_data *data,
6680                           struct trace_buffer *buffer,  void *rec,
6681                           struct ring_buffer_event *event)
6682 {
6683         if (!data->count)
6684                 return;
6685
6686         if (data->count != -1)
6687                 (data->count)--;
6688
6689         hist_enable_trigger(data, buffer, rec, event);
6690 }
6691
6692 static struct event_trigger_ops hist_enable_trigger_ops = {
6693         .trigger                = hist_enable_trigger,
6694         .print                  = event_enable_trigger_print,
6695         .init                   = event_trigger_init,
6696         .free                   = event_enable_trigger_free,
6697 };
6698
6699 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6700         .trigger                = hist_enable_count_trigger,
6701         .print                  = event_enable_trigger_print,
6702         .init                   = event_trigger_init,
6703         .free                   = event_enable_trigger_free,
6704 };
6705
6706 static struct event_trigger_ops hist_disable_trigger_ops = {
6707         .trigger                = hist_enable_trigger,
6708         .print                  = event_enable_trigger_print,
6709         .init                   = event_trigger_init,
6710         .free                   = event_enable_trigger_free,
6711 };
6712
6713 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6714         .trigger                = hist_enable_count_trigger,
6715         .print                  = event_enable_trigger_print,
6716         .init                   = event_trigger_init,
6717         .free                   = event_enable_trigger_free,
6718 };
6719
6720 static struct event_trigger_ops *
6721 hist_enable_get_trigger_ops(char *cmd, char *param)
6722 {
6723         struct event_trigger_ops *ops;
6724         bool enable;
6725
6726         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6727
6728         if (enable)
6729                 ops = param ? &hist_enable_count_trigger_ops :
6730                         &hist_enable_trigger_ops;
6731         else
6732                 ops = param ? &hist_disable_count_trigger_ops :
6733                         &hist_disable_trigger_ops;
6734
6735         return ops;
6736 }
6737
6738 static void hist_enable_unreg_all(struct trace_event_file *file)
6739 {
6740         struct event_trigger_data *test, *n;
6741
6742         list_for_each_entry_safe(test, n, &file->triggers, list) {
6743                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6744                         list_del_rcu(&test->list);
6745                         update_cond_flag(file);
6746                         trace_event_trigger_enable_disable(file, 0);
6747                         if (test->ops->free)
6748                                 test->ops->free(test);
6749                 }
6750         }
6751 }
6752
6753 static struct event_command trigger_hist_enable_cmd = {
6754         .name                   = ENABLE_HIST_STR,
6755         .trigger_type           = ETT_HIST_ENABLE,
6756         .parse                  = event_enable_trigger_parse,
6757         .reg                    = event_enable_register_trigger,
6758         .unreg                  = event_enable_unregister_trigger,
6759         .unreg_all              = hist_enable_unreg_all,
6760         .get_trigger_ops        = hist_enable_get_trigger_ops,
6761         .set_filter             = set_trigger_filter,
6762 };
6763
6764 static struct event_command trigger_hist_disable_cmd = {
6765         .name                   = DISABLE_HIST_STR,
6766         .trigger_type           = ETT_HIST_ENABLE,
6767         .parse                  = event_enable_trigger_parse,
6768         .reg                    = event_enable_register_trigger,
6769         .unreg                  = event_enable_unregister_trigger,
6770         .unreg_all              = hist_enable_unreg_all,
6771         .get_trigger_ops        = hist_enable_get_trigger_ops,
6772         .set_filter             = set_trigger_filter,
6773 };
6774
6775 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6776 {
6777         unregister_event_command(&trigger_hist_enable_cmd);
6778         unregister_event_command(&trigger_hist_disable_cmd);
6779 }
6780
6781 __init int register_trigger_hist_enable_disable_cmds(void)
6782 {
6783         int ret;
6784
6785         ret = register_event_command(&trigger_hist_enable_cmd);
6786         if (WARN_ON(ret < 0))
6787                 return ret;
6788         ret = register_event_command(&trigger_hist_disable_cmd);
6789         if (WARN_ON(ret < 0))
6790                 unregister_trigger_hist_enable_disable_cmds();
6791
6792         return ret;
6793 }