kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace.h"
  23 #include "trace_dynevent.h"
  24
  25 #define SYNTH_SYSTEM            "synthetic"
  26 #define SYNTH_FIELDS_MAX        32
  27
  28 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  29
  30 #define ERRORS                                                          \
  31         C(NONE,                 "No error"),                            \
  32         C(DUPLICATE_VAR,        "Variable already defined"),            \
  33         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  34         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  35         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  36         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  37         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  38         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  39         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  40         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  41         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  42         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  43         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  44         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  45         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  46         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  47         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  48         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  49         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  50         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  51         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  52         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  53         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  54         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  55         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  56         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  57         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  58         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  59         C(INVALID_PARAM,        "Invalid action param"),                \
  60         C(ACTION_NOT_FOUND,     "No action found"),                     \
  61         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  62         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  63         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  64         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  65         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  66         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  67         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  68         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  69         C(FIELD_NOT_FOUND,      "Couldn't find field"),
  70
  71 #undef C
  72 #define C(a, b)         HIST_ERR_##a
  73
  74 enum { ERRORS };
  75
  76 #undef C
  77 #define C(a, b)         b
  78
  79 static const char *err_text[] = { ERRORS };
  80
  81 struct hist_field;
  82
  83 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84                                 struct tracing_map_elt *elt,
  85                                 struct ring_buffer_event *rbe,
  86                                 void *event);
  87
  88 #define HIST_FIELD_OPERANDS_MAX 2
  89 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  90 #define HIST_ACTIONS_MAX        8
  91
  92 enum field_op_id {
  93         FIELD_OP_NONE,
  94         FIELD_OP_PLUS,
  95         FIELD_OP_MINUS,
  96         FIELD_OP_UNARY_MINUS,
  97 };
  98
  99 /*
 100  * A hist_var (histogram variable) contains variable information for
 101  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 102  * flag set.  A hist_var has a variable name e.g. ts0, and is
 103  * associated with a given histogram trigger, as specified by
 104  * hist_data.  The hist_var idx is the unique index assigned to the
 105  * variable by the hist trigger's tracing_map.  The idx is what is
 106  * used to set a variable's value and, by a variable reference, to
 107  * retrieve it.
 108  */
 109 struct hist_var {
 110         char                            *name;
 111         struct hist_trigger_data        *hist_data;
 112         unsigned int                    idx;
 113 };
 114
 115 struct hist_field {
 116         struct ftrace_event_field       *field;
 117         unsigned long                   flags;
 118         hist_field_fn_t                 fn;
 119         unsigned int                    size;
 120         unsigned int                    offset;
 121         unsigned int                    is_signed;
 122         const char                      *type;
 123         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 124         struct hist_trigger_data        *hist_data;
 125
 126         /*
 127          * Variable fields contain variable-specific info in var.
 128          */
 129         struct hist_var                 var;
 130         enum field_op_id                operator;
 131         char                            *system;
 132         char                            *event_name;
 133
 134         /*
 135          * The name field is used for EXPR and VAR_REF fields.  VAR
 136          * fields contain the variable name in var.name.
 137          */
 138         char                            *name;
 139
 140         /*
 141          * When a histogram trigger is hit, if it has any references
 142          * to variables, the values of those variables are collected
 143          * into a var_ref_vals array by resolve_var_refs().  The
 144          * current value of each variable is read from the tracing_map
 145          * using the hist field's hist_var.idx and entered into the
 146          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 147          */
 148         unsigned int                    var_ref_idx;
 149         bool                            read_once;
 150 };
 151
 152 static u64 hist_field_none(struct hist_field *field,
 153                            struct tracing_map_elt *elt,
 154                            struct ring_buffer_event *rbe,
 155                            void *event)
 156 {
 157         return 0;
 158 }
 159
 160 static u64 hist_field_counter(struct hist_field *field,
 161                               struct tracing_map_elt *elt,
 162                               struct ring_buffer_event *rbe,
 163                               void *event)
 164 {
 165         return 1;
 166 }
 167
 168 static u64 hist_field_string(struct hist_field *hist_field,
 169                              struct tracing_map_elt *elt,
 170                              struct ring_buffer_event *rbe,
 171                              void *event)
 172 {
 173         char *addr = (char *)(event + hist_field->field->offset);
 174
 175         return (u64)(unsigned long)addr;
 176 }
 177
 178 static u64 hist_field_dynstring(struct hist_field *hist_field,
 179                                 struct tracing_map_elt *elt,
 180                                 struct ring_buffer_event *rbe,
 181                                 void *event)
 182 {
 183         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 184         int str_loc = str_item & 0xffff;
 185         char *addr = (char *)(event + str_loc);
 186
 187         return (u64)(unsigned long)addr;
 188 }
 189
 190 static u64 hist_field_pstring(struct hist_field *hist_field,
 191                               struct tracing_map_elt *elt,
 192                               struct ring_buffer_event *rbe,
 193                               void *event)
 194 {
 195         char **addr = (char **)(event + hist_field->field->offset);
 196
 197         return (u64)(unsigned long)*addr;
 198 }
 199
 200 static u64 hist_field_log2(struct hist_field *hist_field,
 201                            struct tracing_map_elt *elt,
 202                            struct ring_buffer_event *rbe,
 203                            void *event)
 204 {
 205         struct hist_field *operand = hist_field->operands[0];
 206
 207         u64 val = operand->fn(operand, elt, rbe, event);
 208
 209         return (u64) ilog2(roundup_pow_of_two(val));
 210 }
 211
 212 static u64 hist_field_plus(struct hist_field *hist_field,
 213                            struct tracing_map_elt *elt,
 214                            struct ring_buffer_event *rbe,
 215                            void *event)
 216 {
 217         struct hist_field *operand1 = hist_field->operands[0];
 218         struct hist_field *operand2 = hist_field->operands[1];
 219
 220         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 221         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 222
 223         return val1 + val2;
 224 }
 225
 226 static u64 hist_field_minus(struct hist_field *hist_field,
 227                             struct tracing_map_elt *elt,
 228                             struct ring_buffer_event *rbe,
 229                             void *event)
 230 {
 231         struct hist_field *operand1 = hist_field->operands[0];
 232         struct hist_field *operand2 = hist_field->operands[1];
 233
 234         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 235         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 236
 237         return val1 - val2;
 238 }
 239
 240 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 241                                   struct tracing_map_elt *elt,
 242                                   struct ring_buffer_event *rbe,
 243                                   void *event)
 244 {
 245         struct hist_field *operand = hist_field->operands[0];
 246
 247         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 248         u64 val = (u64)-sval;
 249
 250         return val;
 251 }
 252
 253 #define DEFINE_HIST_FIELD_FN(type)                                      \
 254         static u64 hist_field_##type(struct hist_field *hist_field,     \
 255                                      struct tracing_map_elt *elt,       \
 256                                      struct ring_buffer_event *rbe,     \
 257                                      void *event)                       \
 258 {                                                                       \
 259         type *addr = (type *)(event + hist_field->field->offset);       \
 260                                                                         \
 261         return (u64)(unsigned long)*addr;                               \
 262 }
 263
 264 DEFINE_HIST_FIELD_FN(s64);
 265 DEFINE_HIST_FIELD_FN(u64);
 266 DEFINE_HIST_FIELD_FN(s32);
 267 DEFINE_HIST_FIELD_FN(u32);
 268 DEFINE_HIST_FIELD_FN(s16);
 269 DEFINE_HIST_FIELD_FN(u16);
 270 DEFINE_HIST_FIELD_FN(s8);
 271 DEFINE_HIST_FIELD_FN(u8);
 272
 273 #define for_each_hist_field(i, hist_data)       \
 274         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 275
 276 #define for_each_hist_val_field(i, hist_data)   \
 277         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 278
 279 #define for_each_hist_key_field(i, hist_data)   \
 280         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 281
 282 #define HIST_STACKTRACE_DEPTH   16
 283 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 284 #define HIST_STACKTRACE_SKIP    5
 285
 286 #define HITCOUNT_IDX            0
 287 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 288
 289 enum hist_field_flags {
 290         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 291         HIST_FIELD_FL_KEY               = 1 << 1,
 292         HIST_FIELD_FL_STRING            = 1 << 2,
 293         HIST_FIELD_FL_HEX               = 1 << 3,
 294         HIST_FIELD_FL_SYM               = 1 << 4,
 295         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 296         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 297         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 298         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 299         HIST_FIELD_FL_LOG2              = 1 << 9,
 300         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 301         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 302         HIST_FIELD_FL_VAR               = 1 << 12,
 303         HIST_FIELD_FL_EXPR              = 1 << 13,
 304         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 305         HIST_FIELD_FL_CPU               = 1 << 15,
 306         HIST_FIELD_FL_ALIAS             = 1 << 16,
 307 };
 308
 309 struct var_defs {
 310         unsigned int    n_vars;
 311         char            *name[TRACING_MAP_VARS_MAX];
 312         char            *expr[TRACING_MAP_VARS_MAX];
 313 };
 314
 315 struct hist_trigger_attrs {
 316         char            *keys_str;
 317         char            *vals_str;
 318         char            *sort_key_str;
 319         char            *name;
 320         char            *clock;
 321         bool            pause;
 322         bool            cont;
 323         bool            clear;
 324         bool            ts_in_usecs;
 325         unsigned int    map_bits;
 326
 327         char            *assignment_str[TRACING_MAP_VARS_MAX];
 328         unsigned int    n_assignments;
 329
 330         char            *action_str[HIST_ACTIONS_MAX];
 331         unsigned int    n_actions;
 332
 333         struct var_defs var_defs;
 334 };
 335
 336 struct field_var {
 337         struct hist_field       *var;
 338         struct hist_field       *val;
 339 };
 340
 341 struct field_var_hist {
 342         struct hist_trigger_data        *hist_data;
 343         char                            *cmd;
 344 };
 345
 346 struct hist_trigger_data {
 347         struct hist_field               *fields[HIST_FIELDS_MAX];
 348         unsigned int                    n_vals;
 349         unsigned int                    n_keys;
 350         unsigned int                    n_fields;
 351         unsigned int                    n_vars;
 352         unsigned int                    key_size;
 353         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 354         unsigned int                    n_sort_keys;
 355         struct trace_event_file         *event_file;
 356         struct hist_trigger_attrs       *attrs;
 357         struct tracing_map              *map;
 358         bool                            enable_timestamps;
 359         bool                            remove;
 360         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 361         unsigned int                    n_var_refs;
 362
 363         struct action_data              *actions[HIST_ACTIONS_MAX];
 364         unsigned int                    n_actions;
 365
 366         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 367         unsigned int                    n_field_vars;
 368         unsigned int                    n_field_var_str;
 369         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 370         unsigned int                    n_field_var_hists;
 371
 372         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 373         unsigned int                    n_save_vars;
 374         unsigned int                    n_save_var_str;
 375 };
 376
 377 static int synth_event_create(int argc, const char **argv);
 378 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 379 static int synth_event_release(struct dyn_event *ev);
 380 static bool synth_event_is_busy(struct dyn_event *ev);
 381 static bool synth_event_match(const char *system, const char *event,
 382                         int argc, const char **argv, struct dyn_event *ev);
 383
 384 static struct dyn_event_operations synth_event_ops = {
 385         .create = synth_event_create,
 386         .show = synth_event_show,
 387         .is_busy = synth_event_is_busy,
 388         .free = synth_event_release,
 389         .match = synth_event_match,
 390 };
 391
 392 struct synth_field {
 393         char *type;
 394         char *name;
 395         size_t size;
 396         bool is_signed;
 397         bool is_string;
 398 };
 399
 400 struct synth_event {
 401         struct dyn_event                        devent;
 402         int                                     ref;
 403         char                                    *name;
 404         struct synth_field                      **fields;
 405         unsigned int                            n_fields;
 406         unsigned int                            n_u64;
 407         struct trace_event_class                class;
 408         struct trace_event_call                 call;
 409         struct tracepoint                       *tp;
 410 };
 411
 412 static bool is_synth_event(struct dyn_event *ev)
 413 {
 414         return ev->ops == &synth_event_ops;
 415 }
 416
 417 static struct synth_event *to_synth_event(struct dyn_event *ev)
 418 {
 419         return container_of(ev, struct synth_event, devent);
 420 }
 421
 422 static bool synth_event_is_busy(struct dyn_event *ev)
 423 {
 424         struct synth_event *event = to_synth_event(ev);
 425
 426         return event->ref != 0;
 427 }
 428
 429 static bool synth_event_match(const char *system, const char *event,
 430                         int argc, const char **argv, struct dyn_event *ev)
 431 {
 432         struct synth_event *sev = to_synth_event(ev);
 433
 434         return strcmp(sev->name, event) == 0 &&
 435                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 436 }
 437
 438 struct action_data;
 439
 440 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 441                              struct tracing_map_elt *elt, void *rec,
 442                              struct ring_buffer_event *rbe, void *key,
 443                              struct action_data *data, u64 *var_ref_vals);
 444
 445 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 446
 447 enum handler_id {
 448         HANDLER_ONMATCH = 1,
 449         HANDLER_ONMAX,
 450         HANDLER_ONCHANGE,
 451 };
 452
 453 enum action_id {
 454         ACTION_SAVE = 1,
 455         ACTION_TRACE,
 456         ACTION_SNAPSHOT,
 457 };
 458
 459 struct action_data {
 460         enum handler_id         handler;
 461         enum action_id          action;
 462         char                    *action_name;
 463         action_fn_t             fn;
 464
 465         unsigned int            n_params;
 466         char                    *params[SYNTH_FIELDS_MAX];
 467
 468         /*
 469          * When a histogram trigger is hit, the values of any
 470          * references to variables, including variables being passed
 471          * as parameters to synthetic events, are collected into a
 472          * var_ref_vals array.  This var_ref_idx is the index of the
 473          * first param in the array to be passed to the synthetic
 474          * event invocation.
 475          */
 476         unsigned int            var_ref_idx;
 477         struct synth_event      *synth_event;
 478         bool                    use_trace_keyword;
 479         char                    *synth_event_name;
 480
 481         union {
 482                 struct {
 483                         char                    *event;
 484                         char                    *event_system;
 485                 } match_data;
 486
 487                 struct {
 488                         /*
 489                          * var_str contains the $-unstripped variable
 490                          * name referenced by var_ref, and used when
 491                          * printing the action.  Because var_ref
 492                          * creation is deferred to create_actions(),
 493                          * we need a per-action way to save it until
 494                          * then, thus var_str.
 495                          */
 496                         char                    *var_str;
 497
 498                         /*
 499                          * var_ref refers to the variable being
 500                          * tracked e.g onmax($var).
 501                          */
 502                         struct hist_field       *var_ref;
 503
 504                         /*
 505                          * track_var contains the 'invisible' tracking
 506                          * variable created to keep the current
 507                          * e.g. max value.
 508                          */
 509                         struct hist_field       *track_var;
 510
 511                         check_track_val_fn_t    check_val;
 512                         action_fn_t             save_data;
 513                 } track_data;
 514         };
 515 };
 516
 517 struct track_data {
 518         u64                             track_val;
 519         bool                            updated;
 520
 521         unsigned int                    key_len;
 522         void                            *key;
 523         struct tracing_map_elt          elt;
 524
 525         struct action_data              *action_data;
 526         struct hist_trigger_data        *hist_data;
 527 };
 528
 529 struct hist_elt_data {
 530         char *comm;
 531         u64 *var_ref_vals;
 532         char *field_var_str[SYNTH_FIELDS_MAX];
 533 };
 534
 535 struct snapshot_context {
 536         struct tracing_map_elt  *elt;
 537         void                    *key;
 538 };
 539
 540 static void track_data_free(struct track_data *track_data)
 541 {
 542         struct hist_elt_data *elt_data;
 543
 544         if (!track_data)
 545                 return;
 546
 547         kfree(track_data->key);
 548
 549         elt_data = track_data->elt.private_data;
 550         if (elt_data) {
 551                 kfree(elt_data->comm);
 552                 kfree(elt_data);
 553         }
 554
 555         kfree(track_data);
 556 }
 557
 558 static struct track_data *track_data_alloc(unsigned int key_len,
 559                                            struct action_data *action_data,
 560                                            struct hist_trigger_data *hist_data)
 561 {
 562         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 563         struct hist_elt_data *elt_data;
 564
 565         if (!data)
 566                 return ERR_PTR(-ENOMEM);
 567
 568         data->key = kzalloc(key_len, GFP_KERNEL);
 569         if (!data->key) {
 570                 track_data_free(data);
 571                 return ERR_PTR(-ENOMEM);
 572         }
 573
 574         data->key_len = key_len;
 575         data->action_data = action_data;
 576         data->hist_data = hist_data;
 577
 578         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 579         if (!elt_data) {
 580                 track_data_free(data);
 581                 return ERR_PTR(-ENOMEM);
 582         }
 583         data->elt.private_data = elt_data;
 584
 585         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 586         if (!elt_data->comm) {
 587                 track_data_free(data);
 588                 return ERR_PTR(-ENOMEM);
 589         }
 590
 591         return data;
 592 }
 593
 594 static char last_cmd[MAX_FILTER_STR_VAL];
 595 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 596
 597 static int errpos(char *str)
 598 {
 599         return err_pos(last_cmd, str);
 600 }
 601
 602 static void last_cmd_set(struct trace_event_file *file, char *str)
 603 {
 604         const char *system = NULL, *name = NULL;
 605         struct trace_event_call *call;
 606
 607         if (!str)
 608                 return;
 609
 610         strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 611
 612         if (file) {
 613                 call = file->event_call;
 614
 615                 system = call->class->system;
 616                 if (system) {
 617                         name = trace_event_name(call);
 618                         if (!name)
 619                                 system = NULL;
 620                 }
 621         }
 622
 623         if (system)
 624                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 625 }
 626
 627 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 628 {
 629         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 630                         err_type, err_pos);
 631 }
 632
 633 static void hist_err_clear(void)
 634 {
 635         last_cmd[0] = '\0';
 636         last_cmd_loc[0] = '\0';
 637 }
 638
 639 struct synth_trace_event {
 640         struct trace_entry      ent;
 641         u64                     fields[];
 642 };
 643
 644 static int synth_event_define_fields(struct trace_event_call *call)
 645 {
 646         struct synth_trace_event trace;
 647         int offset = offsetof(typeof(trace), fields);
 648         struct synth_event *event = call->data;
 649         unsigned int i, size, n_u64;
 650         char *name, *type;
 651         bool is_signed;
 652         int ret = 0;
 653
 654         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 655                 size = event->fields[i]->size;
 656                 is_signed = event->fields[i]->is_signed;
 657                 type = event->fields[i]->type;
 658                 name = event->fields[i]->name;
 659                 ret = trace_define_field(call, type, name, offset, size,
 660                                          is_signed, FILTER_OTHER);
 661                 if (ret)
 662                         break;
 663
 664                 if (event->fields[i]->is_string) {
 665                         offset += STR_VAR_LEN_MAX;
 666                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 667                 } else {
 668                         offset += sizeof(u64);
 669                         n_u64++;
 670                 }
 671         }
 672
 673         event->n_u64 = n_u64;
 674
 675         return ret;
 676 }
 677
 678 static bool synth_field_signed(char *type)
 679 {
 680         if (str_has_prefix(type, "u"))
 681                 return false;
 682         if (strcmp(type, "gfp_t") == 0)
 683                 return false;
 684
 685         return true;
 686 }
 687
 688 static int synth_field_is_string(char *type)
 689 {
 690         if (strstr(type, "char[") != NULL)
 691                 return true;
 692
 693         return false;
 694 }
 695
 696 static int synth_field_string_size(char *type)
 697 {
 698         char buf[4], *end, *start;
 699         unsigned int len;
 700         int size, err;
 701
 702         start = strstr(type, "char[");
 703         if (start == NULL)
 704                 return -EINVAL;
 705         start += sizeof("char[") - 1;
 706
 707         end = strchr(type, ']');
 708         if (!end || end < start)
 709                 return -EINVAL;
 710
 711         len = end - start;
 712         if (len > 3)
 713                 return -EINVAL;
 714
 715         strncpy(buf, start, len);
 716         buf[len] = '\0';
 717
 718         err = kstrtouint(buf, 0, &size);
 719         if (err)
 720                 return err;
 721
 722         if (size > STR_VAR_LEN_MAX)
 723                 return -EINVAL;
 724
 725         return size;
 726 }
 727
 728 static int synth_field_size(char *type)
 729 {
 730         int size = 0;
 731
 732         if (strcmp(type, "s64") == 0)
 733                 size = sizeof(s64);
 734         else if (strcmp(type, "u64") == 0)
 735                 size = sizeof(u64);
 736         else if (strcmp(type, "s32") == 0)
 737                 size = sizeof(s32);
 738         else if (strcmp(type, "u32") == 0)
 739                 size = sizeof(u32);
 740         else if (strcmp(type, "s16") == 0)
 741                 size = sizeof(s16);
 742         else if (strcmp(type, "u16") == 0)
 743                 size = sizeof(u16);
 744         else if (strcmp(type, "s8") == 0)
 745                 size = sizeof(s8);
 746         else if (strcmp(type, "u8") == 0)
 747                 size = sizeof(u8);
 748         else if (strcmp(type, "char") == 0)
 749                 size = sizeof(char);
 750         else if (strcmp(type, "unsigned char") == 0)
 751                 size = sizeof(unsigned char);
 752         else if (strcmp(type, "int") == 0)
 753                 size = sizeof(int);
 754         else if (strcmp(type, "unsigned int") == 0)
 755                 size = sizeof(unsigned int);
 756         else if (strcmp(type, "long") == 0)
 757                 size = sizeof(long);
 758         else if (strcmp(type, "unsigned long") == 0)
 759                 size = sizeof(unsigned long);
 760         else if (strcmp(type, "pid_t") == 0)
 761                 size = sizeof(pid_t);
 762         else if (strcmp(type, "gfp_t") == 0)
 763                 size = sizeof(gfp_t);
 764         else if (synth_field_is_string(type))
 765                 size = synth_field_string_size(type);
 766
 767         return size;
 768 }
 769
 770 static const char *synth_field_fmt(char *type)
 771 {
 772         const char *fmt = "%llu";
 773
 774         if (strcmp(type, "s64") == 0)
 775                 fmt = "%lld";
 776         else if (strcmp(type, "u64") == 0)
 777                 fmt = "%llu";
 778         else if (strcmp(type, "s32") == 0)
 779                 fmt = "%d";
 780         else if (strcmp(type, "u32") == 0)
 781                 fmt = "%u";
 782         else if (strcmp(type, "s16") == 0)
 783                 fmt = "%d";
 784         else if (strcmp(type, "u16") == 0)
 785                 fmt = "%u";
 786         else if (strcmp(type, "s8") == 0)
 787                 fmt = "%d";
 788         else if (strcmp(type, "u8") == 0)
 789                 fmt = "%u";
 790         else if (strcmp(type, "char") == 0)
 791                 fmt = "%d";
 792         else if (strcmp(type, "unsigned char") == 0)
 793                 fmt = "%u";
 794         else if (strcmp(type, "int") == 0)
 795                 fmt = "%d";
 796         else if (strcmp(type, "unsigned int") == 0)
 797                 fmt = "%u";
 798         else if (strcmp(type, "long") == 0)
 799                 fmt = "%ld";
 800         else if (strcmp(type, "unsigned long") == 0)
 801                 fmt = "%lu";
 802         else if (strcmp(type, "pid_t") == 0)
 803                 fmt = "%d";
 804         else if (strcmp(type, "gfp_t") == 0)
 805                 fmt = "%x";
 806         else if (synth_field_is_string(type))
 807                 fmt = "%s";
 808
 809         return fmt;
 810 }
 811
 812 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 813                                            int flags,
 814                                            struct trace_event *event)
 815 {
 816         struct trace_array *tr = iter->tr;
 817         struct trace_seq *s = &iter->seq;
 818         struct synth_trace_event *entry;
 819         struct synth_event *se;
 820         unsigned int i, n_u64;
 821         char print_fmt[32];
 822         const char *fmt;
 823
 824         entry = (struct synth_trace_event *)iter->ent;
 825         se = container_of(event, struct synth_event, call.event);
 826
 827         trace_seq_printf(s, "%s: ", se->name);
 828
 829         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 830                 if (trace_seq_has_overflowed(s))
 831                         goto end;
 832
 833                 fmt = synth_field_fmt(se->fields[i]->type);
 834
 835                 /* parameter types */
 836                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 837                         trace_seq_printf(s, "%s ", fmt);
 838
 839                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 840
 841                 /* parameter values */
 842                 if (se->fields[i]->is_string) {
 843                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 844                                          (char *)&entry->fields[n_u64],
 845                                          i == se->n_fields - 1 ? "" : " ");
 846                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 847                 } else {
 848                         struct trace_print_flags __flags[] = {
 849                             __def_gfpflag_names, {-1, NULL} };
 850
 851                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 852                                          entry->fields[n_u64],
 853                                          i == se->n_fields - 1 ? "" : " ");
 854
 855                         if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 856                                 trace_seq_puts(s, " (");
 857                                 trace_print_flags_seq(s, "|",
 858                                                       entry->fields[n_u64],
 859                                                       __flags);
 860                                 trace_seq_putc(s, ')');
 861                         }
 862                         n_u64++;
 863                 }
 864         }
 865 end:
 866         trace_seq_putc(s, '\n');
 867
 868         return trace_handle_return(s);
 869 }
 870
 871 static struct trace_event_functions synth_event_funcs = {
 872         .trace          = print_synth_event
 873 };
 874
 875 static notrace void trace_event_raw_event_synth(void *__data,
 876                                                 u64 *var_ref_vals,
 877                                                 unsigned int var_ref_idx)
 878 {
 879         struct trace_event_file *trace_file = __data;
 880         struct synth_trace_event *entry;
 881         struct trace_event_buffer fbuffer;
 882         struct ring_buffer *buffer;
 883         struct synth_event *event;
 884         unsigned int i, n_u64;
 885         int fields_size = 0;
 886
 887         event = trace_file->event_call->data;
 888
 889         if (trace_trigger_soft_disabled(trace_file))
 890                 return;
 891
 892         fields_size = event->n_u64 * sizeof(u64);
 893
 894         /*
 895          * Avoid ring buffer recursion detection, as this event
 896          * is being performed within another event.
 897          */
 898         buffer = trace_file->tr->trace_buffer.buffer;
 899         ring_buffer_nest_start(buffer);
 900
 901         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 902                                            sizeof(*entry) + fields_size);
 903         if (!entry)
 904                 goto out;
 905
 906         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 907                 if (event->fields[i]->is_string) {
 908                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 909                         char *str_field = (char *)&entry->fields[n_u64];
 910
 911                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 912                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 913                 } else {
 914                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 915                         n_u64++;
 916                 }
 917         }
 918
 919         trace_event_buffer_commit(&fbuffer);
 920 out:
 921         ring_buffer_nest_end(buffer);
 922 }
 923
 924 static void free_synth_event_print_fmt(struct trace_event_call *call)
 925 {
 926         if (call) {
 927                 kfree(call->print_fmt);
 928                 call->print_fmt = NULL;
 929         }
 930 }
 931
 932 static int __set_synth_event_print_fmt(struct synth_event *event,
 933                                        char *buf, int len)
 934 {
 935         const char *fmt;
 936         int pos = 0;
 937         int i;
 938
 939         /* When len=0, we just calculate the needed length */
 940 #define LEN_OR_ZERO (len ? len - pos : 0)
 941
 942         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 943         for (i = 0; i < event->n_fields; i++) {
 944                 fmt = synth_field_fmt(event->fields[i]->type);
 945                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 946                                 event->fields[i]->name, fmt,
 947                                 i == event->n_fields - 1 ? "" : ", ");
 948         }
 949         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 950
 951         for (i = 0; i < event->n_fields; i++) {
 952                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 953                                 ", REC->%s", event->fields[i]->name);
 954         }
 955
 956 #undef LEN_OR_ZERO
 957
 958         /* return the length of print_fmt */
 959         return pos;
 960 }
 961
 962 static int set_synth_event_print_fmt(struct trace_event_call *call)
 963 {
 964         struct synth_event *event = call->data;
 965         char *print_fmt;
 966         int len;
 967
 968         /* First: called with 0 length to calculate the needed length */
 969         len = __set_synth_event_print_fmt(event, NULL, 0);
 970
 971         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 972         if (!print_fmt)
 973                 return -ENOMEM;
 974
 975         /* Second: actually write the @print_fmt */
 976         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 977         call->print_fmt = print_fmt;
 978
 979         return 0;
 980 }
 981
 982 static void free_synth_field(struct synth_field *field)
 983 {
 984         kfree(field->type);
 985         kfree(field->name);
 986         kfree(field);
 987 }
 988
 989 static struct synth_field *parse_synth_field(int argc, const char **argv,
 990                                              int *consumed)
 991 {
 992         struct synth_field *field;
 993         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 994         int len, ret = 0;
 995
 996         if (field_type[0] == ';')
 997                 field_type++;
 998
 999         if (!strcmp(field_type, "unsigned")) {
1000                 if (argc < 3)
1001                         return ERR_PTR(-EINVAL);
1002                 prefix = "unsigned ";
1003                 field_type = argv[1];
1004                 field_name = argv[2];
1005                 *consumed = 3;
1006         } else {
1007                 field_name = argv[1];
1008                 *consumed = 2;
1009         }
1010
1011         field = kzalloc(sizeof(*field), GFP_KERNEL);
1012         if (!field)
1013                 return ERR_PTR(-ENOMEM);
1014
1015         len = strlen(field_name);
1016         array = strchr(field_name, '[');
1017         if (array)
1018                 len -= strlen(array);
1019         else if (field_name[len - 1] == ';')
1020                 len--;
1021
1022         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1023         if (!field->name) {
1024                 ret = -ENOMEM;
1025                 goto free;
1026         }
1027
1028         if (field_type[0] == ';')
1029                 field_type++;
1030         len = strlen(field_type) + 1;
1031         if (array)
1032                 len += strlen(array);
1033         if (prefix)
1034                 len += strlen(prefix);
1035
1036         field->type = kzalloc(len, GFP_KERNEL);
1037         if (!field->type) {
1038                 ret = -ENOMEM;
1039                 goto free;
1040         }
1041         if (prefix)
1042                 strcat(field->type, prefix);
1043         strcat(field->type, field_type);
1044         if (array) {
1045                 strcat(field->type, array);
1046                 if (field->type[len - 1] == ';')
1047                         field->type[len - 1] = '\0';
1048         }
1049
1050         field->size = synth_field_size(field->type);
1051         if (!field->size) {
1052                 ret = -EINVAL;
1053                 goto free;
1054         }
1055
1056         if (synth_field_is_string(field->type))
1057                 field->is_string = true;
1058
1059         field->is_signed = synth_field_signed(field->type);
1060
1061  out:
1062         return field;
1063  free:
1064         free_synth_field(field);
1065         field = ERR_PTR(ret);
1066         goto out;
1067 }
1068
1069 static void free_synth_tracepoint(struct tracepoint *tp)
1070 {
1071         if (!tp)
1072                 return;
1073
1074         kfree(tp->name);
1075         kfree(tp);
1076 }
1077
1078 static struct tracepoint *alloc_synth_tracepoint(char *name)
1079 {
1080         struct tracepoint *tp;
1081
1082         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1083         if (!tp)
1084                 return ERR_PTR(-ENOMEM);
1085
1086         tp->name = kstrdup(name, GFP_KERNEL);
1087         if (!tp->name) {
1088                 kfree(tp);
1089                 return ERR_PTR(-ENOMEM);
1090         }
1091
1092         return tp;
1093 }
1094
1095 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1096                                     unsigned int var_ref_idx);
1097
1098 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1099                                unsigned int var_ref_idx)
1100 {
1101         struct tracepoint *tp = event->tp;
1102
1103         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1104                 struct tracepoint_func *probe_func_ptr;
1105                 synth_probe_func_t probe_func;
1106                 void *__data;
1107
1108                 if (!(cpu_online(raw_smp_processor_id())))
1109                         return;
1110
1111                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1112                 if (probe_func_ptr) {
1113                         do {
1114                                 probe_func = probe_func_ptr->func;
1115                                 __data = probe_func_ptr->data;
1116                                 probe_func(__data, var_ref_vals, var_ref_idx);
1117                         } while ((++probe_func_ptr)->func);
1118                 }
1119         }
1120 }
1121
1122 static struct synth_event *find_synth_event(const char *name)
1123 {
1124         struct dyn_event *pos;
1125         struct synth_event *event;
1126
1127         for_each_dyn_event(pos) {
1128                 if (!is_synth_event(pos))
1129                         continue;
1130                 event = to_synth_event(pos);
1131                 if (strcmp(event->name, name) == 0)
1132                         return event;
1133         }
1134
1135         return NULL;
1136 }
1137
1138 static struct trace_event_fields synth_event_fields_array[] = {
1139         { .type = TRACE_FUNCTION_TYPE,
1140           .define_fields = synth_event_define_fields },
1141         {}
1142 };
1143
1144 static int register_synth_event(struct synth_event *event)
1145 {
1146         struct trace_event_call *call = &event->call;
1147         int ret = 0;
1148
1149         event->call.class = &event->class;
1150         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1151         if (!event->class.system) {
1152                 ret = -ENOMEM;
1153                 goto out;
1154         }
1155
1156         event->tp = alloc_synth_tracepoint(event->name);
1157         if (IS_ERR(event->tp)) {
1158                 ret = PTR_ERR(event->tp);
1159                 event->tp = NULL;
1160                 goto out;
1161         }
1162
1163         INIT_LIST_HEAD(&call->class->fields);
1164         call->event.funcs = &synth_event_funcs;
1165         call->class->fields_array = synth_event_fields_array;
1166
1167         ret = register_trace_event(&call->event);
1168         if (!ret) {
1169                 ret = -ENODEV;
1170                 goto out;
1171         }
1172         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1173         call->class->reg = trace_event_reg;
1174         call->class->probe = trace_event_raw_event_synth;
1175         call->data = event;
1176         call->tp = event->tp;
1177
1178         ret = trace_add_event_call(call);
1179         if (ret) {
1180                 pr_warn("Failed to register synthetic event: %s\n",
1181                         trace_event_name(call));
1182                 goto err;
1183         }
1184
1185         ret = set_synth_event_print_fmt(call);
1186         if (ret < 0) {
1187                 trace_remove_event_call(call);
1188                 goto err;
1189         }
1190  out:
1191         return ret;
1192  err:
1193         unregister_trace_event(&call->event);
1194         goto out;
1195 }
1196
1197 static int unregister_synth_event(struct synth_event *event)
1198 {
1199         struct trace_event_call *call = &event->call;
1200         int ret;
1201
1202         ret = trace_remove_event_call(call);
1203
1204         return ret;
1205 }
1206
1207 static void free_synth_event(struct synth_event *event)
1208 {
1209         unsigned int i;
1210
1211         if (!event)
1212                 return;
1213
1214         for (i = 0; i < event->n_fields; i++)
1215                 free_synth_field(event->fields[i]);
1216
1217         kfree(event->fields);
1218         kfree(event->name);
1219         kfree(event->class.system);
1220         free_synth_tracepoint(event->tp);
1221         free_synth_event_print_fmt(&event->call);
1222         kfree(event);
1223 }
1224
1225 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1226                                              struct synth_field **fields)
1227 {
1228         struct synth_event *event;
1229         unsigned int i;
1230
1231         event = kzalloc(sizeof(*event), GFP_KERNEL);
1232         if (!event) {
1233                 event = ERR_PTR(-ENOMEM);
1234                 goto out;
1235         }
1236
1237         event->name = kstrdup(name, GFP_KERNEL);
1238         if (!event->name) {
1239                 kfree(event);
1240                 event = ERR_PTR(-ENOMEM);
1241                 goto out;
1242         }
1243
1244         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1245         if (!event->fields) {
1246                 free_synth_event(event);
1247                 event = ERR_PTR(-ENOMEM);
1248                 goto out;
1249         }
1250
1251         dyn_event_init(&event->devent, &synth_event_ops);
1252
1253         for (i = 0; i < n_fields; i++)
1254                 event->fields[i] = fields[i];
1255
1256         event->n_fields = n_fields;
1257  out:
1258         return event;
1259 }
1260
1261 static void action_trace(struct hist_trigger_data *hist_data,
1262                          struct tracing_map_elt *elt, void *rec,
1263                          struct ring_buffer_event *rbe, void *key,
1264                          struct action_data *data, u64 *var_ref_vals)
1265 {
1266         struct synth_event *event = data->synth_event;
1267
1268         trace_synth(event, var_ref_vals, data->var_ref_idx);
1269 }
1270
1271 struct hist_var_data {
1272         struct list_head list;
1273         struct hist_trigger_data *hist_data;
1274 };
1275
1276 static int __create_synth_event(int argc, const char *name, const char **argv)
1277 {
1278         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1279         struct synth_event *event = NULL;
1280         int i, consumed = 0, n_fields = 0, ret = 0;
1281
1282         /*
1283          * Argument syntax:
1284          *  - Add synthetic event: <event_name> field[;field] ...
1285          *  - Remove synthetic event: !<event_name> field[;field] ...
1286          *      where 'field' = type field_name
1287          */
1288
1289         if (name[0] == '\0' || argc < 1)
1290                 return -EINVAL;
1291
1292         mutex_lock(&event_mutex);
1293
1294         event = find_synth_event(name);
1295         if (event) {
1296                 ret = -EEXIST;
1297                 goto out;
1298         }
1299
1300         for (i = 0; i < argc - 1; i++) {
1301                 if (strcmp(argv[i], ";") == 0)
1302                         continue;
1303                 if (n_fields == SYNTH_FIELDS_MAX) {
1304                         ret = -EINVAL;
1305                         goto err;
1306                 }
1307
1308                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1309                 if (IS_ERR(field)) {
1310                         ret = PTR_ERR(field);
1311                         goto err;
1312                 }
1313                 fields[n_fields++] = field;
1314                 i += consumed - 1;
1315         }
1316
1317         if (i < argc && strcmp(argv[i], ";") != 0) {
1318                 ret = -EINVAL;
1319                 goto err;
1320         }
1321
1322         event = alloc_synth_event(name, n_fields, fields);
1323         if (IS_ERR(event)) {
1324                 ret = PTR_ERR(event);
1325                 event = NULL;
1326                 goto err;
1327         }
1328         ret = register_synth_event(event);
1329         if (!ret)
1330                 dyn_event_add(&event->devent);
1331         else
1332                 free_synth_event(event);
1333  out:
1334         mutex_unlock(&event_mutex);
1335
1336         return ret;
1337  err:
1338         for (i = 0; i < n_fields; i++)
1339                 free_synth_field(fields[i]);
1340
1341         goto out;
1342 }
1343
1344 static int create_or_delete_synth_event(int argc, char **argv)
1345 {
1346         const char *name = argv[0];
1347         struct synth_event *event = NULL;
1348         int ret;
1349
1350         /* trace_run_command() ensures argc != 0 */
1351         if (name[0] == '!') {
1352                 mutex_lock(&event_mutex);
1353                 event = find_synth_event(name + 1);
1354                 if (event) {
1355                         if (event->ref)
1356                                 ret = -EBUSY;
1357                         else {
1358                                 ret = unregister_synth_event(event);
1359                                 if (!ret) {
1360                                         dyn_event_remove(&event->devent);
1361                                         free_synth_event(event);
1362                                 }
1363                         }
1364                 } else
1365                         ret = -ENOENT;
1366                 mutex_unlock(&event_mutex);
1367                 return ret;
1368         }
1369
1370         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1371         return ret == -ECANCELED ? -EINVAL : ret;
1372 }
1373
1374 static int synth_event_create(int argc, const char **argv)
1375 {
1376         const char *name = argv[0];
1377         int len;
1378
1379         if (name[0] != 's' || name[1] != ':')
1380                 return -ECANCELED;
1381         name += 2;
1382
1383         /* This interface accepts group name prefix */
1384         if (strchr(name, '/')) {
1385                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1386                 if (len == 0)
1387                         return -EINVAL;
1388                 name += len;
1389         }
1390         return __create_synth_event(argc - 1, name, argv + 1);
1391 }
1392
1393 static int synth_event_release(struct dyn_event *ev)
1394 {
1395         struct synth_event *event = to_synth_event(ev);
1396         int ret;
1397
1398         if (event->ref)
1399                 return -EBUSY;
1400
1401         ret = unregister_synth_event(event);
1402         if (ret)
1403                 return ret;
1404
1405         dyn_event_remove(ev);
1406         free_synth_event(event);
1407         return 0;
1408 }
1409
1410 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1411 {
1412         struct synth_field *field;
1413         unsigned int i;
1414
1415         seq_printf(m, "%s\t", event->name);
1416
1417         for (i = 0; i < event->n_fields; i++) {
1418                 field = event->fields[i];
1419
1420                 /* parameter values */
1421                 seq_printf(m, "%s %s%s", field->type, field->name,
1422                            i == event->n_fields - 1 ? "" : "; ");
1423         }
1424
1425         seq_putc(m, '\n');
1426
1427         return 0;
1428 }
1429
1430 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1431 {
1432         struct synth_event *event = to_synth_event(ev);
1433
1434         seq_printf(m, "s:%s/", event->class.system);
1435
1436         return __synth_event_show(m, event);
1437 }
1438
1439 static int synth_events_seq_show(struct seq_file *m, void *v)
1440 {
1441         struct dyn_event *ev = v;
1442
1443         if (!is_synth_event(ev))
1444                 return 0;
1445
1446         return __synth_event_show(m, to_synth_event(ev));
1447 }
1448
1449 static const struct seq_operations synth_events_seq_op = {
1450         .start  = dyn_event_seq_start,
1451         .next   = dyn_event_seq_next,
1452         .stop   = dyn_event_seq_stop,
1453         .show   = synth_events_seq_show,
1454 };
1455
1456 static int synth_events_open(struct inode *inode, struct file *file)
1457 {
1458         int ret;
1459
1460         ret = security_locked_down(LOCKDOWN_TRACEFS);
1461         if (ret)
1462                 return ret;
1463
1464         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1465                 ret = dyn_events_release_all(&synth_event_ops);
1466                 if (ret < 0)
1467                         return ret;
1468         }
1469
1470         return seq_open(file, &synth_events_seq_op);
1471 }
1472
1473 static ssize_t synth_events_write(struct file *file,
1474                                   const char __user *buffer,
1475                                   size_t count, loff_t *ppos)
1476 {
1477         return trace_parse_run_command(file, buffer, count, ppos,
1478                                        create_or_delete_synth_event);
1479 }
1480
1481 static const struct file_operations synth_events_fops = {
1482         .open           = synth_events_open,
1483         .write          = synth_events_write,
1484         .read           = seq_read,
1485         .llseek         = seq_lseek,
1486         .release        = seq_release,
1487 };
1488
1489 static u64 hist_field_timestamp(struct hist_field *hist_field,
1490                                 struct tracing_map_elt *elt,
1491                                 struct ring_buffer_event *rbe,
1492                                 void *event)
1493 {
1494         struct hist_trigger_data *hist_data = hist_field->hist_data;
1495         struct trace_array *tr = hist_data->event_file->tr;
1496
1497         u64 ts = ring_buffer_event_time_stamp(rbe);
1498
1499         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1500                 ts = ns2usecs(ts);
1501
1502         return ts;
1503 }
1504
1505 static u64 hist_field_cpu(struct hist_field *hist_field,
1506                           struct tracing_map_elt *elt,
1507                           struct ring_buffer_event *rbe,
1508                           void *event)
1509 {
1510         int cpu = smp_processor_id();
1511
1512         return cpu;
1513 }
1514
1515 /**
1516  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1517  * @hist_field: The VAR_REF field to check
1518  * @var_data: The hist trigger that owns the variable
1519  * @var_idx: The trigger variable identifier
1520  *
1521  * Check the given VAR_REF field to see whether or not it references
1522  * the given variable associated with the given trigger.
1523  *
1524  * Return: The VAR_REF field if it does reference the variable, NULL if not
1525  */
1526 static struct hist_field *
1527 check_field_for_var_ref(struct hist_field *hist_field,
1528                         struct hist_trigger_data *var_data,
1529                         unsigned int var_idx)
1530 {
1531         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1532
1533         if (hist_field && hist_field->var.idx == var_idx &&
1534             hist_field->var.hist_data == var_data)
1535                 return hist_field;
1536
1537         return NULL;
1538 }
1539
1540 /**
1541  * find_var_ref - Check if a trigger has a reference to a trigger variable
1542  * @hist_data: The hist trigger that might have a reference to the variable
1543  * @var_data: The hist trigger that owns the variable
1544  * @var_idx: The trigger variable identifier
1545  *
1546  * Check the list of var_refs[] on the first hist trigger to see
1547  * whether any of them are references to the variable on the second
1548  * trigger.
1549  *
1550  * Return: The VAR_REF field referencing the variable if so, NULL if not
1551  */
1552 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1553                                        struct hist_trigger_data *var_data,
1554                                        unsigned int var_idx)
1555 {
1556         struct hist_field *hist_field;
1557         unsigned int i;
1558
1559         for (i = 0; i < hist_data->n_var_refs; i++) {
1560                 hist_field = hist_data->var_refs[i];
1561                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1562                         return hist_field;
1563         }
1564
1565         return NULL;
1566 }
1567
1568 /**
1569  * find_any_var_ref - Check if there is a reference to a given trigger variable
1570  * @hist_data: The hist trigger
1571  * @var_idx: The trigger variable identifier
1572  *
1573  * Check to see whether the given variable is currently referenced by
1574  * any other trigger.
1575  *
1576  * The trigger the variable is defined on is explicitly excluded - the
1577  * assumption being that a self-reference doesn't prevent a trigger
1578  * from being removed.
1579  *
1580  * Return: The VAR_REF field referencing the variable if so, NULL if not
1581  */
1582 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1583                                            unsigned int var_idx)
1584 {
1585         struct trace_array *tr = hist_data->event_file->tr;
1586         struct hist_field *found = NULL;
1587         struct hist_var_data *var_data;
1588
1589         list_for_each_entry(var_data, &tr->hist_vars, list) {
1590                 if (var_data->hist_data == hist_data)
1591                         continue;
1592                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1593                 if (found)
1594                         break;
1595         }
1596
1597         return found;
1598 }
1599
1600 /**
1601  * check_var_refs - Check if there is a reference to any of trigger's variables
1602  * @hist_data: The hist trigger
1603  *
1604  * A trigger can define one or more variables.  If any one of them is
1605  * currently referenced by any other trigger, this function will
1606  * determine that.
1607
1608  * Typically used to determine whether or not a trigger can be removed
1609  * - if there are any references to a trigger's variables, it cannot.
1610  *
1611  * Return: True if there is a reference to any of trigger's variables
1612  */
1613 static bool check_var_refs(struct hist_trigger_data *hist_data)
1614 {
1615         struct hist_field *field;
1616         bool found = false;
1617         int i;
1618
1619         for_each_hist_field(i, hist_data) {
1620                 field = hist_data->fields[i];
1621                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1622                         if (find_any_var_ref(hist_data, field->var.idx)) {
1623                                 found = true;
1624                                 break;
1625                         }
1626                 }
1627         }
1628
1629         return found;
1630 }
1631
1632 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1633 {
1634         struct trace_array *tr = hist_data->event_file->tr;
1635         struct hist_var_data *var_data, *found = NULL;
1636
1637         list_for_each_entry(var_data, &tr->hist_vars, list) {
1638                 if (var_data->hist_data == hist_data) {
1639                         found = var_data;
1640                         break;
1641                 }
1642         }
1643
1644         return found;
1645 }
1646
1647 static bool field_has_hist_vars(struct hist_field *hist_field,
1648                                 unsigned int level)
1649 {
1650         int i;
1651
1652         if (level > 3)
1653                 return false;
1654
1655         if (!hist_field)
1656                 return false;
1657
1658         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1659             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1660                 return true;
1661
1662         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1663                 struct hist_field *operand;
1664
1665                 operand = hist_field->operands[i];
1666                 if (field_has_hist_vars(operand, level + 1))
1667                         return true;
1668         }
1669
1670         return false;
1671 }
1672
1673 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1674 {
1675         struct hist_field *hist_field;
1676         int i;
1677
1678         for_each_hist_field(i, hist_data) {
1679                 hist_field = hist_data->fields[i];
1680                 if (field_has_hist_vars(hist_field, 0))
1681                         return true;
1682         }
1683
1684         return false;
1685 }
1686
1687 static int save_hist_vars(struct hist_trigger_data *hist_data)
1688 {
1689         struct trace_array *tr = hist_data->event_file->tr;
1690         struct hist_var_data *var_data;
1691
1692         var_data = find_hist_vars(hist_data);
1693         if (var_data)
1694                 return 0;
1695
1696         if (tracing_check_open_get_tr(tr))
1697                 return -ENODEV;
1698
1699         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1700         if (!var_data) {
1701                 trace_array_put(tr);
1702                 return -ENOMEM;
1703         }
1704
1705         var_data->hist_data = hist_data;
1706         list_add(&var_data->list, &tr->hist_vars);
1707
1708         return 0;
1709 }
1710
1711 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1712 {
1713         struct trace_array *tr = hist_data->event_file->tr;
1714         struct hist_var_data *var_data;
1715
1716         var_data = find_hist_vars(hist_data);
1717         if (!var_data)
1718                 return;
1719
1720         if (WARN_ON(check_var_refs(hist_data)))
1721                 return;
1722
1723         list_del(&var_data->list);
1724
1725         kfree(var_data);
1726
1727         trace_array_put(tr);
1728 }
1729
1730 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1731                                          const char *var_name)
1732 {
1733         struct hist_field *hist_field, *found = NULL;
1734         int i;
1735
1736         for_each_hist_field(i, hist_data) {
1737                 hist_field = hist_data->fields[i];
1738                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1739                     strcmp(hist_field->var.name, var_name) == 0) {
1740                         found = hist_field;
1741                         break;
1742                 }
1743         }
1744
1745         return found;
1746 }
1747
1748 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1749                                    struct trace_event_file *file,
1750                                    const char *var_name)
1751 {
1752         struct hist_trigger_data *test_data;
1753         struct event_trigger_data *test;
1754         struct hist_field *hist_field;
1755
1756         hist_field = find_var_field(hist_data, var_name);
1757         if (hist_field)
1758                 return hist_field;
1759
1760         list_for_each_entry_rcu(test, &file->triggers, list) {
1761                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1762                         test_data = test->private_data;
1763                         hist_field = find_var_field(test_data, var_name);
1764                         if (hist_field)
1765                                 return hist_field;
1766                 }
1767         }
1768
1769         return NULL;
1770 }
1771
1772 static struct trace_event_file *find_var_file(struct trace_array *tr,
1773                                               char *system,
1774                                               char *event_name,
1775                                               char *var_name)
1776 {
1777         struct hist_trigger_data *var_hist_data;
1778         struct hist_var_data *var_data;
1779         struct trace_event_file *file, *found = NULL;
1780
1781         if (system)
1782                 return find_event_file(tr, system, event_name);
1783
1784         list_for_each_entry(var_data, &tr->hist_vars, list) {
1785                 var_hist_data = var_data->hist_data;
1786                 file = var_hist_data->event_file;
1787                 if (file == found)
1788                         continue;
1789
1790                 if (find_var_field(var_hist_data, var_name)) {
1791                         if (found) {
1792                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1793                                 return NULL;
1794                         }
1795
1796                         found = file;
1797                 }
1798         }
1799
1800         return found;
1801 }
1802
1803 static struct hist_field *find_file_var(struct trace_event_file *file,
1804                                         const char *var_name)
1805 {
1806         struct hist_trigger_data *test_data;
1807         struct event_trigger_data *test;
1808         struct hist_field *hist_field;
1809
1810         list_for_each_entry_rcu(test, &file->triggers, list) {
1811                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1812                         test_data = test->private_data;
1813                         hist_field = find_var_field(test_data, var_name);
1814                         if (hist_field)
1815                                 return hist_field;
1816                 }
1817         }
1818
1819         return NULL;
1820 }
1821
1822 static struct hist_field *
1823 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1824 {
1825         struct trace_array *tr = hist_data->event_file->tr;
1826         struct hist_field *hist_field, *found = NULL;
1827         struct trace_event_file *file;
1828         unsigned int i;
1829
1830         for (i = 0; i < hist_data->n_actions; i++) {
1831                 struct action_data *data = hist_data->actions[i];
1832
1833                 if (data->handler == HANDLER_ONMATCH) {
1834                         char *system = data->match_data.event_system;
1835                         char *event_name = data->match_data.event;
1836
1837                         file = find_var_file(tr, system, event_name, var_name);
1838                         if (!file)
1839                                 continue;
1840                         hist_field = find_file_var(file, var_name);
1841                         if (hist_field) {
1842                                 if (found) {
1843                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1844                                                  errpos(var_name));
1845                                         return ERR_PTR(-EINVAL);
1846                                 }
1847
1848                                 found = hist_field;
1849                         }
1850                 }
1851         }
1852         return found;
1853 }
1854
1855 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1856                                          char *system,
1857                                          char *event_name,
1858                                          char *var_name)
1859 {
1860         struct trace_array *tr = hist_data->event_file->tr;
1861         struct hist_field *hist_field = NULL;
1862         struct trace_event_file *file;
1863
1864         if (!system || !event_name) {
1865                 hist_field = find_match_var(hist_data, var_name);
1866                 if (IS_ERR(hist_field))
1867                         return NULL;
1868                 if (hist_field)
1869                         return hist_field;
1870         }
1871
1872         file = find_var_file(tr, system, event_name, var_name);
1873         if (!file)
1874                 return NULL;
1875
1876         hist_field = find_file_var(file, var_name);
1877
1878         return hist_field;
1879 }
1880
1881 static u64 hist_field_var_ref(struct hist_field *hist_field,
1882                               struct tracing_map_elt *elt,
1883                               struct ring_buffer_event *rbe,
1884                               void *event)
1885 {
1886         struct hist_elt_data *elt_data;
1887         u64 var_val = 0;
1888
1889         if (WARN_ON_ONCE(!elt))
1890                 return var_val;
1891
1892         elt_data = elt->private_data;
1893         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1894
1895         return var_val;
1896 }
1897
1898 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1899                              u64 *var_ref_vals, bool self)
1900 {
1901         struct hist_trigger_data *var_data;
1902         struct tracing_map_elt *var_elt;
1903         struct hist_field *hist_field;
1904         unsigned int i, var_idx;
1905         bool resolved = true;
1906         u64 var_val = 0;
1907
1908         for (i = 0; i < hist_data->n_var_refs; i++) {
1909                 hist_field = hist_data->var_refs[i];
1910                 var_idx = hist_field->var.idx;
1911                 var_data = hist_field->var.hist_data;
1912
1913                 if (var_data == NULL) {
1914                         resolved = false;
1915                         break;
1916                 }
1917
1918                 if ((self && var_data != hist_data) ||
1919                     (!self && var_data == hist_data))
1920                         continue;
1921
1922                 var_elt = tracing_map_lookup(var_data->map, key);
1923                 if (!var_elt) {
1924                         resolved = false;
1925                         break;
1926                 }
1927
1928                 if (!tracing_map_var_set(var_elt, var_idx)) {
1929                         resolved = false;
1930                         break;
1931                 }
1932
1933                 if (self || !hist_field->read_once)
1934                         var_val = tracing_map_read_var(var_elt, var_idx);
1935                 else
1936                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1937
1938                 var_ref_vals[i] = var_val;
1939         }
1940
1941         return resolved;
1942 }
1943
1944 static const char *hist_field_name(struct hist_field *field,
1945                                    unsigned int level)
1946 {
1947         const char *field_name = "";
1948
1949         if (level > 1)
1950                 return field_name;
1951
1952         if (field->field)
1953                 field_name = field->field->name;
1954         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1955                  field->flags & HIST_FIELD_FL_ALIAS)
1956                 field_name = hist_field_name(field->operands[0], ++level);
1957         else if (field->flags & HIST_FIELD_FL_CPU)
1958                 field_name = "cpu";
1959         else if (field->flags & HIST_FIELD_FL_EXPR ||
1960                  field->flags & HIST_FIELD_FL_VAR_REF) {
1961                 if (field->system) {
1962                         static char full_name[MAX_FILTER_STR_VAL];
1963
1964                         strcat(full_name, field->system);
1965                         strcat(full_name, ".");
1966                         strcat(full_name, field->event_name);
1967                         strcat(full_name, ".");
1968                         strcat(full_name, field->name);
1969                         field_name = full_name;
1970                 } else
1971                         field_name = field->name;
1972         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1973                 field_name = "common_timestamp";
1974
1975         if (field_name == NULL)
1976                 field_name = "";
1977
1978         return field_name;
1979 }
1980
1981 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1982 {
1983         hist_field_fn_t fn = NULL;
1984
1985         switch (field_size) {
1986         case 8:
1987                 if (field_is_signed)
1988                         fn = hist_field_s64;
1989                 else
1990                         fn = hist_field_u64;
1991                 break;
1992         case 4:
1993                 if (field_is_signed)
1994                         fn = hist_field_s32;
1995                 else
1996                         fn = hist_field_u32;
1997                 break;
1998         case 2:
1999                 if (field_is_signed)
2000                         fn = hist_field_s16;
2001                 else
2002                         fn = hist_field_u16;
2003                 break;
2004         case 1:
2005                 if (field_is_signed)
2006                         fn = hist_field_s8;
2007                 else
2008                         fn = hist_field_u8;
2009                 break;
2010         }
2011
2012         return fn;
2013 }
2014
2015 static int parse_map_size(char *str)
2016 {
2017         unsigned long size, map_bits;
2018         int ret;
2019
2020         strsep(&str, "=");
2021         if (!str) {
2022                 ret = -EINVAL;
2023                 goto out;
2024         }
2025
2026         ret = kstrtoul(str, 0, &size);
2027         if (ret)
2028                 goto out;
2029
2030         map_bits = ilog2(roundup_pow_of_two(size));
2031         if (map_bits < TRACING_MAP_BITS_MIN ||
2032             map_bits > TRACING_MAP_BITS_MAX)
2033                 ret = -EINVAL;
2034         else
2035                 ret = map_bits;
2036  out:
2037         return ret;
2038 }
2039
2040 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2041 {
2042         unsigned int i;
2043
2044         if (!attrs)
2045                 return;
2046
2047         for (i = 0; i < attrs->n_assignments; i++)
2048                 kfree(attrs->assignment_str[i]);
2049
2050         for (i = 0; i < attrs->n_actions; i++)
2051                 kfree(attrs->action_str[i]);
2052
2053         kfree(attrs->name);
2054         kfree(attrs->sort_key_str);
2055         kfree(attrs->keys_str);
2056         kfree(attrs->vals_str);
2057         kfree(attrs->clock);
2058         kfree(attrs);
2059 }
2060
2061 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2062 {
2063         int ret = -EINVAL;
2064
2065         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2066                 return ret;
2067
2068         if ((str_has_prefix(str, "onmatch(")) ||
2069             (str_has_prefix(str, "onmax(")) ||
2070             (str_has_prefix(str, "onchange("))) {
2071                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2072                 if (!attrs->action_str[attrs->n_actions]) {
2073                         ret = -ENOMEM;
2074                         return ret;
2075                 }
2076                 attrs->n_actions++;
2077                 ret = 0;
2078         }
2079         return ret;
2080 }
2081
2082 static int parse_assignment(struct trace_array *tr,
2083                             char *str, struct hist_trigger_attrs *attrs)
2084 {
2085         int ret = 0;
2086
2087         if ((str_has_prefix(str, "key=")) ||
2088             (str_has_prefix(str, "keys="))) {
2089                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
2090                 if (!attrs->keys_str) {
2091                         ret = -ENOMEM;
2092                         goto out;
2093                 }
2094         } else if ((str_has_prefix(str, "val=")) ||
2095                    (str_has_prefix(str, "vals=")) ||
2096                    (str_has_prefix(str, "values="))) {
2097                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
2098                 if (!attrs->vals_str) {
2099                         ret = -ENOMEM;
2100                         goto out;
2101                 }
2102         } else if (str_has_prefix(str, "sort=")) {
2103                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2104                 if (!attrs->sort_key_str) {
2105                         ret = -ENOMEM;
2106                         goto out;
2107                 }
2108         } else if (str_has_prefix(str, "name=")) {
2109                 attrs->name = kstrdup(str, GFP_KERNEL);
2110                 if (!attrs->name) {
2111                         ret = -ENOMEM;
2112                         goto out;
2113                 }
2114         } else if (str_has_prefix(str, "clock=")) {
2115                 strsep(&str, "=");
2116                 if (!str) {
2117                         ret = -EINVAL;
2118                         goto out;
2119                 }
2120
2121                 str = strstrip(str);
2122                 attrs->clock = kstrdup(str, GFP_KERNEL);
2123                 if (!attrs->clock) {
2124                         ret = -ENOMEM;
2125                         goto out;
2126                 }
2127         } else if (str_has_prefix(str, "size=")) {
2128                 int map_bits = parse_map_size(str);
2129
2130                 if (map_bits < 0) {
2131                         ret = map_bits;
2132                         goto out;
2133                 }
2134                 attrs->map_bits = map_bits;
2135         } else {
2136                 char *assignment;
2137
2138                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2139                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2140                         ret = -EINVAL;
2141                         goto out;
2142                 }
2143
2144                 assignment = kstrdup(str, GFP_KERNEL);
2145                 if (!assignment) {
2146                         ret = -ENOMEM;
2147                         goto out;
2148                 }
2149
2150                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2151         }
2152  out:
2153         return ret;
2154 }
2155
2156 static struct hist_trigger_attrs *
2157 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
2158 {
2159         struct hist_trigger_attrs *attrs;
2160         int ret = 0;
2161
2162         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2163         if (!attrs)
2164                 return ERR_PTR(-ENOMEM);
2165
2166         while (trigger_str) {
2167                 char *str = strsep(&trigger_str, ":");
2168
2169                 if (strchr(str, '=')) {
2170                         ret = parse_assignment(tr, str, attrs);
2171                         if (ret)
2172                                 goto free;
2173                 } else if (strcmp(str, "pause") == 0)
2174                         attrs->pause = true;
2175                 else if ((strcmp(str, "cont") == 0) ||
2176                          (strcmp(str, "continue") == 0))
2177                         attrs->cont = true;
2178                 else if (strcmp(str, "clear") == 0)
2179                         attrs->clear = true;
2180                 else {
2181                         ret = parse_action(str, attrs);
2182                         if (ret)
2183                                 goto free;
2184                 }
2185         }
2186
2187         if (!attrs->keys_str) {
2188                 ret = -EINVAL;
2189                 goto free;
2190         }
2191
2192         if (!attrs->clock) {
2193                 attrs->clock = kstrdup("global", GFP_KERNEL);
2194                 if (!attrs->clock) {
2195                         ret = -ENOMEM;
2196                         goto free;
2197                 }
2198         }
2199
2200         return attrs;
2201  free:
2202         destroy_hist_trigger_attrs(attrs);
2203
2204         return ERR_PTR(ret);
2205 }
2206
2207 static inline void save_comm(char *comm, struct task_struct *task)
2208 {
2209         if (!task->pid) {
2210                 strcpy(comm, "<idle>");
2211                 return;
2212         }
2213
2214         if (WARN_ON_ONCE(task->pid < 0)) {
2215                 strcpy(comm, "<XXX>");
2216                 return;
2217         }
2218
2219         strncpy(comm, task->comm, TASK_COMM_LEN);
2220 }
2221
2222 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2223 {
2224         unsigned int i;
2225
2226         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2227                 kfree(elt_data->field_var_str[i]);
2228
2229         kfree(elt_data->comm);
2230         kfree(elt_data);
2231 }
2232
2233 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2234 {
2235         struct hist_elt_data *elt_data = elt->private_data;
2236
2237         hist_elt_data_free(elt_data);
2238 }
2239
2240 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2241 {
2242         struct hist_trigger_data *hist_data = elt->map->private_data;
2243         unsigned int size = TASK_COMM_LEN;
2244         struct hist_elt_data *elt_data;
2245         struct hist_field *key_field;
2246         unsigned int i, n_str;
2247
2248         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2249         if (!elt_data)
2250                 return -ENOMEM;
2251
2252         for_each_hist_key_field(i, hist_data) {
2253                 key_field = hist_data->fields[i];
2254
2255                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2256                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2257                         if (!elt_data->comm) {
2258                                 kfree(elt_data);
2259                                 return -ENOMEM;
2260                         }
2261                         break;
2262                 }
2263         }
2264
2265         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2266
2267         size = STR_VAR_LEN_MAX;
2268
2269         for (i = 0; i < n_str; i++) {
2270                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2271                 if (!elt_data->field_var_str[i]) {
2272                         hist_elt_data_free(elt_data);
2273                         return -ENOMEM;
2274                 }
2275         }
2276
2277         elt->private_data = elt_data;
2278
2279         return 0;
2280 }
2281
2282 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2283 {
2284         struct hist_elt_data *elt_data = elt->private_data;
2285
2286         if (elt_data->comm)
2287                 save_comm(elt_data->comm, current);
2288 }
2289
2290 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2291         .elt_alloc      = hist_trigger_elt_data_alloc,
2292         .elt_free       = hist_trigger_elt_data_free,
2293         .elt_init       = hist_trigger_elt_data_init,
2294 };
2295
2296 static const char *get_hist_field_flags(struct hist_field *hist_field)
2297 {
2298         const char *flags_str = NULL;
2299
2300         if (hist_field->flags & HIST_FIELD_FL_HEX)
2301                 flags_str = "hex";
2302         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2303                 flags_str = "sym";
2304         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2305                 flags_str = "sym-offset";
2306         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2307                 flags_str = "execname";
2308         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2309                 flags_str = "syscall";
2310         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2311                 flags_str = "log2";
2312         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2313                 flags_str = "usecs";
2314
2315         return flags_str;
2316 }
2317
2318 static void expr_field_str(struct hist_field *field, char *expr)
2319 {
2320         if (field->flags & HIST_FIELD_FL_VAR_REF)
2321                 strcat(expr, "$");
2322
2323         strcat(expr, hist_field_name(field, 0));
2324
2325         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2326                 const char *flags_str = get_hist_field_flags(field);
2327
2328                 if (flags_str) {
2329                         strcat(expr, ".");
2330                         strcat(expr, flags_str);
2331                 }
2332         }
2333 }
2334
2335 static char *expr_str(struct hist_field *field, unsigned int level)
2336 {
2337         char *expr;
2338
2339         if (level > 1)
2340                 return NULL;
2341
2342         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2343         if (!expr)
2344                 return NULL;
2345
2346         if (!field->operands[0]) {
2347                 expr_field_str(field, expr);
2348                 return expr;
2349         }
2350
2351         if (field->operator == FIELD_OP_UNARY_MINUS) {
2352                 char *subexpr;
2353
2354                 strcat(expr, "-(");
2355                 subexpr = expr_str(field->operands[0], ++level);
2356                 if (!subexpr) {
2357                         kfree(expr);
2358                         return NULL;
2359                 }
2360                 strcat(expr, subexpr);
2361                 strcat(expr, ")");
2362
2363                 kfree(subexpr);
2364
2365                 return expr;
2366         }
2367
2368         expr_field_str(field->operands[0], expr);
2369
2370         switch (field->operator) {
2371         case FIELD_OP_MINUS:
2372                 strcat(expr, "-");
2373                 break;
2374         case FIELD_OP_PLUS:
2375                 strcat(expr, "+");
2376                 break;
2377         default:
2378                 kfree(expr);
2379                 return NULL;
2380         }
2381
2382         expr_field_str(field->operands[1], expr);
2383
2384         return expr;
2385 }
2386
2387 static int contains_operator(char *str)
2388 {
2389         enum field_op_id field_op = FIELD_OP_NONE;
2390         char *op;
2391
2392         op = strpbrk(str, "+-");
2393         if (!op)
2394                 return FIELD_OP_NONE;
2395
2396         switch (*op) {
2397         case '-':
2398                 if (*str == '-')
2399                         field_op = FIELD_OP_UNARY_MINUS;
2400                 else
2401                         field_op = FIELD_OP_MINUS;
2402                 break;
2403         case '+':
2404                 field_op = FIELD_OP_PLUS;
2405                 break;
2406         default:
2407                 break;
2408         }
2409
2410         return field_op;
2411 }
2412
2413 static void __destroy_hist_field(struct hist_field *hist_field)
2414 {
2415         kfree(hist_field->var.name);
2416         kfree(hist_field->name);
2417         kfree(hist_field->type);
2418
2419         kfree(hist_field);
2420 }
2421
2422 static void destroy_hist_field(struct hist_field *hist_field,
2423                                unsigned int level)
2424 {
2425         unsigned int i;
2426
2427         if (level > 3)
2428                 return;
2429
2430         if (!hist_field)
2431                 return;
2432
2433         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2434                 return; /* var refs will be destroyed separately */
2435
2436         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2437                 destroy_hist_field(hist_field->operands[i], level + 1);
2438
2439         __destroy_hist_field(hist_field);
2440 }
2441
2442 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2443                                             struct ftrace_event_field *field,
2444                                             unsigned long flags,
2445                                             char *var_name)
2446 {
2447         struct hist_field *hist_field;
2448
2449         if (field && is_function_field(field))
2450                 return NULL;
2451
2452         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2453         if (!hist_field)
2454                 return NULL;
2455
2456         hist_field->hist_data = hist_data;
2457
2458         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2459                 goto out; /* caller will populate */
2460
2461         if (flags & HIST_FIELD_FL_VAR_REF) {
2462                 hist_field->fn = hist_field_var_ref;
2463                 goto out;
2464         }
2465
2466         if (flags & HIST_FIELD_FL_HITCOUNT) {
2467                 hist_field->fn = hist_field_counter;
2468                 hist_field->size = sizeof(u64);
2469                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2470                 if (!hist_field->type)
2471                         goto free;
2472                 goto out;
2473         }
2474
2475         if (flags & HIST_FIELD_FL_STACKTRACE) {
2476                 hist_field->fn = hist_field_none;
2477                 goto out;
2478         }
2479
2480         if (flags & HIST_FIELD_FL_LOG2) {
2481                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2482                 hist_field->fn = hist_field_log2;
2483                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2484                 hist_field->size = hist_field->operands[0]->size;
2485                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2486                 if (!hist_field->type)
2487                         goto free;
2488                 goto out;
2489         }
2490
2491         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2492                 hist_field->fn = hist_field_timestamp;
2493                 hist_field->size = sizeof(u64);
2494                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2495                 if (!hist_field->type)
2496                         goto free;
2497                 goto out;
2498         }
2499
2500         if (flags & HIST_FIELD_FL_CPU) {
2501                 hist_field->fn = hist_field_cpu;
2502                 hist_field->size = sizeof(int);
2503                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2504                 if (!hist_field->type)
2505                         goto free;
2506                 goto out;
2507         }
2508
2509         if (WARN_ON_ONCE(!field))
2510                 goto out;
2511
2512         if (is_string_field(field)) {
2513                 flags |= HIST_FIELD_FL_STRING;
2514
2515                 hist_field->size = MAX_FILTER_STR_VAL;
2516                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2517                 if (!hist_field->type)
2518                         goto free;
2519
2520                 if (field->filter_type == FILTER_STATIC_STRING)
2521                         hist_field->fn = hist_field_string;
2522                 else if (field->filter_type == FILTER_DYN_STRING)
2523                         hist_field->fn = hist_field_dynstring;
2524                 else
2525                         hist_field->fn = hist_field_pstring;
2526         } else {
2527                 hist_field->size = field->size;
2528                 hist_field->is_signed = field->is_signed;
2529                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2530                 if (!hist_field->type)
2531                         goto free;
2532
2533                 hist_field->fn = select_value_fn(field->size,
2534                                                  field->is_signed);
2535                 if (!hist_field->fn) {
2536                         destroy_hist_field(hist_field, 0);
2537                         return NULL;
2538                 }
2539         }
2540  out:
2541         hist_field->field = field;
2542         hist_field->flags = flags;
2543
2544         if (var_name) {
2545                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2546                 if (!hist_field->var.name)
2547                         goto free;
2548         }
2549
2550         return hist_field;
2551  free:
2552         destroy_hist_field(hist_field, 0);
2553         return NULL;
2554 }
2555
2556 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2557 {
2558         unsigned int i;
2559
2560         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2561                 if (hist_data->fields[i]) {
2562                         destroy_hist_field(hist_data->fields[i], 0);
2563                         hist_data->fields[i] = NULL;
2564                 }
2565         }
2566
2567         for (i = 0; i < hist_data->n_var_refs; i++) {
2568                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2569                 __destroy_hist_field(hist_data->var_refs[i]);
2570                 hist_data->var_refs[i] = NULL;
2571         }
2572 }
2573
2574 static int init_var_ref(struct hist_field *ref_field,
2575                         struct hist_field *var_field,
2576                         char *system, char *event_name)
2577 {
2578         int err = 0;
2579
2580         ref_field->var.idx = var_field->var.idx;
2581         ref_field->var.hist_data = var_field->hist_data;
2582         ref_field->size = var_field->size;
2583         ref_field->is_signed = var_field->is_signed;
2584         ref_field->flags |= var_field->flags &
2585                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2586
2587         if (system) {
2588                 ref_field->system = kstrdup(system, GFP_KERNEL);
2589                 if (!ref_field->system)
2590                         return -ENOMEM;
2591         }
2592
2593         if (event_name) {
2594                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2595                 if (!ref_field->event_name) {
2596                         err = -ENOMEM;
2597                         goto free;
2598                 }
2599         }
2600
2601         if (var_field->var.name) {
2602                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2603                 if (!ref_field->name) {
2604                         err = -ENOMEM;
2605                         goto free;
2606                 }
2607         } else if (var_field->name) {
2608                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2609                 if (!ref_field->name) {
2610                         err = -ENOMEM;
2611                         goto free;
2612                 }
2613         }
2614
2615         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2616         if (!ref_field->type) {
2617                 err = -ENOMEM;
2618                 goto free;
2619         }
2620  out:
2621         return err;
2622  free:
2623         kfree(ref_field->system);
2624         kfree(ref_field->event_name);
2625         kfree(ref_field->name);
2626
2627         goto out;
2628 }
2629
2630 /**
2631  * create_var_ref - Create a variable reference and attach it to trigger
2632  * @hist_data: The trigger that will be referencing the variable
2633  * @var_field: The VAR field to create a reference to
2634  * @system: The optional system string
2635  * @event_name: The optional event_name string
2636  *
2637  * Given a variable hist_field, create a VAR_REF hist_field that
2638  * represents a reference to it.
2639  *
2640  * This function also adds the reference to the trigger that
2641  * now references the variable.
2642  *
2643  * Return: The VAR_REF field if successful, NULL if not
2644  */
2645 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2646                                          struct hist_field *var_field,
2647                                          char *system, char *event_name)
2648 {
2649         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2650         struct hist_field *ref_field;
2651
2652         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2653         if (ref_field) {
2654                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2655                         destroy_hist_field(ref_field, 0);
2656                         return NULL;
2657                 }
2658
2659                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2660                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2661         }
2662
2663         return ref_field;
2664 }
2665
2666 static bool is_var_ref(char *var_name)
2667 {
2668         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2669                 return false;
2670
2671         return true;
2672 }
2673
2674 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2675                                  char *var_name)
2676 {
2677         char *name, *field;
2678         unsigned int i;
2679
2680         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2681                 name = hist_data->attrs->var_defs.name[i];
2682
2683                 if (strcmp(var_name, name) == 0) {
2684                         field = hist_data->attrs->var_defs.expr[i];
2685                         if (contains_operator(field) || is_var_ref(field))
2686                                 continue;
2687                         return field;
2688                 }
2689         }
2690
2691         return NULL;
2692 }
2693
2694 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2695                                  char *system, char *event_name,
2696                                  char *var_name)
2697 {
2698         struct trace_event_call *call;
2699
2700         if (system && event_name) {
2701                 call = hist_data->event_file->event_call;
2702
2703                 if (strcmp(system, call->class->system) != 0)
2704                         return NULL;
2705
2706                 if (strcmp(event_name, trace_event_name(call)) != 0)
2707                         return NULL;
2708         }
2709
2710         if (!!system != !!event_name)
2711                 return NULL;
2712
2713         if (!is_var_ref(var_name))
2714                 return NULL;
2715
2716         var_name++;
2717
2718         return field_name_from_var(hist_data, var_name);
2719 }
2720
2721 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2722                                         char *system, char *event_name,
2723                                         char *var_name)
2724 {
2725         struct hist_field *var_field = NULL, *ref_field = NULL;
2726         struct trace_array *tr = hist_data->event_file->tr;
2727
2728         if (!is_var_ref(var_name))
2729                 return NULL;
2730
2731         var_name++;
2732
2733         var_field = find_event_var(hist_data, system, event_name, var_name);
2734         if (var_field)
2735                 ref_field = create_var_ref(hist_data, var_field,
2736                                            system, event_name);
2737
2738         if (!ref_field)
2739                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2740
2741         return ref_field;
2742 }
2743
2744 static struct ftrace_event_field *
2745 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2746             char *field_str, unsigned long *flags)
2747 {
2748         struct ftrace_event_field *field = NULL;
2749         char *field_name, *modifier, *str;
2750         struct trace_array *tr = file->tr;
2751
2752         modifier = str = kstrdup(field_str, GFP_KERNEL);
2753         if (!modifier)
2754                 return ERR_PTR(-ENOMEM);
2755
2756         field_name = strsep(&modifier, ".");
2757         if (modifier) {
2758                 if (strcmp(modifier, "hex") == 0)
2759                         *flags |= HIST_FIELD_FL_HEX;
2760                 else if (strcmp(modifier, "sym") == 0)
2761                         *flags |= HIST_FIELD_FL_SYM;
2762                 else if (strcmp(modifier, "sym-offset") == 0)
2763                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2764                 else if ((strcmp(modifier, "execname") == 0) &&
2765                          (strcmp(field_name, "common_pid") == 0))
2766                         *flags |= HIST_FIELD_FL_EXECNAME;
2767                 else if (strcmp(modifier, "syscall") == 0)
2768                         *flags |= HIST_FIELD_FL_SYSCALL;
2769                 else if (strcmp(modifier, "log2") == 0)
2770                         *flags |= HIST_FIELD_FL_LOG2;
2771                 else if (strcmp(modifier, "usecs") == 0)
2772                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2773                 else {
2774                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2775                         field = ERR_PTR(-EINVAL);
2776                         goto out;
2777                 }
2778         }
2779
2780         if (strcmp(field_name, "common_timestamp") == 0) {
2781                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2782                 hist_data->enable_timestamps = true;
2783                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2784                         hist_data->attrs->ts_in_usecs = true;
2785         } else if (strcmp(field_name, "cpu") == 0)
2786                 *flags |= HIST_FIELD_FL_CPU;
2787         else {
2788                 field = trace_find_event_field(file->event_call, field_name);
2789                 if (!field || !field->size) {
2790                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2791                         field = ERR_PTR(-EINVAL);
2792                         goto out;
2793                 }
2794         }
2795  out:
2796         kfree(str);
2797
2798         return field;
2799 }
2800
2801 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2802                                        struct hist_field *var_ref,
2803                                        char *var_name)
2804 {
2805         struct hist_field *alias = NULL;
2806         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2807
2808         alias = create_hist_field(hist_data, NULL, flags, var_name);
2809         if (!alias)
2810                 return NULL;
2811
2812         alias->fn = var_ref->fn;
2813         alias->operands[0] = var_ref;
2814
2815         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2816                 destroy_hist_field(alias, 0);
2817                 return NULL;
2818         }
2819
2820         alias->var_ref_idx = var_ref->var_ref_idx;
2821
2822         return alias;
2823 }
2824
2825 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2826                                      struct trace_event_file *file, char *str,
2827                                      unsigned long *flags, char *var_name)
2828 {
2829         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2830         struct ftrace_event_field *field = NULL;
2831         struct hist_field *hist_field = NULL;
2832         int ret = 0;
2833
2834         s = strchr(str, '.');
2835         if (s) {
2836                 s = strchr(++s, '.');
2837                 if (s) {
2838                         ref_system = strsep(&str, ".");
2839                         if (!str) {
2840                                 ret = -EINVAL;
2841                                 goto out;
2842                         }
2843                         ref_event = strsep(&str, ".");
2844                         if (!str) {
2845                                 ret = -EINVAL;
2846                                 goto out;
2847                         }
2848                         ref_var = str;
2849                 }
2850         }
2851
2852         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2853         if (!s) {
2854                 hist_field = parse_var_ref(hist_data, ref_system,
2855                                            ref_event, ref_var);
2856                 if (hist_field) {
2857                         if (var_name) {
2858                                 hist_field = create_alias(hist_data, hist_field, var_name);
2859                                 if (!hist_field) {
2860                                         ret = -ENOMEM;
2861                                         goto out;
2862                                 }
2863                         }
2864                         return hist_field;
2865                 }
2866         } else
2867                 str = s;
2868
2869         field = parse_field(hist_data, file, str, flags);
2870         if (IS_ERR(field)) {
2871                 ret = PTR_ERR(field);
2872                 goto out;
2873         }
2874
2875         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2876         if (!hist_field) {
2877                 ret = -ENOMEM;
2878                 goto out;
2879         }
2880
2881         return hist_field;
2882  out:
2883         return ERR_PTR(ret);
2884 }
2885
2886 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2887                                      struct trace_event_file *file,
2888                                      char *str, unsigned long flags,
2889                                      char *var_name, unsigned int level);
2890
2891 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2892                                       struct trace_event_file *file,
2893                                       char *str, unsigned long flags,
2894                                       char *var_name, unsigned int level)
2895 {
2896         struct hist_field *operand1, *expr = NULL;
2897         unsigned long operand_flags;
2898         int ret = 0;
2899         char *s;
2900
2901         /* we support only -(xxx) i.e. explicit parens required */
2902
2903         if (level > 3) {
2904                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2905                 ret = -EINVAL;
2906                 goto free;
2907         }
2908
2909         str++; /* skip leading '-' */
2910
2911         s = strchr(str, '(');
2912         if (s)
2913                 str++;
2914         else {
2915                 ret = -EINVAL;
2916                 goto free;
2917         }
2918
2919         s = strrchr(str, ')');
2920         if (s)
2921                 *s = '\0';
2922         else {
2923                 ret = -EINVAL; /* no closing ')' */
2924                 goto free;
2925         }
2926
2927         flags |= HIST_FIELD_FL_EXPR;
2928         expr = create_hist_field(hist_data, NULL, flags, var_name);
2929         if (!expr) {
2930                 ret = -ENOMEM;
2931                 goto free;
2932         }
2933
2934         operand_flags = 0;
2935         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2936         if (IS_ERR(operand1)) {
2937                 ret = PTR_ERR(operand1);
2938                 goto free;
2939         }
2940
2941         expr->flags |= operand1->flags &
2942                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2943         expr->fn = hist_field_unary_minus;
2944         expr->operands[0] = operand1;
2945         expr->operator = FIELD_OP_UNARY_MINUS;
2946         expr->name = expr_str(expr, 0);
2947         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2948         if (!expr->type) {
2949                 ret = -ENOMEM;
2950                 goto free;
2951         }
2952
2953         return expr;
2954  free:
2955         destroy_hist_field(expr, 0);
2956         return ERR_PTR(ret);
2957 }
2958
2959 static int check_expr_operands(struct trace_array *tr,
2960                                struct hist_field *operand1,
2961                                struct hist_field *operand2)
2962 {
2963         unsigned long operand1_flags = operand1->flags;
2964         unsigned long operand2_flags = operand2->flags;
2965
2966         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2967             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2968                 struct hist_field *var;
2969
2970                 var = find_var_field(operand1->var.hist_data, operand1->name);
2971                 if (!var)
2972                         return -EINVAL;
2973                 operand1_flags = var->flags;
2974         }
2975
2976         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2977             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2978                 struct hist_field *var;
2979
2980                 var = find_var_field(operand2->var.hist_data, operand2->name);
2981                 if (!var)
2982                         return -EINVAL;
2983                 operand2_flags = var->flags;
2984         }
2985
2986         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2987             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2988                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2989                 return -EINVAL;
2990         }
2991
2992         return 0;
2993 }
2994
2995 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2996                                      struct trace_event_file *file,
2997                                      char *str, unsigned long flags,
2998                                      char *var_name, unsigned int level)
2999 {
3000         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
3001         unsigned long operand_flags;
3002         int field_op, ret = -EINVAL;
3003         char *sep, *operand1_str;
3004
3005         if (level > 3) {
3006                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3007                 return ERR_PTR(-EINVAL);
3008         }
3009
3010         field_op = contains_operator(str);
3011
3012         if (field_op == FIELD_OP_NONE)
3013                 return parse_atom(hist_data, file, str, &flags, var_name);
3014
3015         if (field_op == FIELD_OP_UNARY_MINUS)
3016                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
3017
3018         switch (field_op) {
3019         case FIELD_OP_MINUS:
3020                 sep = "-";
3021                 break;
3022         case FIELD_OP_PLUS:
3023                 sep = "+";
3024                 break;
3025         default:
3026                 goto free;
3027         }
3028
3029         operand1_str = strsep(&str, sep);
3030         if (!operand1_str || !str)
3031                 goto free;
3032
3033         operand_flags = 0;
3034         operand1 = parse_atom(hist_data, file, operand1_str,
3035                               &operand_flags, NULL);
3036         if (IS_ERR(operand1)) {
3037                 ret = PTR_ERR(operand1);
3038                 operand1 = NULL;
3039                 goto free;
3040         }
3041
3042         /* rest of string could be another expression e.g. b+c in a+b+c */
3043         operand_flags = 0;
3044         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3045         if (IS_ERR(operand2)) {
3046                 ret = PTR_ERR(operand2);
3047                 operand2 = NULL;
3048                 goto free;
3049         }
3050
3051         ret = check_expr_operands(file->tr, operand1, operand2);
3052         if (ret)
3053                 goto free;
3054
3055         flags |= HIST_FIELD_FL_EXPR;
3056
3057         flags |= operand1->flags &
3058                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3059
3060         expr = create_hist_field(hist_data, NULL, flags, var_name);
3061         if (!expr) {
3062                 ret = -ENOMEM;
3063                 goto free;
3064         }
3065
3066         operand1->read_once = true;
3067         operand2->read_once = true;
3068
3069         expr->operands[0] = operand1;
3070         expr->operands[1] = operand2;
3071         expr->operator = field_op;
3072         expr->name = expr_str(expr, 0);
3073         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3074         if (!expr->type) {
3075                 ret = -ENOMEM;
3076                 goto free;
3077         }
3078
3079         switch (field_op) {
3080         case FIELD_OP_MINUS:
3081                 expr->fn = hist_field_minus;
3082                 break;
3083         case FIELD_OP_PLUS:
3084                 expr->fn = hist_field_plus;
3085                 break;
3086         default:
3087                 ret = -EINVAL;
3088                 goto free;
3089         }
3090
3091         return expr;
3092  free:
3093         destroy_hist_field(operand1, 0);
3094         destroy_hist_field(operand2, 0);
3095         destroy_hist_field(expr, 0);
3096
3097         return ERR_PTR(ret);
3098 }
3099
3100 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3101                                  struct trace_event_file *file)
3102 {
3103         struct event_trigger_data *test;
3104
3105         list_for_each_entry_rcu(test, &file->triggers, list) {
3106                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3107                         if (test->private_data == hist_data)
3108                                 return test->filter_str;
3109                 }
3110         }
3111
3112         return NULL;
3113 }
3114
3115 static struct event_command trigger_hist_cmd;
3116 static int event_hist_trigger_func(struct event_command *cmd_ops,
3117                                    struct trace_event_file *file,
3118                                    char *glob, char *cmd, char *param);
3119
3120 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3121                             struct hist_trigger_data *hist_data,
3122                             unsigned int n_keys)
3123 {
3124         struct hist_field *target_hist_field, *hist_field;
3125         unsigned int n, i, j;
3126
3127         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3128                 return false;
3129
3130         i = hist_data->n_vals;
3131         j = target_hist_data->n_vals;
3132
3133         for (n = 0; n < n_keys; n++) {
3134                 hist_field = hist_data->fields[i + n];
3135                 target_hist_field = target_hist_data->fields[j + n];
3136
3137                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3138                         return false;
3139                 if (hist_field->size != target_hist_field->size)
3140                         return false;
3141                 if (hist_field->is_signed != target_hist_field->is_signed)
3142                         return false;
3143         }
3144
3145         return true;
3146 }
3147
3148 static struct hist_trigger_data *
3149 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3150                      struct trace_event_file *file)
3151 {
3152         struct hist_trigger_data *hist_data;
3153         struct event_trigger_data *test;
3154         unsigned int n_keys;
3155
3156         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3157
3158         list_for_each_entry_rcu(test, &file->triggers, list) {
3159                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3160                         hist_data = test->private_data;
3161
3162                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3163                                 return hist_data;
3164                 }
3165         }
3166
3167         return NULL;
3168 }
3169
3170 static struct trace_event_file *event_file(struct trace_array *tr,
3171                                            char *system, char *event_name)
3172 {
3173         struct trace_event_file *file;
3174
3175         file = __find_event_file(tr, system, event_name);
3176         if (!file)
3177                 return ERR_PTR(-EINVAL);
3178
3179         return file;
3180 }
3181
3182 static struct hist_field *
3183 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3184                          char *system, char *event_name, char *field_name)
3185 {
3186         struct hist_field *event_var;
3187         char *synthetic_name;
3188
3189         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3190         if (!synthetic_name)
3191                 return ERR_PTR(-ENOMEM);
3192
3193         strcpy(synthetic_name, "synthetic_");
3194         strcat(synthetic_name, field_name);
3195
3196         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3197
3198         kfree(synthetic_name);
3199
3200         return event_var;
3201 }
3202
3203 /**
3204  * create_field_var_hist - Automatically create a histogram and var for a field
3205  * @target_hist_data: The target hist trigger
3206  * @subsys_name: Optional subsystem name
3207  * @event_name: Optional event name
3208  * @field_name: The name of the field (and the resulting variable)
3209  *
3210  * Hist trigger actions fetch data from variables, not directly from
3211  * events.  However, for convenience, users are allowed to directly
3212  * specify an event field in an action, which will be automatically
3213  * converted into a variable on their behalf.
3214
3215  * If a user specifies a field on an event that isn't the event the
3216  * histogram currently being defined (the target event histogram), the
3217  * only way that can be accomplished is if a new hist trigger is
3218  * created and the field variable defined on that.
3219  *
3220  * This function creates a new histogram compatible with the target
3221  * event (meaning a histogram with the same key as the target
3222  * histogram), and creates a variable for the specified field, but
3223  * with 'synthetic_' prepended to the variable name in order to avoid
3224  * collision with normal field variables.
3225  *
3226  * Return: The variable created for the field.
3227  */
3228 static struct hist_field *
3229 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3230                       char *subsys_name, char *event_name, char *field_name)
3231 {
3232         struct trace_array *tr = target_hist_data->event_file->tr;
3233         struct hist_field *event_var = ERR_PTR(-EINVAL);
3234         struct hist_trigger_data *hist_data;
3235         unsigned int i, n, first = true;
3236         struct field_var_hist *var_hist;
3237         struct trace_event_file *file;
3238         struct hist_field *key_field;
3239         char *saved_filter;
3240         char *cmd;
3241         int ret;
3242
3243         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3244                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3245                 return ERR_PTR(-EINVAL);
3246         }
3247
3248         file = event_file(tr, subsys_name, event_name);
3249
3250         if (IS_ERR(file)) {
3251                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3252                 ret = PTR_ERR(file);
3253                 return ERR_PTR(ret);
3254         }
3255
3256         /*
3257          * Look for a histogram compatible with target.  We'll use the
3258          * found histogram specification to create a new matching
3259          * histogram with our variable on it.  target_hist_data is not
3260          * yet a registered histogram so we can't use that.
3261          */
3262         hist_data = find_compatible_hist(target_hist_data, file);
3263         if (!hist_data) {
3264                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3265                 return ERR_PTR(-EINVAL);
3266         }
3267
3268         /* See if a synthetic field variable has already been created */
3269         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3270                                              event_name, field_name);
3271         if (!IS_ERR_OR_NULL(event_var))
3272                 return event_var;
3273
3274         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3275         if (!var_hist)
3276                 return ERR_PTR(-ENOMEM);
3277
3278         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3279         if (!cmd) {
3280                 kfree(var_hist);
3281                 return ERR_PTR(-ENOMEM);
3282         }
3283
3284         /* Use the same keys as the compatible histogram */
3285         strcat(cmd, "keys=");
3286
3287         for_each_hist_key_field(i, hist_data) {
3288                 key_field = hist_data->fields[i];
3289                 if (!first)
3290                         strcat(cmd, ",");
3291                 strcat(cmd, key_field->field->name);
3292                 first = false;
3293         }
3294
3295         /* Create the synthetic field variable specification */
3296         strcat(cmd, ":synthetic_");
3297         strcat(cmd, field_name);
3298         strcat(cmd, "=");
3299         strcat(cmd, field_name);
3300
3301         /* Use the same filter as the compatible histogram */
3302         saved_filter = find_trigger_filter(hist_data, file);
3303         if (saved_filter) {
3304                 strcat(cmd, " if ");
3305                 strcat(cmd, saved_filter);
3306         }
3307
3308         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3309         if (!var_hist->cmd) {
3310                 kfree(cmd);
3311                 kfree(var_hist);
3312                 return ERR_PTR(-ENOMEM);
3313         }
3314
3315         /* Save the compatible histogram information */
3316         var_hist->hist_data = hist_data;
3317
3318         /* Create the new histogram with our variable */
3319         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3320                                       "", "hist", cmd);
3321         if (ret) {
3322                 kfree(cmd);
3323                 kfree(var_hist->cmd);
3324                 kfree(var_hist);
3325                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3326                 return ERR_PTR(ret);
3327         }
3328
3329         kfree(cmd);
3330
3331         /* If we can't find the variable, something went wrong */
3332         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3333                                              event_name, field_name);
3334         if (IS_ERR_OR_NULL(event_var)) {
3335                 kfree(var_hist->cmd);
3336                 kfree(var_hist);
3337                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3338                 return ERR_PTR(-EINVAL);
3339         }
3340
3341         n = target_hist_data->n_field_var_hists;
3342         target_hist_data->field_var_hists[n] = var_hist;
3343         target_hist_data->n_field_var_hists++;
3344
3345         return event_var;
3346 }
3347
3348 static struct hist_field *
3349 find_target_event_var(struct hist_trigger_data *hist_data,
3350                       char *subsys_name, char *event_name, char *var_name)
3351 {
3352         struct trace_event_file *file = hist_data->event_file;
3353         struct hist_field *hist_field = NULL;
3354
3355         if (subsys_name) {
3356                 struct trace_event_call *call;
3357
3358                 if (!event_name)
3359                         return NULL;
3360
3361                 call = file->event_call;
3362
3363                 if (strcmp(subsys_name, call->class->system) != 0)
3364                         return NULL;
3365
3366                 if (strcmp(event_name, trace_event_name(call)) != 0)
3367                         return NULL;
3368         }
3369
3370         hist_field = find_var_field(hist_data, var_name);
3371
3372         return hist_field;
3373 }
3374
3375 static inline void __update_field_vars(struct tracing_map_elt *elt,
3376                                        struct ring_buffer_event *rbe,
3377                                        void *rec,
3378                                        struct field_var **field_vars,
3379                                        unsigned int n_field_vars,
3380                                        unsigned int field_var_str_start)
3381 {
3382         struct hist_elt_data *elt_data = elt->private_data;
3383         unsigned int i, j, var_idx;
3384         u64 var_val;
3385
3386         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3387                 struct field_var *field_var = field_vars[i];
3388                 struct hist_field *var = field_var->var;
3389                 struct hist_field *val = field_var->val;
3390
3391                 var_val = val->fn(val, elt, rbe, rec);
3392                 var_idx = var->var.idx;
3393
3394                 if (val->flags & HIST_FIELD_FL_STRING) {
3395                         char *str = elt_data->field_var_str[j++];
3396                         char *val_str = (char *)(uintptr_t)var_val;
3397
3398                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3399                         var_val = (u64)(uintptr_t)str;
3400                 }
3401                 tracing_map_set_var(elt, var_idx, var_val);
3402         }
3403 }
3404
3405 static void update_field_vars(struct hist_trigger_data *hist_data,
3406                               struct tracing_map_elt *elt,
3407                               struct ring_buffer_event *rbe,
3408                               void *rec)
3409 {
3410         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3411                             hist_data->n_field_vars, 0);
3412 }
3413
3414 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3415                                  struct tracing_map_elt *elt, void *rec,
3416                                  struct ring_buffer_event *rbe, void *key,
3417                                  struct action_data *data, u64 *var_ref_vals)
3418 {
3419         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3420                             hist_data->n_save_vars, hist_data->n_field_var_str);
3421 }
3422
3423 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3424                                      struct trace_event_file *file,
3425                                      char *name, int size, const char *type)
3426 {
3427         struct hist_field *var;
3428         int idx;
3429
3430         if (find_var(hist_data, file, name) && !hist_data->remove) {
3431                 var = ERR_PTR(-EINVAL);
3432                 goto out;
3433         }
3434
3435         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3436         if (!var) {
3437                 var = ERR_PTR(-ENOMEM);
3438                 goto out;
3439         }
3440
3441         idx = tracing_map_add_var(hist_data->map);
3442         if (idx < 0) {
3443                 kfree(var);
3444                 var = ERR_PTR(-EINVAL);
3445                 goto out;
3446         }
3447
3448         var->flags = HIST_FIELD_FL_VAR;
3449         var->var.idx = idx;
3450         var->var.hist_data = var->hist_data = hist_data;
3451         var->size = size;
3452         var->var.name = kstrdup(name, GFP_KERNEL);
3453         var->type = kstrdup(type, GFP_KERNEL);
3454         if (!var->var.name || !var->type) {
3455                 kfree(var->var.name);
3456                 kfree(var->type);
3457                 kfree(var);
3458                 var = ERR_PTR(-ENOMEM);
3459         }
3460  out:
3461         return var;
3462 }
3463
3464 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3465                                           struct trace_event_file *file,
3466                                           char *field_name)
3467 {
3468         struct hist_field *val = NULL, *var = NULL;
3469         unsigned long flags = HIST_FIELD_FL_VAR;
3470         struct trace_array *tr = file->tr;
3471         struct field_var *field_var;
3472         int ret = 0;
3473
3474         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3475                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3476                 ret = -EINVAL;
3477                 goto err;
3478         }
3479
3480         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3481         if (IS_ERR(val)) {
3482                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3483                 ret = PTR_ERR(val);
3484                 goto err;
3485         }
3486
3487         var = create_var(hist_data, file, field_name, val->size, val->type);
3488         if (IS_ERR(var)) {
3489                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3490                 kfree(val);
3491                 ret = PTR_ERR(var);
3492                 goto err;
3493         }
3494
3495         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3496         if (!field_var) {
3497                 kfree(val);
3498                 kfree(var);
3499                 ret =  -ENOMEM;
3500                 goto err;
3501         }
3502
3503         field_var->var = var;
3504         field_var->val = val;
3505  out:
3506         return field_var;
3507  err:
3508         field_var = ERR_PTR(ret);
3509         goto out;
3510 }
3511
3512 /**
3513  * create_target_field_var - Automatically create a variable for a field
3514  * @target_hist_data: The target hist trigger
3515  * @subsys_name: Optional subsystem name
3516  * @event_name: Optional event name
3517  * @var_name: The name of the field (and the resulting variable)
3518  *
3519  * Hist trigger actions fetch data from variables, not directly from
3520  * events.  However, for convenience, users are allowed to directly
3521  * specify an event field in an action, which will be automatically
3522  * converted into a variable on their behalf.
3523
3524  * This function creates a field variable with the name var_name on
3525  * the hist trigger currently being defined on the target event.  If
3526  * subsys_name and event_name are specified, this function simply
3527  * verifies that they do in fact match the target event subsystem and
3528  * event name.
3529  *
3530  * Return: The variable created for the field.
3531  */
3532 static struct field_var *
3533 create_target_field_var(struct hist_trigger_data *target_hist_data,
3534                         char *subsys_name, char *event_name, char *var_name)
3535 {
3536         struct trace_event_file *file = target_hist_data->event_file;
3537
3538         if (subsys_name) {
3539                 struct trace_event_call *call;
3540
3541                 if (!event_name)
3542                         return NULL;
3543
3544                 call = file->event_call;
3545
3546                 if (strcmp(subsys_name, call->class->system) != 0)
3547                         return NULL;
3548
3549                 if (strcmp(event_name, trace_event_name(call)) != 0)
3550                         return NULL;
3551         }
3552
3553         return create_field_var(target_hist_data, file, var_name);
3554 }
3555
3556 static bool check_track_val_max(u64 track_val, u64 var_val)
3557 {
3558         if (var_val <= track_val)
3559                 return false;
3560
3561         return true;
3562 }
3563
3564 static bool check_track_val_changed(u64 track_val, u64 var_val)
3565 {
3566         if (var_val == track_val)
3567                 return false;
3568
3569         return true;
3570 }
3571
3572 static u64 get_track_val(struct hist_trigger_data *hist_data,
3573                          struct tracing_map_elt *elt,
3574                          struct action_data *data)
3575 {
3576         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3577         u64 track_val;
3578
3579         track_val = tracing_map_read_var(elt, track_var_idx);
3580
3581         return track_val;
3582 }
3583
3584 static void save_track_val(struct hist_trigger_data *hist_data,
3585                            struct tracing_map_elt *elt,
3586                            struct action_data *data, u64 var_val)
3587 {
3588         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3589
3590         tracing_map_set_var(elt, track_var_idx, var_val);
3591 }
3592
3593 static void save_track_data(struct hist_trigger_data *hist_data,
3594                             struct tracing_map_elt *elt, void *rec,
3595                             struct ring_buffer_event *rbe, void *key,
3596                             struct action_data *data, u64 *var_ref_vals)
3597 {
3598         if (data->track_data.save_data)
3599                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3600 }
3601
3602 static bool check_track_val(struct tracing_map_elt *elt,
3603                             struct action_data *data,
3604                             u64 var_val)
3605 {
3606         struct hist_trigger_data *hist_data;
3607         u64 track_val;
3608
3609         hist_data = data->track_data.track_var->hist_data;
3610         track_val = get_track_val(hist_data, elt, data);
3611
3612         return data->track_data.check_val(track_val, var_val);
3613 }
3614
3615 #ifdef CONFIG_TRACER_SNAPSHOT
3616 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3617 {
3618         /* called with tr->max_lock held */
3619         struct track_data *track_data = tr->cond_snapshot->cond_data;
3620         struct hist_elt_data *elt_data, *track_elt_data;
3621         struct snapshot_context *context = cond_data;
3622         struct action_data *action;
3623         u64 track_val;
3624
3625         if (!track_data)
3626                 return false;
3627
3628         action = track_data->action_data;
3629
3630         track_val = get_track_val(track_data->hist_data, context->elt,
3631                                   track_data->action_data);
3632
3633         if (!action->track_data.check_val(track_data->track_val, track_val))
3634                 return false;
3635
3636         track_data->track_val = track_val;
3637         memcpy(track_data->key, context->key, track_data->key_len);
3638
3639         elt_data = context->elt->private_data;
3640         track_elt_data = track_data->elt.private_data;
3641         if (elt_data->comm)
3642                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3643
3644         track_data->updated = true;
3645
3646         return true;
3647 }
3648
3649 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3650                                      struct tracing_map_elt *elt, void *rec,
3651                                      struct ring_buffer_event *rbe, void *key,
3652                                      struct action_data *data,
3653                                      u64 *var_ref_vals)
3654 {
3655         struct trace_event_file *file = hist_data->event_file;
3656         struct snapshot_context context;
3657
3658         context.elt = elt;
3659         context.key = key;
3660
3661         tracing_snapshot_cond(file->tr, &context);
3662 }
3663
3664 static void hist_trigger_print_key(struct seq_file *m,
3665                                    struct hist_trigger_data *hist_data,
3666                                    void *key,
3667                                    struct tracing_map_elt *elt);
3668
3669 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3670 {
3671         unsigned int i;
3672
3673         if (!hist_data->n_actions)
3674                 return NULL;
3675
3676         for (i = 0; i < hist_data->n_actions; i++) {
3677                 struct action_data *data = hist_data->actions[i];
3678
3679                 if (data->action == ACTION_SNAPSHOT)
3680                         return data;
3681         }
3682
3683         return NULL;
3684 }
3685
3686 static void track_data_snapshot_print(struct seq_file *m,
3687                                       struct hist_trigger_data *hist_data)
3688 {
3689         struct trace_event_file *file = hist_data->event_file;
3690         struct track_data *track_data;
3691         struct action_data *action;
3692
3693         track_data = tracing_cond_snapshot_data(file->tr);
3694         if (!track_data)
3695                 return;
3696
3697         if (!track_data->updated)
3698                 return;
3699
3700         action = snapshot_action(hist_data);
3701         if (!action)
3702                 return;
3703
3704         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3705         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3706                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3707                    action->track_data.var_str, track_data->track_val);
3708
3709         seq_puts(m, "\ttriggered by event with key: ");
3710         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3711         seq_putc(m, '\n');
3712 }
3713 #else
3714 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3715 {
3716         return false;
3717 }
3718 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3719                                      struct tracing_map_elt *elt, void *rec,
3720                                      struct ring_buffer_event *rbe, void *key,
3721                                      struct action_data *data,
3722                                      u64 *var_ref_vals) {}
3723 static void track_data_snapshot_print(struct seq_file *m,
3724                                       struct hist_trigger_data *hist_data) {}
3725 #endif /* CONFIG_TRACER_SNAPSHOT */
3726
3727 static void track_data_print(struct seq_file *m,
3728                              struct hist_trigger_data *hist_data,
3729                              struct tracing_map_elt *elt,
3730                              struct action_data *data)
3731 {
3732         u64 track_val = get_track_val(hist_data, elt, data);
3733         unsigned int i, save_var_idx;
3734
3735         if (data->handler == HANDLER_ONMAX)
3736                 seq_printf(m, "\n\tmax: %10llu", track_val);
3737         else if (data->handler == HANDLER_ONCHANGE)
3738                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3739
3740         if (data->action == ACTION_SNAPSHOT)
3741                 return;
3742
3743         for (i = 0; i < hist_data->n_save_vars; i++) {
3744                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3745                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3746                 u64 val;
3747
3748                 save_var_idx = save_var->var.idx;
3749
3750                 val = tracing_map_read_var(elt, save_var_idx);
3751
3752                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3753                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3754                                    (char *)(uintptr_t)(val));
3755                 } else
3756                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3757         }
3758 }
3759
3760 static void ontrack_action(struct hist_trigger_data *hist_data,
3761                            struct tracing_map_elt *elt, void *rec,
3762                            struct ring_buffer_event *rbe, void *key,
3763                            struct action_data *data, u64 *var_ref_vals)
3764 {
3765         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3766
3767         if (check_track_val(elt, data, var_val)) {
3768                 save_track_val(hist_data, elt, data, var_val);
3769                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3770         }
3771 }
3772
3773 static void action_data_destroy(struct action_data *data)
3774 {
3775         unsigned int i;
3776
3777         lockdep_assert_held(&event_mutex);
3778
3779         kfree(data->action_name);
3780
3781         for (i = 0; i < data->n_params; i++)
3782                 kfree(data->params[i]);
3783
3784         if (data->synth_event)
3785                 data->synth_event->ref--;
3786
3787         kfree(data->synth_event_name);
3788
3789         kfree(data);
3790 }
3791
3792 static void track_data_destroy(struct hist_trigger_data *hist_data,
3793                                struct action_data *data)
3794 {
3795         struct trace_event_file *file = hist_data->event_file;
3796
3797         destroy_hist_field(data->track_data.track_var, 0);
3798
3799         if (data->action == ACTION_SNAPSHOT) {
3800                 struct track_data *track_data;
3801
3802                 track_data = tracing_cond_snapshot_data(file->tr);
3803                 if (track_data && track_data->hist_data == hist_data) {
3804                         tracing_snapshot_cond_disable(file->tr);
3805                         track_data_free(track_data);
3806                 }
3807         }
3808
3809         kfree(data->track_data.var_str);
3810
3811         action_data_destroy(data);
3812 }
3813
3814 static int action_create(struct hist_trigger_data *hist_data,
3815                          struct action_data *data);
3816
3817 static int track_data_create(struct hist_trigger_data *hist_data,
3818                              struct action_data *data)
3819 {
3820         struct hist_field *var_field, *ref_field, *track_var = NULL;
3821         struct trace_event_file *file = hist_data->event_file;
3822         struct trace_array *tr = file->tr;
3823         char *track_data_var_str;
3824         int ret = 0;
3825
3826         track_data_var_str = data->track_data.var_str;
3827         if (track_data_var_str[0] != '$') {
3828                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3829                 return -EINVAL;
3830         }
3831         track_data_var_str++;
3832
3833         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3834         if (!var_field) {
3835                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3836                 return -EINVAL;
3837         }
3838
3839         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3840         if (!ref_field)
3841                 return -ENOMEM;
3842
3843         data->track_data.var_ref = ref_field;
3844
3845         if (data->handler == HANDLER_ONMAX)
3846                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3847         if (IS_ERR(track_var)) {
3848                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3849                 ret = PTR_ERR(track_var);
3850                 goto out;
3851         }
3852
3853         if (data->handler == HANDLER_ONCHANGE)
3854                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3855         if (IS_ERR(track_var)) {
3856                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3857                 ret = PTR_ERR(track_var);
3858                 goto out;
3859         }
3860         data->track_data.track_var = track_var;
3861
3862         ret = action_create(hist_data, data);
3863  out:
3864         return ret;
3865 }
3866
3867 static int parse_action_params(struct trace_array *tr, char *params,
3868                                struct action_data *data)
3869 {
3870         char *param, *saved_param;
3871         bool first_param = true;
3872         int ret = 0;
3873
3874         while (params) {
3875                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3876                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3877                         goto out;
3878                 }
3879
3880                 param = strsep(&params, ",");
3881                 if (!param) {
3882                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3883                         ret = -EINVAL;
3884                         goto out;
3885                 }
3886
3887                 param = strstrip(param);
3888                 if (strlen(param) < 2) {
3889                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3890                         ret = -EINVAL;
3891                         goto out;
3892                 }
3893
3894                 saved_param = kstrdup(param, GFP_KERNEL);
3895                 if (!saved_param) {
3896                         ret = -ENOMEM;
3897                         goto out;
3898                 }
3899
3900                 if (first_param && data->use_trace_keyword) {
3901                         data->synth_event_name = saved_param;
3902                         first_param = false;
3903                         continue;
3904                 }
3905                 first_param = false;
3906
3907                 data->params[data->n_params++] = saved_param;
3908         }
3909  out:
3910         return ret;
3911 }
3912
3913 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3914                         enum handler_id handler)
3915 {
3916         char *action_name;
3917         int ret = 0;
3918
3919         strsep(&str, ".");
3920         if (!str) {
3921                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3922                 ret = -EINVAL;
3923                 goto out;
3924         }
3925
3926         action_name = strsep(&str, "(");
3927         if (!action_name || !str) {
3928                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3929                 ret = -EINVAL;
3930                 goto out;
3931         }
3932
3933         if (str_has_prefix(action_name, "save")) {
3934                 char *params = strsep(&str, ")");
3935
3936                 if (!params) {
3937                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3938                         ret = -EINVAL;
3939                         goto out;
3940                 }
3941
3942                 ret = parse_action_params(tr, params, data);
3943                 if (ret)
3944                         goto out;
3945
3946                 if (handler == HANDLER_ONMAX)
3947                         data->track_data.check_val = check_track_val_max;
3948                 else if (handler == HANDLER_ONCHANGE)
3949                         data->track_data.check_val = check_track_val_changed;
3950                 else {
3951                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3952                         ret = -EINVAL;
3953                         goto out;
3954                 }
3955
3956                 data->track_data.save_data = save_track_data_vars;
3957                 data->fn = ontrack_action;
3958                 data->action = ACTION_SAVE;
3959         } else if (str_has_prefix(action_name, "snapshot")) {
3960                 char *params = strsep(&str, ")");
3961
3962                 if (!str) {
3963                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3964                         ret = -EINVAL;
3965                         goto out;
3966                 }
3967
3968                 if (handler == HANDLER_ONMAX)
3969                         data->track_data.check_val = check_track_val_max;
3970                 else if (handler == HANDLER_ONCHANGE)
3971                         data->track_data.check_val = check_track_val_changed;
3972                 else {
3973                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3974                         ret = -EINVAL;
3975                         goto out;
3976                 }
3977
3978                 data->track_data.save_data = save_track_data_snapshot;
3979                 data->fn = ontrack_action;
3980                 data->action = ACTION_SNAPSHOT;
3981         } else {
3982                 char *params = strsep(&str, ")");
3983
3984                 if (str_has_prefix(action_name, "trace"))
3985                         data->use_trace_keyword = true;
3986
3987                 if (params) {
3988                         ret = parse_action_params(tr, params, data);
3989                         if (ret)
3990                                 goto out;
3991                 }
3992
3993                 if (handler == HANDLER_ONMAX)
3994                         data->track_data.check_val = check_track_val_max;
3995                 else if (handler == HANDLER_ONCHANGE)
3996                         data->track_data.check_val = check_track_val_changed;
3997
3998                 if (handler != HANDLER_ONMATCH) {
3999                         data->track_data.save_data = action_trace;
4000                         data->fn = ontrack_action;
4001                 } else
4002                         data->fn = action_trace;
4003
4004                 data->action = ACTION_TRACE;
4005         }
4006
4007         data->action_name = kstrdup(action_name, GFP_KERNEL);
4008         if (!data->action_name) {
4009                 ret = -ENOMEM;
4010                 goto out;
4011         }
4012
4013         data->handler = handler;
4014  out:
4015         return ret;
4016 }
4017
4018 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4019                                             char *str, enum handler_id handler)
4020 {
4021         struct action_data *data;
4022         int ret = -EINVAL;
4023         char *var_str;
4024
4025         data = kzalloc(sizeof(*data), GFP_KERNEL);
4026         if (!data)
4027                 return ERR_PTR(-ENOMEM);
4028
4029         var_str = strsep(&str, ")");
4030         if (!var_str || !str) {
4031                 ret = -EINVAL;
4032                 goto free;
4033         }
4034
4035         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4036         if (!data->track_data.var_str) {
4037                 ret = -ENOMEM;
4038                 goto free;
4039         }
4040
4041         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4042         if (ret)
4043                 goto free;
4044  out:
4045         return data;
4046  free:
4047         track_data_destroy(hist_data, data);
4048         data = ERR_PTR(ret);
4049         goto out;
4050 }
4051
4052 static void onmatch_destroy(struct action_data *data)
4053 {
4054         kfree(data->match_data.event);
4055         kfree(data->match_data.event_system);
4056
4057         action_data_destroy(data);
4058 }
4059
4060 static void destroy_field_var(struct field_var *field_var)
4061 {
4062         if (!field_var)
4063                 return;
4064
4065         destroy_hist_field(field_var->var, 0);
4066         destroy_hist_field(field_var->val, 0);
4067
4068         kfree(field_var);
4069 }
4070
4071 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4072 {
4073         unsigned int i;
4074
4075         for (i = 0; i < hist_data->n_field_vars; i++)
4076                 destroy_field_var(hist_data->field_vars[i]);
4077 }
4078
4079 static void save_field_var(struct hist_trigger_data *hist_data,
4080                            struct field_var *field_var)
4081 {
4082         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4083
4084         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4085                 hist_data->n_field_var_str++;
4086 }
4087
4088
4089 static int check_synth_field(struct synth_event *event,
4090                              struct hist_field *hist_field,
4091                              unsigned int field_pos)
4092 {
4093         struct synth_field *field;
4094
4095         if (field_pos >= event->n_fields)
4096                 return -EINVAL;
4097
4098         field = event->fields[field_pos];
4099
4100         if (strcmp(field->type, hist_field->type) != 0)
4101                 return -EINVAL;
4102
4103         return 0;
4104 }
4105
4106 static struct hist_field *
4107 trace_action_find_var(struct hist_trigger_data *hist_data,
4108                       struct action_data *data,
4109                       char *system, char *event, char *var)
4110 {
4111         struct trace_array *tr = hist_data->event_file->tr;
4112         struct hist_field *hist_field;
4113
4114         var++; /* skip '$' */
4115
4116         hist_field = find_target_event_var(hist_data, system, event, var);
4117         if (!hist_field) {
4118                 if (!system && data->handler == HANDLER_ONMATCH) {
4119                         system = data->match_data.event_system;
4120                         event = data->match_data.event;
4121                 }
4122
4123                 hist_field = find_event_var(hist_data, system, event, var);
4124         }
4125
4126         if (!hist_field)
4127                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4128
4129         return hist_field;
4130 }
4131
4132 static struct hist_field *
4133 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4134                               struct action_data *data, char *system,
4135                               char *event, char *var)
4136 {
4137         struct hist_field *hist_field = NULL;
4138         struct field_var *field_var;
4139
4140         /*
4141          * First try to create a field var on the target event (the
4142          * currently being defined).  This will create a variable for
4143          * unqualified fields on the target event, or if qualified,
4144          * target fields that have qualified names matching the target.
4145          */
4146         field_var = create_target_field_var(hist_data, system, event, var);
4147
4148         if (field_var && !IS_ERR(field_var)) {
4149                 save_field_var(hist_data, field_var);
4150                 hist_field = field_var->var;
4151         } else {
4152                 field_var = NULL;
4153                 /*
4154                  * If no explicit system.event is specfied, default to
4155                  * looking for fields on the onmatch(system.event.xxx)
4156                  * event.
4157                  */
4158                 if (!system && data->handler == HANDLER_ONMATCH) {
4159                         system = data->match_data.event_system;
4160                         event = data->match_data.event;
4161                 }
4162
4163                 /*
4164                  * At this point, we're looking at a field on another
4165                  * event.  Because we can't modify a hist trigger on
4166                  * another event to add a variable for a field, we need
4167                  * to create a new trigger on that event and create the
4168                  * variable at the same time.
4169                  */
4170                 hist_field = create_field_var_hist(hist_data, system, event, var);
4171                 if (IS_ERR(hist_field))
4172                         goto free;
4173         }
4174  out:
4175         return hist_field;
4176  free:
4177         destroy_field_var(field_var);
4178         hist_field = NULL;
4179         goto out;
4180 }
4181
4182 static int trace_action_create(struct hist_trigger_data *hist_data,
4183                                struct action_data *data)
4184 {
4185         struct trace_array *tr = hist_data->event_file->tr;
4186         char *event_name, *param, *system = NULL;
4187         struct hist_field *hist_field, *var_ref;
4188         unsigned int i, var_ref_idx;
4189         unsigned int field_pos = 0;
4190         struct synth_event *event;
4191         char *synth_event_name;
4192         int ret = 0;
4193
4194         lockdep_assert_held(&event_mutex);
4195
4196         if (data->use_trace_keyword)
4197                 synth_event_name = data->synth_event_name;
4198         else
4199                 synth_event_name = data->action_name;
4200
4201         event = find_synth_event(synth_event_name);
4202         if (!event) {
4203                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4204                 return -EINVAL;
4205         }
4206
4207         event->ref++;
4208
4209         var_ref_idx = hist_data->n_var_refs;
4210
4211         for (i = 0; i < data->n_params; i++) {
4212                 char *p;
4213
4214                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4215                 if (!param) {
4216                         ret = -ENOMEM;
4217                         goto err;
4218                 }
4219
4220                 system = strsep(&param, ".");
4221                 if (!param) {
4222                         param = (char *)system;
4223                         system = event_name = NULL;
4224                 } else {
4225                         event_name = strsep(&param, ".");
4226                         if (!param) {
4227                                 kfree(p);
4228                                 ret = -EINVAL;
4229                                 goto err;
4230                         }
4231                 }
4232
4233                 if (param[0] == '$')
4234                         hist_field = trace_action_find_var(hist_data, data,
4235                                                            system, event_name,
4236                                                            param);
4237                 else
4238                         hist_field = trace_action_create_field_var(hist_data,
4239                                                                    data,
4240                                                                    system,
4241                                                                    event_name,
4242                                                                    param);
4243
4244                 if (!hist_field) {
4245                         kfree(p);
4246                         ret = -EINVAL;
4247                         goto err;
4248                 }
4249
4250                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4251                         var_ref = create_var_ref(hist_data, hist_field,
4252                                                  system, event_name);
4253                         if (!var_ref) {
4254                                 kfree(p);
4255                                 ret = -ENOMEM;
4256                                 goto err;
4257                         }
4258
4259                         field_pos++;
4260                         kfree(p);
4261                         continue;
4262                 }
4263
4264                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4265                 kfree(p);
4266                 ret = -EINVAL;
4267                 goto err;
4268         }
4269
4270         if (field_pos != event->n_fields) {
4271                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4272                 ret = -EINVAL;
4273                 goto err;
4274         }
4275
4276         data->synth_event = event;
4277         data->var_ref_idx = var_ref_idx;
4278  out:
4279         return ret;
4280  err:
4281         event->ref--;
4282
4283         goto out;
4284 }
4285
4286 static int action_create(struct hist_trigger_data *hist_data,
4287                          struct action_data *data)
4288 {
4289         struct trace_event_file *file = hist_data->event_file;
4290         struct trace_array *tr = file->tr;
4291         struct track_data *track_data;
4292         struct field_var *field_var;
4293         unsigned int i;
4294         char *param;
4295         int ret = 0;
4296
4297         if (data->action == ACTION_TRACE)
4298                 return trace_action_create(hist_data, data);
4299
4300         if (data->action == ACTION_SNAPSHOT) {
4301                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4302                 if (IS_ERR(track_data)) {
4303                         ret = PTR_ERR(track_data);
4304                         goto out;
4305                 }
4306
4307                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4308                                                    cond_snapshot_update);
4309                 if (ret)
4310                         track_data_free(track_data);
4311
4312                 goto out;
4313         }
4314
4315         if (data->action == ACTION_SAVE) {
4316                 if (hist_data->n_save_vars) {
4317                         ret = -EEXIST;
4318                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4319                         goto out;
4320                 }
4321
4322                 for (i = 0; i < data->n_params; i++) {
4323                         param = kstrdup(data->params[i], GFP_KERNEL);
4324                         if (!param) {
4325                                 ret = -ENOMEM;
4326                                 goto out;
4327                         }
4328
4329                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4330                         if (IS_ERR(field_var)) {
4331                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4332                                          errpos(param));
4333                                 ret = PTR_ERR(field_var);
4334                                 kfree(param);
4335                                 goto out;
4336                         }
4337
4338                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4339                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4340                                 hist_data->n_save_var_str++;
4341                         kfree(param);
4342                 }
4343         }
4344  out:
4345         return ret;
4346 }
4347
4348 static int onmatch_create(struct hist_trigger_data *hist_data,
4349                           struct action_data *data)
4350 {
4351         return action_create(hist_data, data);
4352 }
4353
4354 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4355 {
4356         char *match_event, *match_event_system;
4357         struct action_data *data;
4358         int ret = -EINVAL;
4359
4360         data = kzalloc(sizeof(*data), GFP_KERNEL);
4361         if (!data)
4362                 return ERR_PTR(-ENOMEM);
4363
4364         match_event = strsep(&str, ")");
4365         if (!match_event || !str) {
4366                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4367                 goto free;
4368         }
4369
4370         match_event_system = strsep(&match_event, ".");
4371         if (!match_event) {
4372                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4373                 goto free;
4374         }
4375
4376         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4377                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4378                 goto free;
4379         }
4380
4381         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4382         if (!data->match_data.event) {
4383                 ret = -ENOMEM;
4384                 goto free;
4385         }
4386
4387         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4388         if (!data->match_data.event_system) {
4389                 ret = -ENOMEM;
4390                 goto free;
4391         }
4392
4393         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4394         if (ret)
4395                 goto free;
4396  out:
4397         return data;
4398  free:
4399         onmatch_destroy(data);
4400         data = ERR_PTR(ret);
4401         goto out;
4402 }
4403
4404 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4405 {
4406         hist_data->fields[HITCOUNT_IDX] =
4407                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4408         if (!hist_data->fields[HITCOUNT_IDX])
4409                 return -ENOMEM;
4410
4411         hist_data->n_vals++;
4412         hist_data->n_fields++;
4413
4414         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4415                 return -EINVAL;
4416
4417         return 0;
4418 }
4419
4420 static int __create_val_field(struct hist_trigger_data *hist_data,
4421                               unsigned int val_idx,
4422                               struct trace_event_file *file,
4423                               char *var_name, char *field_str,
4424                               unsigned long flags)
4425 {
4426         struct hist_field *hist_field;
4427         int ret = 0;
4428
4429         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4430         if (IS_ERR(hist_field)) {
4431                 ret = PTR_ERR(hist_field);
4432                 goto out;
4433         }
4434
4435         hist_data->fields[val_idx] = hist_field;
4436
4437         ++hist_data->n_vals;
4438         ++hist_data->n_fields;
4439
4440         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4441                 ret = -EINVAL;
4442  out:
4443         return ret;
4444 }
4445
4446 static int create_val_field(struct hist_trigger_data *hist_data,
4447                             unsigned int val_idx,
4448                             struct trace_event_file *file,
4449                             char *field_str)
4450 {
4451         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4452                 return -EINVAL;
4453
4454         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4455 }
4456
4457 static int create_var_field(struct hist_trigger_data *hist_data,
4458                             unsigned int val_idx,
4459                             struct trace_event_file *file,
4460                             char *var_name, char *expr_str)
4461 {
4462         struct trace_array *tr = hist_data->event_file->tr;
4463         unsigned long flags = 0;
4464
4465         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4466                 return -EINVAL;
4467
4468         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4469                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4470                 return -EINVAL;
4471         }
4472
4473         flags |= HIST_FIELD_FL_VAR;
4474         hist_data->n_vars++;
4475         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4476                 return -EINVAL;
4477
4478         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4479 }
4480
4481 static int create_val_fields(struct hist_trigger_data *hist_data,
4482                              struct trace_event_file *file)
4483 {
4484         char *fields_str, *field_str;
4485         unsigned int i, j = 1;
4486         int ret;
4487
4488         ret = create_hitcount_val(hist_data);
4489         if (ret)
4490                 goto out;
4491
4492         fields_str = hist_data->attrs->vals_str;
4493         if (!fields_str)
4494                 goto out;
4495
4496         strsep(&fields_str, "=");
4497         if (!fields_str)
4498                 goto out;
4499
4500         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4501                      j < TRACING_MAP_VALS_MAX; i++) {
4502                 field_str = strsep(&fields_str, ",");
4503                 if (!field_str)
4504                         break;
4505
4506                 if (strcmp(field_str, "hitcount") == 0)
4507                         continue;
4508
4509                 ret = create_val_field(hist_data, j++, file, field_str);
4510                 if (ret)
4511                         goto out;
4512         }
4513
4514         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4515                 ret = -EINVAL;
4516  out:
4517         return ret;
4518 }
4519
4520 static int create_key_field(struct hist_trigger_data *hist_data,
4521                             unsigned int key_idx,
4522                             unsigned int key_offset,
4523                             struct trace_event_file *file,
4524                             char *field_str)
4525 {
4526         struct trace_array *tr = hist_data->event_file->tr;
4527         struct hist_field *hist_field = NULL;
4528         unsigned long flags = 0;
4529         unsigned int key_size;
4530         int ret = 0;
4531
4532         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4533                 return -EINVAL;
4534
4535         flags |= HIST_FIELD_FL_KEY;
4536
4537         if (strcmp(field_str, "stacktrace") == 0) {
4538                 flags |= HIST_FIELD_FL_STACKTRACE;
4539                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4540                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4541         } else {
4542                 hist_field = parse_expr(hist_data, file, field_str, flags,
4543                                         NULL, 0);
4544                 if (IS_ERR(hist_field)) {
4545                         ret = PTR_ERR(hist_field);
4546                         goto out;
4547                 }
4548
4549                 if (field_has_hist_vars(hist_field, 0)) {
4550                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4551                         destroy_hist_field(hist_field, 0);
4552                         ret = -EINVAL;
4553                         goto out;
4554                 }
4555
4556                 key_size = hist_field->size;
4557         }
4558
4559         hist_data->fields[key_idx] = hist_field;
4560
4561         key_size = ALIGN(key_size, sizeof(u64));
4562         hist_data->fields[key_idx]->size = key_size;
4563         hist_data->fields[key_idx]->offset = key_offset;
4564
4565         hist_data->key_size += key_size;
4566
4567         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4568                 ret = -EINVAL;
4569                 goto out;
4570         }
4571
4572         hist_data->n_keys++;
4573         hist_data->n_fields++;
4574
4575         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4576                 return -EINVAL;
4577
4578         ret = key_size;
4579  out:
4580         return ret;
4581 }
4582
4583 static int create_key_fields(struct hist_trigger_data *hist_data,
4584                              struct trace_event_file *file)
4585 {
4586         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4587         char *fields_str, *field_str;
4588         int ret = -EINVAL;
4589
4590         fields_str = hist_data->attrs->keys_str;
4591         if (!fields_str)
4592                 goto out;
4593
4594         strsep(&fields_str, "=");
4595         if (!fields_str)
4596                 goto out;
4597
4598         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4599                 field_str = strsep(&fields_str, ",");
4600                 if (!field_str)
4601                         break;
4602                 ret = create_key_field(hist_data, i, key_offset,
4603                                        file, field_str);
4604                 if (ret < 0)
4605                         goto out;
4606                 key_offset += ret;
4607         }
4608         if (fields_str) {
4609                 ret = -EINVAL;
4610                 goto out;
4611         }
4612         ret = 0;
4613  out:
4614         return ret;
4615 }
4616
4617 static int create_var_fields(struct hist_trigger_data *hist_data,
4618                              struct trace_event_file *file)
4619 {
4620         unsigned int i, j = hist_data->n_vals;
4621         int ret = 0;
4622
4623         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4624
4625         for (i = 0; i < n_vars; i++) {
4626                 char *var_name = hist_data->attrs->var_defs.name[i];
4627                 char *expr = hist_data->attrs->var_defs.expr[i];
4628
4629                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4630                 if (ret)
4631                         goto out;
4632         }
4633  out:
4634         return ret;
4635 }
4636
4637 static void free_var_defs(struct hist_trigger_data *hist_data)
4638 {
4639         unsigned int i;
4640
4641         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4642                 kfree(hist_data->attrs->var_defs.name[i]);
4643                 kfree(hist_data->attrs->var_defs.expr[i]);
4644         }
4645
4646         hist_data->attrs->var_defs.n_vars = 0;
4647 }
4648
4649 static int parse_var_defs(struct hist_trigger_data *hist_data)
4650 {
4651         struct trace_array *tr = hist_data->event_file->tr;
4652         char *s, *str, *var_name, *field_str;
4653         unsigned int i, j, n_vars = 0;
4654         int ret = 0;
4655
4656         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4657                 str = hist_data->attrs->assignment_str[i];
4658                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4659                         field_str = strsep(&str, ",");
4660                         if (!field_str)
4661                                 break;
4662
4663                         var_name = strsep(&field_str, "=");
4664                         if (!var_name || !field_str) {
4665                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4666                                          errpos(var_name));
4667                                 ret = -EINVAL;
4668                                 goto free;
4669                         }
4670
4671                         if (n_vars == TRACING_MAP_VARS_MAX) {
4672                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4673                                 ret = -EINVAL;
4674                                 goto free;
4675                         }
4676
4677                         s = kstrdup(var_name, GFP_KERNEL);
4678                         if (!s) {
4679                                 ret = -ENOMEM;
4680                                 goto free;
4681                         }
4682                         hist_data->attrs->var_defs.name[n_vars] = s;
4683
4684                         s = kstrdup(field_str, GFP_KERNEL);
4685                         if (!s) {
4686                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4687                                 ret = -ENOMEM;
4688                                 goto free;
4689                         }
4690                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4691
4692                         hist_data->attrs->var_defs.n_vars = n_vars;
4693                 }
4694         }
4695
4696         return ret;
4697  free:
4698         free_var_defs(hist_data);
4699
4700         return ret;
4701 }
4702
4703 static int create_hist_fields(struct hist_trigger_data *hist_data,
4704                               struct trace_event_file *file)
4705 {
4706         int ret;
4707
4708         ret = parse_var_defs(hist_data);
4709         if (ret)
4710                 goto out;
4711
4712         ret = create_val_fields(hist_data, file);
4713         if (ret)
4714                 goto out;
4715
4716         ret = create_var_fields(hist_data, file);
4717         if (ret)
4718                 goto out;
4719
4720         ret = create_key_fields(hist_data, file);
4721         if (ret)
4722                 goto out;
4723  out:
4724         free_var_defs(hist_data);
4725
4726         return ret;
4727 }
4728
4729 static int is_descending(const char *str)
4730 {
4731         if (!str)
4732                 return 0;
4733
4734         if (strcmp(str, "descending") == 0)
4735                 return 1;
4736
4737         if (strcmp(str, "ascending") == 0)
4738                 return 0;
4739
4740         return -EINVAL;
4741 }
4742
4743 static int create_sort_keys(struct hist_trigger_data *hist_data)
4744 {
4745         char *fields_str = hist_data->attrs->sort_key_str;
4746         struct tracing_map_sort_key *sort_key;
4747         int descending, ret = 0;
4748         unsigned int i, j, k;
4749
4750         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4751
4752         if (!fields_str)
4753                 goto out;
4754
4755         strsep(&fields_str, "=");
4756         if (!fields_str) {
4757                 ret = -EINVAL;
4758                 goto out;
4759         }
4760
4761         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4762                 struct hist_field *hist_field;
4763                 char *field_str, *field_name;
4764                 const char *test_name;
4765
4766                 sort_key = &hist_data->sort_keys[i];
4767
4768                 field_str = strsep(&fields_str, ",");
4769                 if (!field_str) {
4770                         if (i == 0)
4771                                 ret = -EINVAL;
4772                         break;
4773                 }
4774
4775                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4776                         ret = -EINVAL;
4777                         break;
4778                 }
4779
4780                 field_name = strsep(&field_str, ".");
4781                 if (!field_name) {
4782                         ret = -EINVAL;
4783                         break;
4784                 }
4785
4786                 if (strcmp(field_name, "hitcount") == 0) {
4787                         descending = is_descending(field_str);
4788                         if (descending < 0) {
4789                                 ret = descending;
4790                                 break;
4791                         }
4792                         sort_key->descending = descending;
4793                         continue;
4794                 }
4795
4796                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4797                         unsigned int idx;
4798
4799                         hist_field = hist_data->fields[j];
4800                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4801                                 continue;
4802
4803                         idx = k++;
4804
4805                         test_name = hist_field_name(hist_field, 0);
4806
4807                         if (strcmp(field_name, test_name) == 0) {
4808                                 sort_key->field_idx = idx;
4809                                 descending = is_descending(field_str);
4810                                 if (descending < 0) {
4811                                         ret = descending;
4812                                         goto out;
4813                                 }
4814                                 sort_key->descending = descending;
4815                                 break;
4816                         }
4817                 }
4818                 if (j == hist_data->n_fields) {
4819                         ret = -EINVAL;
4820                         break;
4821                 }
4822         }
4823
4824         hist_data->n_sort_keys = i;
4825  out:
4826         return ret;
4827 }
4828
4829 static void destroy_actions(struct hist_trigger_data *hist_data)
4830 {
4831         unsigned int i;
4832
4833         for (i = 0; i < hist_data->n_actions; i++) {
4834                 struct action_data *data = hist_data->actions[i];
4835
4836                 if (data->handler == HANDLER_ONMATCH)
4837                         onmatch_destroy(data);
4838                 else if (data->handler == HANDLER_ONMAX ||
4839                          data->handler == HANDLER_ONCHANGE)
4840                         track_data_destroy(hist_data, data);
4841                 else
4842                         kfree(data);
4843         }
4844 }
4845
4846 static int parse_actions(struct hist_trigger_data *hist_data)
4847 {
4848         struct trace_array *tr = hist_data->event_file->tr;
4849         struct action_data *data;
4850         unsigned int i;
4851         int ret = 0;
4852         char *str;
4853         int len;
4854
4855         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4856                 str = hist_data->attrs->action_str[i];
4857
4858                 if ((len = str_has_prefix(str, "onmatch("))) {
4859                         char *action_str = str + len;
4860
4861                         data = onmatch_parse(tr, action_str);
4862                         if (IS_ERR(data)) {
4863                                 ret = PTR_ERR(data);
4864                                 break;
4865                         }
4866                 } else if ((len = str_has_prefix(str, "onmax("))) {
4867                         char *action_str = str + len;
4868
4869                         data = track_data_parse(hist_data, action_str,
4870                                                 HANDLER_ONMAX);
4871                         if (IS_ERR(data)) {
4872                                 ret = PTR_ERR(data);
4873                                 break;
4874                         }
4875                 } else if ((len = str_has_prefix(str, "onchange("))) {
4876                         char *action_str = str + len;
4877
4878                         data = track_data_parse(hist_data, action_str,
4879                                                 HANDLER_ONCHANGE);
4880                         if (IS_ERR(data)) {
4881                                 ret = PTR_ERR(data);
4882                                 break;
4883                         }
4884                 } else {
4885                         ret = -EINVAL;
4886                         break;
4887                 }
4888
4889                 hist_data->actions[hist_data->n_actions++] = data;
4890         }
4891
4892         return ret;
4893 }
4894
4895 static int create_actions(struct hist_trigger_data *hist_data)
4896 {
4897         struct action_data *data;
4898         unsigned int i;
4899         int ret = 0;
4900
4901         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4902                 data = hist_data->actions[i];
4903
4904                 if (data->handler == HANDLER_ONMATCH) {
4905                         ret = onmatch_create(hist_data, data);
4906                         if (ret)
4907                                 break;
4908                 } else if (data->handler == HANDLER_ONMAX ||
4909                            data->handler == HANDLER_ONCHANGE) {
4910                         ret = track_data_create(hist_data, data);
4911                         if (ret)
4912                                 break;
4913                 } else {
4914                         ret = -EINVAL;
4915                         break;
4916                 }
4917         }
4918
4919         return ret;
4920 }
4921
4922 static void print_actions(struct seq_file *m,
4923                           struct hist_trigger_data *hist_data,
4924                           struct tracing_map_elt *elt)
4925 {
4926         unsigned int i;
4927
4928         for (i = 0; i < hist_data->n_actions; i++) {
4929                 struct action_data *data = hist_data->actions[i];
4930
4931                 if (data->action == ACTION_SNAPSHOT)
4932                         continue;
4933
4934                 if (data->handler == HANDLER_ONMAX ||
4935                     data->handler == HANDLER_ONCHANGE)
4936                         track_data_print(m, hist_data, elt, data);
4937         }
4938 }
4939
4940 static void print_action_spec(struct seq_file *m,
4941                               struct hist_trigger_data *hist_data,
4942                               struct action_data *data)
4943 {
4944         unsigned int i;
4945
4946         if (data->action == ACTION_SAVE) {
4947                 for (i = 0; i < hist_data->n_save_vars; i++) {
4948                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4949                         if (i < hist_data->n_save_vars - 1)
4950                                 seq_puts(m, ",");
4951                 }
4952         } else if (data->action == ACTION_TRACE) {
4953                 if (data->use_trace_keyword)
4954                         seq_printf(m, "%s", data->synth_event_name);
4955                 for (i = 0; i < data->n_params; i++) {
4956                         if (i || data->use_trace_keyword)
4957                                 seq_puts(m, ",");
4958                         seq_printf(m, "%s", data->params[i]);
4959                 }
4960         }
4961 }
4962
4963 static void print_track_data_spec(struct seq_file *m,
4964                                   struct hist_trigger_data *hist_data,
4965                                   struct action_data *data)
4966 {
4967         if (data->handler == HANDLER_ONMAX)
4968                 seq_puts(m, ":onmax(");
4969         else if (data->handler == HANDLER_ONCHANGE)
4970                 seq_puts(m, ":onchange(");
4971         seq_printf(m, "%s", data->track_data.var_str);
4972         seq_printf(m, ").%s(", data->action_name);
4973
4974         print_action_spec(m, hist_data, data);
4975
4976         seq_puts(m, ")");
4977 }
4978
4979 static void print_onmatch_spec(struct seq_file *m,
4980                                struct hist_trigger_data *hist_data,
4981                                struct action_data *data)
4982 {
4983         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4984                    data->match_data.event);
4985
4986         seq_printf(m, "%s(", data->action_name);
4987
4988         print_action_spec(m, hist_data, data);
4989
4990         seq_puts(m, ")");
4991 }
4992
4993 static bool actions_match(struct hist_trigger_data *hist_data,
4994                           struct hist_trigger_data *hist_data_test)
4995 {
4996         unsigned int i, j;
4997
4998         if (hist_data->n_actions != hist_data_test->n_actions)
4999                 return false;
5000
5001         for (i = 0; i < hist_data->n_actions; i++) {
5002                 struct action_data *data = hist_data->actions[i];
5003                 struct action_data *data_test = hist_data_test->actions[i];
5004                 char *action_name, *action_name_test;
5005
5006                 if (data->handler != data_test->handler)
5007                         return false;
5008                 if (data->action != data_test->action)
5009                         return false;
5010
5011                 if (data->n_params != data_test->n_params)
5012                         return false;
5013
5014                 for (j = 0; j < data->n_params; j++) {
5015                         if (strcmp(data->params[j], data_test->params[j]) != 0)
5016                                 return false;
5017                 }
5018
5019                 if (data->use_trace_keyword)
5020                         action_name = data->synth_event_name;
5021                 else
5022                         action_name = data->action_name;
5023
5024                 if (data_test->use_trace_keyword)
5025                         action_name_test = data_test->synth_event_name;
5026                 else
5027                         action_name_test = data_test->action_name;
5028
5029                 if (strcmp(action_name, action_name_test) != 0)
5030                         return false;
5031
5032                 if (data->handler == HANDLER_ONMATCH) {
5033                         if (strcmp(data->match_data.event_system,
5034                                    data_test->match_data.event_system) != 0)
5035                                 return false;
5036                         if (strcmp(data->match_data.event,
5037                                    data_test->match_data.event) != 0)
5038                                 return false;
5039                 } else if (data->handler == HANDLER_ONMAX ||
5040                            data->handler == HANDLER_ONCHANGE) {
5041                         if (strcmp(data->track_data.var_str,
5042                                    data_test->track_data.var_str) != 0)
5043                                 return false;
5044                 }
5045         }
5046
5047         return true;
5048 }
5049
5050
5051 static void print_actions_spec(struct seq_file *m,
5052                                struct hist_trigger_data *hist_data)
5053 {
5054         unsigned int i;
5055
5056         for (i = 0; i < hist_data->n_actions; i++) {
5057                 struct action_data *data = hist_data->actions[i];
5058
5059                 if (data->handler == HANDLER_ONMATCH)
5060                         print_onmatch_spec(m, hist_data, data);
5061                 else if (data->handler == HANDLER_ONMAX ||
5062                          data->handler == HANDLER_ONCHANGE)
5063                         print_track_data_spec(m, hist_data, data);
5064         }
5065 }
5066
5067 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5068 {
5069         unsigned int i;
5070
5071         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5072                 kfree(hist_data->field_var_hists[i]->cmd);
5073                 kfree(hist_data->field_var_hists[i]);
5074         }
5075 }
5076
5077 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5078 {
5079         if (!hist_data)
5080                 return;
5081
5082         destroy_hist_trigger_attrs(hist_data->attrs);
5083         destroy_hist_fields(hist_data);
5084         tracing_map_destroy(hist_data->map);
5085
5086         destroy_actions(hist_data);
5087         destroy_field_vars(hist_data);
5088         destroy_field_var_hists(hist_data);
5089
5090         kfree(hist_data);
5091 }
5092
5093 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5094 {
5095         struct tracing_map *map = hist_data->map;
5096         struct ftrace_event_field *field;
5097         struct hist_field *hist_field;
5098         int i, idx = 0;
5099
5100         for_each_hist_field(i, hist_data) {
5101                 hist_field = hist_data->fields[i];
5102                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5103                         tracing_map_cmp_fn_t cmp_fn;
5104
5105                         field = hist_field->field;
5106
5107                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5108                                 cmp_fn = tracing_map_cmp_none;
5109                         else if (!field)
5110                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5111                                                              hist_field->is_signed);
5112                         else if (is_string_field(field))
5113                                 cmp_fn = tracing_map_cmp_string;
5114                         else
5115                                 cmp_fn = tracing_map_cmp_num(field->size,
5116                                                              field->is_signed);
5117                         idx = tracing_map_add_key_field(map,
5118                                                         hist_field->offset,
5119                                                         cmp_fn);
5120                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5121                         idx = tracing_map_add_sum_field(map);
5122
5123                 if (idx < 0)
5124                         return idx;
5125
5126                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5127                         idx = tracing_map_add_var(map);
5128                         if (idx < 0)
5129                                 return idx;
5130                         hist_field->var.idx = idx;
5131                         hist_field->var.hist_data = hist_data;
5132                 }
5133         }
5134
5135         return 0;
5136 }
5137
5138 static struct hist_trigger_data *
5139 create_hist_data(unsigned int map_bits,
5140                  struct hist_trigger_attrs *attrs,
5141                  struct trace_event_file *file,
5142                  bool remove)
5143 {
5144         const struct tracing_map_ops *map_ops = NULL;
5145         struct hist_trigger_data *hist_data;
5146         int ret = 0;
5147
5148         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5149         if (!hist_data)
5150                 return ERR_PTR(-ENOMEM);
5151
5152         hist_data->attrs = attrs;
5153         hist_data->remove = remove;
5154         hist_data->event_file = file;
5155
5156         ret = parse_actions(hist_data);
5157         if (ret)
5158                 goto free;
5159
5160         ret = create_hist_fields(hist_data, file);
5161         if (ret)
5162                 goto free;
5163
5164         ret = create_sort_keys(hist_data);
5165         if (ret)
5166                 goto free;
5167
5168         map_ops = &hist_trigger_elt_data_ops;
5169
5170         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5171                                             map_ops, hist_data);
5172         if (IS_ERR(hist_data->map)) {
5173                 ret = PTR_ERR(hist_data->map);
5174                 hist_data->map = NULL;
5175                 goto free;
5176         }
5177
5178         ret = create_tracing_map_fields(hist_data);
5179         if (ret)
5180                 goto free;
5181  out:
5182         return hist_data;
5183  free:
5184         hist_data->attrs = NULL;
5185
5186         destroy_hist_data(hist_data);
5187
5188         hist_data = ERR_PTR(ret);
5189
5190         goto out;
5191 }
5192
5193 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5194                                     struct tracing_map_elt *elt, void *rec,
5195                                     struct ring_buffer_event *rbe,
5196                                     u64 *var_ref_vals)
5197 {
5198         struct hist_elt_data *elt_data;
5199         struct hist_field *hist_field;
5200         unsigned int i, var_idx;
5201         u64 hist_val;
5202
5203         elt_data = elt->private_data;
5204         elt_data->var_ref_vals = var_ref_vals;
5205
5206         for_each_hist_val_field(i, hist_data) {
5207                 hist_field = hist_data->fields[i];
5208                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5209                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5210                         var_idx = hist_field->var.idx;
5211                         tracing_map_set_var(elt, var_idx, hist_val);
5212                         continue;
5213                 }
5214                 tracing_map_update_sum(elt, i, hist_val);
5215         }
5216
5217         for_each_hist_key_field(i, hist_data) {
5218                 hist_field = hist_data->fields[i];
5219                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5220                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5221                         var_idx = hist_field->var.idx;
5222                         tracing_map_set_var(elt, var_idx, hist_val);
5223                 }
5224         }
5225
5226         update_field_vars(hist_data, elt, rbe, rec);
5227 }
5228
5229 static inline void add_to_key(char *compound_key, void *key,
5230                               struct hist_field *key_field, void *rec)
5231 {
5232         size_t size = key_field->size;
5233
5234         if (key_field->flags & HIST_FIELD_FL_STRING) {
5235                 struct ftrace_event_field *field;
5236
5237                 field = key_field->field;
5238                 if (field->filter_type == FILTER_DYN_STRING)
5239                         size = *(u32 *)(rec + field->offset) >> 16;
5240                 else if (field->filter_type == FILTER_PTR_STRING)
5241                         size = strlen(key);
5242                 else if (field->filter_type == FILTER_STATIC_STRING)
5243                         size = field->size;
5244
5245                 /* ensure NULL-termination */
5246                 if (size > key_field->size - 1)
5247                         size = key_field->size - 1;
5248
5249                 strncpy(compound_key + key_field->offset, (char *)key, size);
5250         } else
5251                 memcpy(compound_key + key_field->offset, key, size);
5252 }
5253
5254 static void
5255 hist_trigger_actions(struct hist_trigger_data *hist_data,
5256                      struct tracing_map_elt *elt, void *rec,
5257                      struct ring_buffer_event *rbe, void *key,
5258                      u64 *var_ref_vals)
5259 {
5260         struct action_data *data;
5261         unsigned int i;
5262
5263         for (i = 0; i < hist_data->n_actions; i++) {
5264                 data = hist_data->actions[i];
5265                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5266         }
5267 }
5268
5269 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5270                                struct ring_buffer_event *rbe)
5271 {
5272         struct hist_trigger_data *hist_data = data->private_data;
5273         bool use_compound_key = (hist_data->n_keys > 1);
5274         unsigned long entries[HIST_STACKTRACE_DEPTH];
5275         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5276         char compound_key[HIST_KEY_SIZE_MAX];
5277         struct tracing_map_elt *elt = NULL;
5278         struct hist_field *key_field;
5279         u64 field_contents;
5280         void *key = NULL;
5281         unsigned int i;
5282
5283         memset(compound_key, 0, hist_data->key_size);
5284
5285         for_each_hist_key_field(i, hist_data) {
5286                 key_field = hist_data->fields[i];
5287
5288                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5289                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5290                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5291                                          HIST_STACKTRACE_SKIP);
5292                         key = entries;
5293                 } else {
5294                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5295                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5296                                 key = (void *)(unsigned long)field_contents;
5297                                 use_compound_key = true;
5298                         } else
5299                                 key = (void *)&field_contents;
5300                 }
5301
5302                 if (use_compound_key)
5303                         add_to_key(compound_key, key, key_field, rec);
5304         }
5305
5306         if (use_compound_key)
5307                 key = compound_key;
5308
5309         if (hist_data->n_var_refs &&
5310             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5311                 return;
5312
5313         elt = tracing_map_insert(hist_data->map, key);
5314         if (!elt)
5315                 return;
5316
5317         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5318
5319         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5320                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5321 }
5322
5323 static void hist_trigger_stacktrace_print(struct seq_file *m,
5324                                           unsigned long *stacktrace_entries,
5325                                           unsigned int max_entries)
5326 {
5327         char str[KSYM_SYMBOL_LEN];
5328         unsigned int spaces = 8;
5329         unsigned int i;
5330
5331         for (i = 0; i < max_entries; i++) {
5332                 if (!stacktrace_entries[i])
5333                         return;
5334
5335                 seq_printf(m, "%*c", 1 + spaces, ' ');
5336                 sprint_symbol(str, stacktrace_entries[i]);
5337                 seq_printf(m, "%s\n", str);
5338         }
5339 }
5340
5341 static void hist_trigger_print_key(struct seq_file *m,
5342                                    struct hist_trigger_data *hist_data,
5343                                    void *key,
5344                                    struct tracing_map_elt *elt)
5345 {
5346         struct hist_field *key_field;
5347         char str[KSYM_SYMBOL_LEN];
5348         bool multiline = false;
5349         const char *field_name;
5350         unsigned int i;
5351         u64 uval;
5352
5353         seq_puts(m, "{ ");
5354
5355         for_each_hist_key_field(i, hist_data) {
5356                 key_field = hist_data->fields[i];
5357
5358                 if (i > hist_data->n_vals)
5359                         seq_puts(m, ", ");
5360
5361                 field_name = hist_field_name(key_field, 0);
5362
5363                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5364                         uval = *(u64 *)(key + key_field->offset);
5365                         seq_printf(m, "%s: %llx", field_name, uval);
5366                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5367                         uval = *(u64 *)(key + key_field->offset);
5368                         sprint_symbol_no_offset(str, uval);
5369                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5370                                    uval, str);
5371                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5372                         uval = *(u64 *)(key + key_field->offset);
5373                         sprint_symbol(str, uval);
5374                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5375                                    uval, str);
5376                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5377                         struct hist_elt_data *elt_data = elt->private_data;
5378                         char *comm;
5379
5380                         if (WARN_ON_ONCE(!elt_data))
5381                                 return;
5382
5383                         comm = elt_data->comm;
5384
5385                         uval = *(u64 *)(key + key_field->offset);
5386                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5387                                    comm, uval);
5388                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5389                         const char *syscall_name;
5390
5391                         uval = *(u64 *)(key + key_field->offset);
5392                         syscall_name = get_syscall_name(uval);
5393                         if (!syscall_name)
5394                                 syscall_name = "unknown_syscall";
5395
5396                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5397                                    syscall_name, uval);
5398                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5399                         seq_puts(m, "stacktrace:\n");
5400                         hist_trigger_stacktrace_print(m,
5401                                                       key + key_field->offset,
5402                                                       HIST_STACKTRACE_DEPTH);
5403                         multiline = true;
5404                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5405                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5406                                    *(u64 *)(key + key_field->offset));
5407                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5408                         seq_printf(m, "%s: %-50s", field_name,
5409                                    (char *)(key + key_field->offset));
5410                 } else {
5411                         uval = *(u64 *)(key + key_field->offset);
5412                         seq_printf(m, "%s: %10llu", field_name, uval);
5413                 }
5414         }
5415
5416         if (!multiline)
5417                 seq_puts(m, " ");
5418
5419         seq_puts(m, "}");
5420 }
5421
5422 static void hist_trigger_entry_print(struct seq_file *m,
5423                                      struct hist_trigger_data *hist_data,
5424                                      void *key,
5425                                      struct tracing_map_elt *elt)
5426 {
5427         const char *field_name;
5428         unsigned int i;
5429
5430         hist_trigger_print_key(m, hist_data, key, elt);
5431
5432         seq_printf(m, " hitcount: %10llu",
5433                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5434
5435         for (i = 1; i < hist_data->n_vals; i++) {
5436                 field_name = hist_field_name(hist_data->fields[i], 0);
5437
5438                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5439                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5440                         continue;
5441
5442                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5443                         seq_printf(m, "  %s: %10llx", field_name,
5444                                    tracing_map_read_sum(elt, i));
5445                 } else {
5446                         seq_printf(m, "  %s: %10llu", field_name,
5447                                    tracing_map_read_sum(elt, i));
5448                 }
5449         }
5450
5451         print_actions(m, hist_data, elt);
5452
5453         seq_puts(m, "\n");
5454 }
5455
5456 static int print_entries(struct seq_file *m,
5457                          struct hist_trigger_data *hist_data)
5458 {
5459         struct tracing_map_sort_entry **sort_entries = NULL;
5460         struct tracing_map *map = hist_data->map;
5461         int i, n_entries;
5462
5463         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5464                                              hist_data->n_sort_keys,
5465                                              &sort_entries);
5466         if (n_entries < 0)
5467                 return n_entries;
5468
5469         for (i = 0; i < n_entries; i++)
5470                 hist_trigger_entry_print(m, hist_data,
5471                                          sort_entries[i]->key,
5472                                          sort_entries[i]->elt);
5473
5474         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5475
5476         return n_entries;
5477 }
5478
5479 static void hist_trigger_show(struct seq_file *m,
5480                               struct event_trigger_data *data, int n)
5481 {
5482         struct hist_trigger_data *hist_data;
5483         int n_entries;
5484
5485         if (n > 0)
5486                 seq_puts(m, "\n\n");
5487
5488         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5489         data->ops->print(m, data->ops, data);
5490         seq_puts(m, "#\n\n");
5491
5492         hist_data = data->private_data;
5493         n_entries = print_entries(m, hist_data);
5494         if (n_entries < 0)
5495                 n_entries = 0;
5496
5497         track_data_snapshot_print(m, hist_data);
5498
5499         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5500                    (u64)atomic64_read(&hist_data->map->hits),
5501                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5502 }
5503
5504 static int hist_show(struct seq_file *m, void *v)
5505 {
5506         struct event_trigger_data *data;
5507         struct trace_event_file *event_file;
5508         int n = 0, ret = 0;
5509
5510         mutex_lock(&event_mutex);
5511
5512         event_file = event_file_data(m->private);
5513         if (unlikely(!event_file)) {
5514                 ret = -ENODEV;
5515                 goto out_unlock;
5516         }
5517
5518         list_for_each_entry_rcu(data, &event_file->triggers, list) {
5519                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5520                         hist_trigger_show(m, data, n++);
5521         }
5522
5523  out_unlock:
5524         mutex_unlock(&event_mutex);
5525
5526         return ret;
5527 }
5528
5529 static int event_hist_open(struct inode *inode, struct file *file)
5530 {
5531         int ret;
5532
5533         ret = security_locked_down(LOCKDOWN_TRACEFS);
5534         if (ret)
5535                 return ret;
5536
5537         return single_open(file, hist_show, file);
5538 }
5539
5540 const struct file_operations event_hist_fops = {
5541         .open = event_hist_open,
5542         .read = seq_read,
5543         .llseek = seq_lseek,
5544         .release = single_release,
5545 };
5546
5547 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5548 {
5549         const char *field_name = hist_field_name(hist_field, 0);
5550
5551         if (hist_field->var.name)
5552                 seq_printf(m, "%s=", hist_field->var.name);
5553
5554         if (hist_field->flags & HIST_FIELD_FL_CPU)
5555                 seq_puts(m, "cpu");
5556         else if (field_name) {
5557                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5558                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5559                         seq_putc(m, '$');
5560                 seq_printf(m, "%s", field_name);
5561         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5562                 seq_puts(m, "common_timestamp");
5563
5564         if (hist_field->flags) {
5565                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5566                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5567                         const char *flags = get_hist_field_flags(hist_field);
5568
5569                         if (flags)
5570                                 seq_printf(m, ".%s", flags);
5571                 }
5572         }
5573 }
5574
5575 static int event_hist_trigger_print(struct seq_file *m,
5576                                     struct event_trigger_ops *ops,
5577                                     struct event_trigger_data *data)
5578 {
5579         struct hist_trigger_data *hist_data = data->private_data;
5580         struct hist_field *field;
5581         bool have_var = false;
5582         unsigned int i;
5583
5584         seq_puts(m, "hist:");
5585
5586         if (data->name)
5587                 seq_printf(m, "%s:", data->name);
5588
5589         seq_puts(m, "keys=");
5590
5591         for_each_hist_key_field(i, hist_data) {
5592                 field = hist_data->fields[i];
5593
5594                 if (i > hist_data->n_vals)
5595                         seq_puts(m, ",");
5596
5597                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5598                         seq_puts(m, "stacktrace");
5599                 else
5600                         hist_field_print(m, field);
5601         }
5602
5603         seq_puts(m, ":vals=");
5604
5605         for_each_hist_val_field(i, hist_data) {
5606                 field = hist_data->fields[i];
5607                 if (field->flags & HIST_FIELD_FL_VAR) {
5608                         have_var = true;
5609                         continue;
5610                 }
5611
5612                 if (i == HITCOUNT_IDX)
5613                         seq_puts(m, "hitcount");
5614                 else {
5615                         seq_puts(m, ",");
5616                         hist_field_print(m, field);
5617                 }
5618         }
5619
5620         if (have_var) {
5621                 unsigned int n = 0;
5622
5623                 seq_puts(m, ":");
5624
5625                 for_each_hist_val_field(i, hist_data) {
5626                         field = hist_data->fields[i];
5627
5628                         if (field->flags & HIST_FIELD_FL_VAR) {
5629                                 if (n++)
5630                                         seq_puts(m, ",");
5631                                 hist_field_print(m, field);
5632                         }
5633                 }
5634         }
5635
5636         seq_puts(m, ":sort=");
5637
5638         for (i = 0; i < hist_data->n_sort_keys; i++) {
5639                 struct tracing_map_sort_key *sort_key;
5640                 unsigned int idx, first_key_idx;
5641
5642                 /* skip VAR vals */
5643                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5644
5645                 sort_key = &hist_data->sort_keys[i];
5646                 idx = sort_key->field_idx;
5647
5648                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5649                         return -EINVAL;
5650
5651                 if (i > 0)
5652                         seq_puts(m, ",");
5653
5654                 if (idx == HITCOUNT_IDX)
5655                         seq_puts(m, "hitcount");
5656                 else {
5657                         if (idx >= first_key_idx)
5658                                 idx += hist_data->n_vars;
5659                         hist_field_print(m, hist_data->fields[idx]);
5660                 }
5661
5662                 if (sort_key->descending)
5663                         seq_puts(m, ".descending");
5664         }
5665         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5666         if (hist_data->enable_timestamps)
5667                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5668
5669         print_actions_spec(m, hist_data);
5670
5671         if (data->filter_str)
5672                 seq_printf(m, " if %s", data->filter_str);
5673
5674         if (data->paused)
5675                 seq_puts(m, " [paused]");
5676         else
5677                 seq_puts(m, " [active]");
5678
5679         seq_putc(m, '\n');
5680
5681         return 0;
5682 }
5683
5684 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5685                                    struct event_trigger_data *data)
5686 {
5687         struct hist_trigger_data *hist_data = data->private_data;
5688
5689         if (!data->ref && hist_data->attrs->name)
5690                 save_named_trigger(hist_data->attrs->name, data);
5691
5692         data->ref++;
5693
5694         return 0;
5695 }
5696
5697 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5698 {
5699         struct trace_event_file *file;
5700         unsigned int i;
5701         char *cmd;
5702         int ret;
5703
5704         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5705                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5706                 cmd = hist_data->field_var_hists[i]->cmd;
5707                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5708                                               "!hist", "hist", cmd);
5709         }
5710 }
5711
5712 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5713                                     struct event_trigger_data *data)
5714 {
5715         struct hist_trigger_data *hist_data = data->private_data;
5716
5717         if (WARN_ON_ONCE(data->ref <= 0))
5718                 return;
5719
5720         data->ref--;
5721         if (!data->ref) {
5722                 if (data->name)
5723                         del_named_trigger(data);
5724
5725                 trigger_data_free(data);
5726
5727                 remove_hist_vars(hist_data);
5728
5729                 unregister_field_var_hists(hist_data);
5730
5731                 destroy_hist_data(hist_data);
5732         }
5733 }
5734
5735 static struct event_trigger_ops event_hist_trigger_ops = {
5736         .func                   = event_hist_trigger,
5737         .print                  = event_hist_trigger_print,
5738         .init                   = event_hist_trigger_init,
5739         .free                   = event_hist_trigger_free,
5740 };
5741
5742 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5743                                          struct event_trigger_data *data)
5744 {
5745         data->ref++;
5746
5747         save_named_trigger(data->named_data->name, data);
5748
5749         event_hist_trigger_init(ops, data->named_data);
5750
5751         return 0;
5752 }
5753
5754 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5755                                           struct event_trigger_data *data)
5756 {
5757         if (WARN_ON_ONCE(data->ref <= 0))
5758                 return;
5759
5760         event_hist_trigger_free(ops, data->named_data);
5761
5762         data->ref--;
5763         if (!data->ref) {
5764                 del_named_trigger(data);
5765                 trigger_data_free(data);
5766         }
5767 }
5768
5769 static struct event_trigger_ops event_hist_trigger_named_ops = {
5770         .func                   = event_hist_trigger,
5771         .print                  = event_hist_trigger_print,
5772         .init                   = event_hist_trigger_named_init,
5773         .free                   = event_hist_trigger_named_free,
5774 };
5775
5776 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5777                                                             char *param)
5778 {
5779         return &event_hist_trigger_ops;
5780 }
5781
5782 static void hist_clear(struct event_trigger_data *data)
5783 {
5784         struct hist_trigger_data *hist_data = data->private_data;
5785
5786         if (data->name)
5787                 pause_named_trigger(data);
5788
5789         tracepoint_synchronize_unregister();
5790
5791         tracing_map_clear(hist_data->map);
5792
5793         if (data->name)
5794                 unpause_named_trigger(data);
5795 }
5796
5797 static bool compatible_field(struct ftrace_event_field *field,
5798                              struct ftrace_event_field *test_field)
5799 {
5800         if (field == test_field)
5801                 return true;
5802         if (field == NULL || test_field == NULL)
5803                 return false;
5804         if (strcmp(field->name, test_field->name) != 0)
5805                 return false;
5806         if (strcmp(field->type, test_field->type) != 0)
5807                 return false;
5808         if (field->size != test_field->size)
5809                 return false;
5810         if (field->is_signed != test_field->is_signed)
5811                 return false;
5812
5813         return true;
5814 }
5815
5816 static bool hist_trigger_match(struct event_trigger_data *data,
5817                                struct event_trigger_data *data_test,
5818                                struct event_trigger_data *named_data,
5819                                bool ignore_filter)
5820 {
5821         struct tracing_map_sort_key *sort_key, *sort_key_test;
5822         struct hist_trigger_data *hist_data, *hist_data_test;
5823         struct hist_field *key_field, *key_field_test;
5824         unsigned int i;
5825
5826         if (named_data && (named_data != data_test) &&
5827             (named_data != data_test->named_data))
5828                 return false;
5829
5830         if (!named_data && is_named_trigger(data_test))
5831                 return false;
5832
5833         hist_data = data->private_data;
5834         hist_data_test = data_test->private_data;
5835
5836         if (hist_data->n_vals != hist_data_test->n_vals ||
5837             hist_data->n_fields != hist_data_test->n_fields ||
5838             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5839                 return false;
5840
5841         if (!ignore_filter) {
5842                 if ((data->filter_str && !data_test->filter_str) ||
5843                    (!data->filter_str && data_test->filter_str))
5844                         return false;
5845         }
5846
5847         for_each_hist_field(i, hist_data) {
5848                 key_field = hist_data->fields[i];
5849                 key_field_test = hist_data_test->fields[i];
5850
5851                 if (key_field->flags != key_field_test->flags)
5852                         return false;
5853                 if (!compatible_field(key_field->field, key_field_test->field))
5854                         return false;
5855                 if (key_field->offset != key_field_test->offset)
5856                         return false;
5857                 if (key_field->size != key_field_test->size)
5858                         return false;
5859                 if (key_field->is_signed != key_field_test->is_signed)
5860                         return false;
5861                 if (!!key_field->var.name != !!key_field_test->var.name)
5862                         return false;
5863                 if (key_field->var.name &&
5864                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5865                         return false;
5866         }
5867
5868         for (i = 0; i < hist_data->n_sort_keys; i++) {
5869                 sort_key = &hist_data->sort_keys[i];
5870                 sort_key_test = &hist_data_test->sort_keys[i];
5871
5872                 if (sort_key->field_idx != sort_key_test->field_idx ||
5873                     sort_key->descending != sort_key_test->descending)
5874                         return false;
5875         }
5876
5877         if (!ignore_filter && data->filter_str &&
5878             (strcmp(data->filter_str, data_test->filter_str) != 0))
5879                 return false;
5880
5881         if (!actions_match(hist_data, hist_data_test))
5882                 return false;
5883
5884         return true;
5885 }
5886
5887 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5888                                  struct event_trigger_data *data,
5889                                  struct trace_event_file *file)
5890 {
5891         struct hist_trigger_data *hist_data = data->private_data;
5892         struct event_trigger_data *test, *named_data = NULL;
5893         struct trace_array *tr = file->tr;
5894         int ret = 0;
5895
5896         if (hist_data->attrs->name) {
5897                 named_data = find_named_trigger(hist_data->attrs->name);
5898                 if (named_data) {
5899                         if (!hist_trigger_match(data, named_data, named_data,
5900                                                 true)) {
5901                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5902                                 ret = -EINVAL;
5903                                 goto out;
5904                         }
5905                 }
5906         }
5907
5908         if (hist_data->attrs->name && !named_data)
5909                 goto new;
5910
5911         list_for_each_entry_rcu(test, &file->triggers, list) {
5912                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5913                         if (!hist_trigger_match(data, test, named_data, false))
5914                                 continue;
5915                         if (hist_data->attrs->pause)
5916                                 test->paused = true;
5917                         else if (hist_data->attrs->cont)
5918                                 test->paused = false;
5919                         else if (hist_data->attrs->clear)
5920                                 hist_clear(test);
5921                         else {
5922                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5923                                 ret = -EEXIST;
5924                         }
5925                         goto out;
5926                 }
5927         }
5928  new:
5929         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5930                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5931                 ret = -ENOENT;
5932                 goto out;
5933         }
5934
5935         if (hist_data->attrs->pause)
5936                 data->paused = true;
5937
5938         if (named_data) {
5939                 data->private_data = named_data->private_data;
5940                 set_named_trigger_data(data, named_data);
5941                 data->ops = &event_hist_trigger_named_ops;
5942         }
5943
5944         if (data->ops->init) {
5945                 ret = data->ops->init(data->ops, data);
5946                 if (ret < 0)
5947                         goto out;
5948         }
5949
5950         if (hist_data->enable_timestamps) {
5951                 char *clock = hist_data->attrs->clock;
5952
5953                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5954                 if (ret) {
5955                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5956                         goto out;
5957                 }
5958
5959                 tracing_set_time_stamp_abs(file->tr, true);
5960         }
5961
5962         if (named_data)
5963                 destroy_hist_data(hist_data);
5964
5965         ret++;
5966  out:
5967         return ret;
5968 }
5969
5970 static int hist_trigger_enable(struct event_trigger_data *data,
5971                                struct trace_event_file *file)
5972 {
5973         int ret = 0;
5974
5975         list_add_tail_rcu(&data->list, &file->triggers);
5976
5977         update_cond_flag(file);
5978
5979         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5980                 list_del_rcu(&data->list);
5981                 update_cond_flag(file);
5982                 ret--;
5983         }
5984
5985         return ret;
5986 }
5987
5988 static bool have_hist_trigger_match(struct event_trigger_data *data,
5989                                     struct trace_event_file *file)
5990 {
5991         struct hist_trigger_data *hist_data = data->private_data;
5992         struct event_trigger_data *test, *named_data = NULL;
5993         bool match = false;
5994
5995         if (hist_data->attrs->name)
5996                 named_data = find_named_trigger(hist_data->attrs->name);
5997
5998         list_for_each_entry_rcu(test, &file->triggers, list) {
5999                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6000                         if (hist_trigger_match(data, test, named_data, false)) {
6001                                 match = true;
6002                                 break;
6003                         }
6004                 }
6005         }
6006
6007         return match;
6008 }
6009
6010 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6011                                     struct trace_event_file *file)
6012 {
6013         struct hist_trigger_data *hist_data = data->private_data;
6014         struct event_trigger_data *test, *named_data = NULL;
6015
6016         if (hist_data->attrs->name)
6017                 named_data = find_named_trigger(hist_data->attrs->name);
6018
6019         list_for_each_entry_rcu(test, &file->triggers, list) {
6020                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6021                         if (!hist_trigger_match(data, test, named_data, false))
6022                                 continue;
6023                         hist_data = test->private_data;
6024                         if (check_var_refs(hist_data))
6025                                 return true;
6026                         break;
6027                 }
6028         }
6029
6030         return false;
6031 }
6032
6033 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6034                                     struct event_trigger_data *data,
6035                                     struct trace_event_file *file)
6036 {
6037         struct hist_trigger_data *hist_data = data->private_data;
6038         struct event_trigger_data *test, *named_data = NULL;
6039         bool unregistered = false;
6040
6041         if (hist_data->attrs->name)
6042                 named_data = find_named_trigger(hist_data->attrs->name);
6043
6044         list_for_each_entry_rcu(test, &file->triggers, list) {
6045                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6046                         if (!hist_trigger_match(data, test, named_data, false))
6047                                 continue;
6048                         unregistered = true;
6049                         list_del_rcu(&test->list);
6050                         trace_event_trigger_enable_disable(file, 0);
6051                         update_cond_flag(file);
6052                         break;
6053                 }
6054         }
6055
6056         if (unregistered && test->ops->free)
6057                 test->ops->free(test->ops, test);
6058
6059         if (hist_data->enable_timestamps) {
6060                 if (!hist_data->remove || unregistered)
6061                         tracing_set_time_stamp_abs(file->tr, false);
6062         }
6063 }
6064
6065 static bool hist_file_check_refs(struct trace_event_file *file)
6066 {
6067         struct hist_trigger_data *hist_data;
6068         struct event_trigger_data *test;
6069
6070         list_for_each_entry_rcu(test, &file->triggers, list) {
6071                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6072                         hist_data = test->private_data;
6073                         if (check_var_refs(hist_data))
6074                                 return true;
6075                 }
6076         }
6077
6078         return false;
6079 }
6080
6081 static void hist_unreg_all(struct trace_event_file *file)
6082 {
6083         struct event_trigger_data *test, *n;
6084         struct hist_trigger_data *hist_data;
6085         struct synth_event *se;
6086         const char *se_name;
6087
6088         lockdep_assert_held(&event_mutex);
6089
6090         if (hist_file_check_refs(file))
6091                 return;
6092
6093         list_for_each_entry_safe(test, n, &file->triggers, list) {
6094                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6095                         hist_data = test->private_data;
6096                         list_del_rcu(&test->list);
6097                         trace_event_trigger_enable_disable(file, 0);
6098
6099                         se_name = trace_event_name(file->event_call);
6100                         se = find_synth_event(se_name);
6101                         if (se)
6102                                 se->ref--;
6103
6104                         update_cond_flag(file);
6105                         if (hist_data->enable_timestamps)
6106                                 tracing_set_time_stamp_abs(file->tr, false);
6107                         if (test->ops->free)
6108                                 test->ops->free(test->ops, test);
6109                 }
6110         }
6111 }
6112
6113 static int event_hist_trigger_func(struct event_command *cmd_ops,
6114                                    struct trace_event_file *file,
6115                                    char *glob, char *cmd, char *param)
6116 {
6117         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6118         struct event_trigger_data *trigger_data;
6119         struct hist_trigger_attrs *attrs;
6120         struct event_trigger_ops *trigger_ops;
6121         struct hist_trigger_data *hist_data;
6122         struct synth_event *se;
6123         const char *se_name;
6124         bool remove = false;
6125         char *trigger, *p;
6126         int ret = 0;
6127
6128         lockdep_assert_held(&event_mutex);
6129
6130         if (glob && strlen(glob)) {
6131                 hist_err_clear();
6132                 last_cmd_set(file, param);
6133         }
6134
6135         if (!param)
6136                 return -EINVAL;
6137
6138         if (glob[0] == '!')
6139                 remove = true;
6140
6141         /*
6142          * separate the trigger from the filter (k:v [if filter])
6143          * allowing for whitespace in the trigger
6144          */
6145         p = trigger = param;
6146         do {
6147                 p = strstr(p, "if");
6148                 if (!p)
6149                         break;
6150                 if (p == param)
6151                         return -EINVAL;
6152                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6153                         p++;
6154                         continue;
6155                 }
6156                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6157                         return -EINVAL;
6158                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6159                         p++;
6160                         continue;
6161                 }
6162                 break;
6163         } while (p);
6164
6165         if (!p)
6166                 param = NULL;
6167         else {
6168                 *(p - 1) = '\0';
6169                 param = strstrip(p);
6170                 trigger = strstrip(trigger);
6171         }
6172
6173         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6174         if (IS_ERR(attrs))
6175                 return PTR_ERR(attrs);
6176
6177         if (attrs->map_bits)
6178                 hist_trigger_bits = attrs->map_bits;
6179
6180         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6181         if (IS_ERR(hist_data)) {
6182                 destroy_hist_trigger_attrs(attrs);
6183                 return PTR_ERR(hist_data);
6184         }
6185
6186         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6187
6188         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6189         if (!trigger_data) {
6190                 ret = -ENOMEM;
6191                 goto out_free;
6192         }
6193
6194         trigger_data->count = -1;
6195         trigger_data->ops = trigger_ops;
6196         trigger_data->cmd_ops = cmd_ops;
6197
6198         INIT_LIST_HEAD(&trigger_data->list);
6199         RCU_INIT_POINTER(trigger_data->filter, NULL);
6200
6201         trigger_data->private_data = hist_data;
6202
6203         /* if param is non-empty, it's supposed to be a filter */
6204         if (param && cmd_ops->set_filter) {
6205                 ret = cmd_ops->set_filter(param, trigger_data, file);
6206                 if (ret < 0)
6207                         goto out_free;
6208         }
6209
6210         if (remove) {
6211                 if (!have_hist_trigger_match(trigger_data, file))
6212                         goto out_free;
6213
6214                 if (hist_trigger_check_refs(trigger_data, file)) {
6215                         ret = -EBUSY;
6216                         goto out_free;
6217                 }
6218
6219                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6220                 se_name = trace_event_name(file->event_call);
6221                 se = find_synth_event(se_name);
6222                 if (se)
6223                         se->ref--;
6224                 ret = 0;
6225                 goto out_free;
6226         }
6227
6228         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6229         /*
6230          * The above returns on success the # of triggers registered,
6231          * but if it didn't register any it returns zero.  Consider no
6232          * triggers registered a failure too.
6233          */
6234         if (!ret) {
6235                 if (!(attrs->pause || attrs->cont || attrs->clear))
6236                         ret = -ENOENT;
6237                 goto out_free;
6238         } else if (ret < 0)
6239                 goto out_free;
6240
6241         if (get_named_trigger_data(trigger_data))
6242                 goto enable;
6243
6244         if (has_hist_vars(hist_data))
6245                 save_hist_vars(hist_data);
6246
6247         ret = create_actions(hist_data);
6248         if (ret)
6249                 goto out_unreg;
6250
6251         ret = tracing_map_init(hist_data->map);
6252         if (ret)
6253                 goto out_unreg;
6254 enable:
6255         ret = hist_trigger_enable(trigger_data, file);
6256         if (ret)
6257                 goto out_unreg;
6258
6259         se_name = trace_event_name(file->event_call);
6260         se = find_synth_event(se_name);
6261         if (se)
6262                 se->ref++;
6263         /* Just return zero, not the number of registered triggers */
6264         ret = 0;
6265  out:
6266         if (ret == 0)
6267                 hist_err_clear();
6268
6269         return ret;
6270  out_unreg:
6271         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6272  out_free:
6273         if (cmd_ops->set_filter)
6274                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6275
6276         remove_hist_vars(hist_data);
6277
6278         kfree(trigger_data);
6279
6280         destroy_hist_data(hist_data);
6281         goto out;
6282 }
6283
6284 static struct event_command trigger_hist_cmd = {
6285         .name                   = "hist",
6286         .trigger_type           = ETT_EVENT_HIST,
6287         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6288         .func                   = event_hist_trigger_func,
6289         .reg                    = hist_register_trigger,
6290         .unreg                  = hist_unregister_trigger,
6291         .unreg_all              = hist_unreg_all,
6292         .get_trigger_ops        = event_hist_get_trigger_ops,
6293         .set_filter             = set_trigger_filter,
6294 };
6295
6296 __init int register_trigger_hist_cmd(void)
6297 {
6298         int ret;
6299
6300         ret = register_event_command(&trigger_hist_cmd);
6301         WARN_ON(ret < 0);
6302
6303         return ret;
6304 }
6305
6306 static void
6307 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6308                     struct ring_buffer_event *event)
6309 {
6310         struct enable_trigger_data *enable_data = data->private_data;
6311         struct event_trigger_data *test;
6312
6313         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
6314                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6315                         if (enable_data->enable)
6316                                 test->paused = false;
6317                         else
6318                                 test->paused = true;
6319                 }
6320         }
6321 }
6322
6323 static void
6324 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6325                           struct ring_buffer_event *event)
6326 {
6327         if (!data->count)
6328                 return;
6329
6330         if (data->count != -1)
6331                 (data->count)--;
6332
6333         hist_enable_trigger(data, rec, event);
6334 }
6335
6336 static struct event_trigger_ops hist_enable_trigger_ops = {
6337         .func                   = hist_enable_trigger,
6338         .print                  = event_enable_trigger_print,
6339         .init                   = event_trigger_init,
6340         .free                   = event_enable_trigger_free,
6341 };
6342
6343 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6344         .func                   = hist_enable_count_trigger,
6345         .print                  = event_enable_trigger_print,
6346         .init                   = event_trigger_init,
6347         .free                   = event_enable_trigger_free,
6348 };
6349
6350 static struct event_trigger_ops hist_disable_trigger_ops = {
6351         .func                   = hist_enable_trigger,
6352         .print                  = event_enable_trigger_print,
6353         .init                   = event_trigger_init,
6354         .free                   = event_enable_trigger_free,
6355 };
6356
6357 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6358         .func                   = hist_enable_count_trigger,
6359         .print                  = event_enable_trigger_print,
6360         .init                   = event_trigger_init,
6361         .free                   = event_enable_trigger_free,
6362 };
6363
6364 static struct event_trigger_ops *
6365 hist_enable_get_trigger_ops(char *cmd, char *param)
6366 {
6367         struct event_trigger_ops *ops;
6368         bool enable;
6369
6370         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6371
6372         if (enable)
6373                 ops = param ? &hist_enable_count_trigger_ops :
6374                         &hist_enable_trigger_ops;
6375         else
6376                 ops = param ? &hist_disable_count_trigger_ops :
6377                         &hist_disable_trigger_ops;
6378
6379         return ops;
6380 }
6381
6382 static void hist_enable_unreg_all(struct trace_event_file *file)
6383 {
6384         struct event_trigger_data *test, *n;
6385
6386         list_for_each_entry_safe(test, n, &file->triggers, list) {
6387                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6388                         list_del_rcu(&test->list);
6389                         update_cond_flag(file);
6390                         trace_event_trigger_enable_disable(file, 0);
6391                         if (test->ops->free)
6392                                 test->ops->free(test->ops, test);
6393                 }
6394         }
6395 }
6396
6397 static struct event_command trigger_hist_enable_cmd = {
6398         .name                   = ENABLE_HIST_STR,
6399         .trigger_type           = ETT_HIST_ENABLE,
6400         .func                   = event_enable_trigger_func,
6401         .reg                    = event_enable_register_trigger,
6402         .unreg                  = event_enable_unregister_trigger,
6403         .unreg_all              = hist_enable_unreg_all,
6404         .get_trigger_ops        = hist_enable_get_trigger_ops,
6405         .set_filter             = set_trigger_filter,
6406 };
6407
6408 static struct event_command trigger_hist_disable_cmd = {
6409         .name                   = DISABLE_HIST_STR,
6410         .trigger_type           = ETT_HIST_ENABLE,
6411         .func                   = event_enable_trigger_func,
6412         .reg                    = event_enable_register_trigger,
6413         .unreg                  = event_enable_unregister_trigger,
6414         .unreg_all              = hist_enable_unreg_all,
6415         .get_trigger_ops        = hist_enable_get_trigger_ops,
6416         .set_filter             = set_trigger_filter,
6417 };
6418
6419 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6420 {
6421         unregister_event_command(&trigger_hist_enable_cmd);
6422         unregister_event_command(&trigger_hist_disable_cmd);
6423 }
6424
6425 __init int register_trigger_hist_enable_disable_cmds(void)
6426 {
6427         int ret;
6428
6429         ret = register_event_command(&trigger_hist_enable_cmd);
6430         if (WARN_ON(ret < 0))
6431                 return ret;
6432         ret = register_event_command(&trigger_hist_disable_cmd);
6433         if (WARN_ON(ret < 0))
6434                 unregister_trigger_hist_enable_disable_cmds();
6435
6436         return ret;
6437 }
6438
6439 static __init int trace_events_hist_init(void)
6440 {
6441         struct dentry *entry = NULL;
6442         struct dentry *d_tracer;
6443         int err = 0;
6444
6445         err = dyn_event_register(&synth_event_ops);
6446         if (err) {
6447                 pr_warn("Could not register synth_event_ops\n");
6448                 return err;
6449         }
6450
6451         d_tracer = tracing_init_dentry();
6452         if (IS_ERR(d_tracer)) {
6453                 err = PTR_ERR(d_tracer);
6454                 goto err;
6455         }
6456
6457         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6458                                     NULL, &synth_events_fops);
6459         if (!entry) {
6460                 err = -ENODEV;
6461                 goto err;
6462         }
6463
6464         return err;
6465  err:
6466         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6467
6468         return err;
6469 }
6470
6471 fs_initcall(trace_events_hist_init);