kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace.h"
  23 #include "trace_dynevent.h"
  24
  25 #define SYNTH_SYSTEM            "synthetic"
  26 #define SYNTH_FIELDS_MAX        32
  27
  28 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  29
  30 #define ERRORS                                                          \
  31         C(NONE,                 "No error"),                            \
  32         C(DUPLICATE_VAR,        "Variable already defined"),            \
  33         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  34         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  35         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  36         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  37         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  38         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  39         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  40         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  41         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  42         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  43         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  44         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  45         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  46         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  47         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  48         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  49         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  50         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  51         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  52         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  53         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  54         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  55         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  56         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  57         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  58         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  59         C(INVALID_PARAM,        "Invalid action param"),                \
  60         C(ACTION_NOT_FOUND,     "No action found"),                     \
  61         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  62         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  63         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  64         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  65         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  66         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  67         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  68         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  69         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  70         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  71         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  72         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  73         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  74         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  75
  76 #undef C
  77 #define C(a, b)         HIST_ERR_##a
  78
  79 enum { ERRORS };
  80
  81 #undef C
  82 #define C(a, b)         b
  83
  84 static const char *err_text[] = { ERRORS };
  85
  86 struct hist_field;
  87
  88 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  89                                 struct tracing_map_elt *elt,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96
  97 enum field_op_id {
  98         FIELD_OP_NONE,
  99         FIELD_OP_PLUS,
 100         FIELD_OP_MINUS,
 101         FIELD_OP_UNARY_MINUS,
 102 };
 103
 104 /*
 105  * A hist_var (histogram variable) contains variable information for
 106  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 107  * flag set.  A hist_var has a variable name e.g. ts0, and is
 108  * associated with a given histogram trigger, as specified by
 109  * hist_data.  The hist_var idx is the unique index assigned to the
 110  * variable by the hist trigger's tracing_map.  The idx is what is
 111  * used to set a variable's value and, by a variable reference, to
 112  * retrieve it.
 113  */
 114 struct hist_var {
 115         char                            *name;
 116         struct hist_trigger_data        *hist_data;
 117         unsigned int                    idx;
 118 };
 119
 120 struct hist_field {
 121         struct ftrace_event_field       *field;
 122         unsigned long                   flags;
 123         hist_field_fn_t                 fn;
 124         unsigned int                    ref;
 125         unsigned int                    size;
 126         unsigned int                    offset;
 127         unsigned int                    is_signed;
 128         const char                      *type;
 129         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 130         struct hist_trigger_data        *hist_data;
 131
 132         /*
 133          * Variable fields contain variable-specific info in var.
 134          */
 135         struct hist_var                 var;
 136         enum field_op_id                operator;
 137         char                            *system;
 138         char                            *event_name;
 139
 140         /*
 141          * The name field is used for EXPR and VAR_REF fields.  VAR
 142          * fields contain the variable name in var.name.
 143          */
 144         char                            *name;
 145
 146         /*
 147          * When a histogram trigger is hit, if it has any references
 148          * to variables, the values of those variables are collected
 149          * into a var_ref_vals array by resolve_var_refs().  The
 150          * current value of each variable is read from the tracing_map
 151          * using the hist field's hist_var.idx and entered into the
 152          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 153          */
 154         unsigned int                    var_ref_idx;
 155         bool                            read_once;
 156 };
 157
 158 static u64 hist_field_none(struct hist_field *field,
 159                            struct tracing_map_elt *elt,
 160                            struct ring_buffer_event *rbe,
 161                            void *event)
 162 {
 163         return 0;
 164 }
 165
 166 static u64 hist_field_counter(struct hist_field *field,
 167                               struct tracing_map_elt *elt,
 168                               struct ring_buffer_event *rbe,
 169                               void *event)
 170 {
 171         return 1;
 172 }
 173
 174 static u64 hist_field_string(struct hist_field *hist_field,
 175                              struct tracing_map_elt *elt,
 176                              struct ring_buffer_event *rbe,
 177                              void *event)
 178 {
 179         char *addr = (char *)(event + hist_field->field->offset);
 180
 181         return (u64)(unsigned long)addr;
 182 }
 183
 184 static u64 hist_field_dynstring(struct hist_field *hist_field,
 185                                 struct tracing_map_elt *elt,
 186                                 struct ring_buffer_event *rbe,
 187                                 void *event)
 188 {
 189         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 190         int str_loc = str_item & 0xffff;
 191         char *addr = (char *)(event + str_loc);
 192
 193         return (u64)(unsigned long)addr;
 194 }
 195
 196 static u64 hist_field_pstring(struct hist_field *hist_field,
 197                               struct tracing_map_elt *elt,
 198                               struct ring_buffer_event *rbe,
 199                               void *event)
 200 {
 201         char **addr = (char **)(event + hist_field->field->offset);
 202
 203         return (u64)(unsigned long)*addr;
 204 }
 205
 206 static u64 hist_field_log2(struct hist_field *hist_field,
 207                            struct tracing_map_elt *elt,
 208                            struct ring_buffer_event *rbe,
 209                            void *event)
 210 {
 211         struct hist_field *operand = hist_field->operands[0];
 212
 213         u64 val = operand->fn(operand, elt, rbe, event);
 214
 215         return (u64) ilog2(roundup_pow_of_two(val));
 216 }
 217
 218 static u64 hist_field_plus(struct hist_field *hist_field,
 219                            struct tracing_map_elt *elt,
 220                            struct ring_buffer_event *rbe,
 221                            void *event)
 222 {
 223         struct hist_field *operand1 = hist_field->operands[0];
 224         struct hist_field *operand2 = hist_field->operands[1];
 225
 226         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 227         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 228
 229         return val1 + val2;
 230 }
 231
 232 static u64 hist_field_minus(struct hist_field *hist_field,
 233                             struct tracing_map_elt *elt,
 234                             struct ring_buffer_event *rbe,
 235                             void *event)
 236 {
 237         struct hist_field *operand1 = hist_field->operands[0];
 238         struct hist_field *operand2 = hist_field->operands[1];
 239
 240         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 241         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 242
 243         return val1 - val2;
 244 }
 245
 246 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 247                                   struct tracing_map_elt *elt,
 248                                   struct ring_buffer_event *rbe,
 249                                   void *event)
 250 {
 251         struct hist_field *operand = hist_field->operands[0];
 252
 253         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 254         u64 val = (u64)-sval;
 255
 256         return val;
 257 }
 258
 259 #define DEFINE_HIST_FIELD_FN(type)                                      \
 260         static u64 hist_field_##type(struct hist_field *hist_field,     \
 261                                      struct tracing_map_elt *elt,       \
 262                                      struct ring_buffer_event *rbe,     \
 263                                      void *event)                       \
 264 {                                                                       \
 265         type *addr = (type *)(event + hist_field->field->offset);       \
 266                                                                         \
 267         return (u64)(unsigned long)*addr;                               \
 268 }
 269
 270 DEFINE_HIST_FIELD_FN(s64);
 271 DEFINE_HIST_FIELD_FN(u64);
 272 DEFINE_HIST_FIELD_FN(s32);
 273 DEFINE_HIST_FIELD_FN(u32);
 274 DEFINE_HIST_FIELD_FN(s16);
 275 DEFINE_HIST_FIELD_FN(u16);
 276 DEFINE_HIST_FIELD_FN(s8);
 277 DEFINE_HIST_FIELD_FN(u8);
 278
 279 #define for_each_hist_field(i, hist_data)       \
 280         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 281
 282 #define for_each_hist_val_field(i, hist_data)   \
 283         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 284
 285 #define for_each_hist_key_field(i, hist_data)   \
 286         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 287
 288 #define HIST_STACKTRACE_DEPTH   16
 289 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 290 #define HIST_STACKTRACE_SKIP    5
 291
 292 #define HITCOUNT_IDX            0
 293 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 294
 295 enum hist_field_flags {
 296         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 297         HIST_FIELD_FL_KEY               = 1 << 1,
 298         HIST_FIELD_FL_STRING            = 1 << 2,
 299         HIST_FIELD_FL_HEX               = 1 << 3,
 300         HIST_FIELD_FL_SYM               = 1 << 4,
 301         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 302         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 303         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 304         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 305         HIST_FIELD_FL_LOG2              = 1 << 9,
 306         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 307         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 308         HIST_FIELD_FL_VAR               = 1 << 12,
 309         HIST_FIELD_FL_EXPR              = 1 << 13,
 310         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 311         HIST_FIELD_FL_CPU               = 1 << 15,
 312         HIST_FIELD_FL_ALIAS             = 1 << 16,
 313 };
 314
 315 struct var_defs {
 316         unsigned int    n_vars;
 317         char            *name[TRACING_MAP_VARS_MAX];
 318         char            *expr[TRACING_MAP_VARS_MAX];
 319 };
 320
 321 struct hist_trigger_attrs {
 322         char            *keys_str;
 323         char            *vals_str;
 324         char            *sort_key_str;
 325         char            *name;
 326         char            *clock;
 327         bool            pause;
 328         bool            cont;
 329         bool            clear;
 330         bool            ts_in_usecs;
 331         unsigned int    map_bits;
 332
 333         char            *assignment_str[TRACING_MAP_VARS_MAX];
 334         unsigned int    n_assignments;
 335
 336         char            *action_str[HIST_ACTIONS_MAX];
 337         unsigned int    n_actions;
 338
 339         struct var_defs var_defs;
 340 };
 341
 342 struct field_var {
 343         struct hist_field       *var;
 344         struct hist_field       *val;
 345 };
 346
 347 struct field_var_hist {
 348         struct hist_trigger_data        *hist_data;
 349         char                            *cmd;
 350 };
 351
 352 struct hist_trigger_data {
 353         struct hist_field               *fields[HIST_FIELDS_MAX];
 354         unsigned int                    n_vals;
 355         unsigned int                    n_keys;
 356         unsigned int                    n_fields;
 357         unsigned int                    n_vars;
 358         unsigned int                    key_size;
 359         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 360         unsigned int                    n_sort_keys;
 361         struct trace_event_file         *event_file;
 362         struct hist_trigger_attrs       *attrs;
 363         struct tracing_map              *map;
 364         bool                            enable_timestamps;
 365         bool                            remove;
 366         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 367         unsigned int                    n_var_refs;
 368
 369         struct action_data              *actions[HIST_ACTIONS_MAX];
 370         unsigned int                    n_actions;
 371
 372         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 373         unsigned int                    n_field_vars;
 374         unsigned int                    n_field_var_str;
 375         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 376         unsigned int                    n_field_var_hists;
 377
 378         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 379         unsigned int                    n_save_vars;
 380         unsigned int                    n_save_var_str;
 381 };
 382
 383 static int create_synth_event(int argc, const char **argv);
 384 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 385 static int synth_event_release(struct dyn_event *ev);
 386 static bool synth_event_is_busy(struct dyn_event *ev);
 387 static bool synth_event_match(const char *system, const char *event,
 388                         int argc, const char **argv, struct dyn_event *ev);
 389
 390 static struct dyn_event_operations synth_event_ops = {
 391         .create = create_synth_event,
 392         .show = synth_event_show,
 393         .is_busy = synth_event_is_busy,
 394         .free = synth_event_release,
 395         .match = synth_event_match,
 396 };
 397
 398 struct synth_field {
 399         char *type;
 400         char *name;
 401         size_t size;
 402         unsigned int offset;
 403         bool is_signed;
 404         bool is_string;
 405 };
 406
 407 struct synth_event {
 408         struct dyn_event                        devent;
 409         int                                     ref;
 410         char                                    *name;
 411         struct synth_field                      **fields;
 412         unsigned int                            n_fields;
 413         unsigned int                            n_u64;
 414         struct trace_event_class                class;
 415         struct trace_event_call                 call;
 416         struct tracepoint                       *tp;
 417         struct module                           *mod;
 418 };
 419
 420 static bool is_synth_event(struct dyn_event *ev)
 421 {
 422         return ev->ops == &synth_event_ops;
 423 }
 424
 425 static struct synth_event *to_synth_event(struct dyn_event *ev)
 426 {
 427         return container_of(ev, struct synth_event, devent);
 428 }
 429
 430 static bool synth_event_is_busy(struct dyn_event *ev)
 431 {
 432         struct synth_event *event = to_synth_event(ev);
 433
 434         return event->ref != 0;
 435 }
 436
 437 static bool synth_event_match(const char *system, const char *event,
 438                         int argc, const char **argv, struct dyn_event *ev)
 439 {
 440         struct synth_event *sev = to_synth_event(ev);
 441
 442         return strcmp(sev->name, event) == 0 &&
 443                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 444 }
 445
 446 struct action_data;
 447
 448 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 449                              struct tracing_map_elt *elt, void *rec,
 450                              struct ring_buffer_event *rbe, void *key,
 451                              struct action_data *data, u64 *var_ref_vals);
 452
 453 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 454
 455 enum handler_id {
 456         HANDLER_ONMATCH = 1,
 457         HANDLER_ONMAX,
 458         HANDLER_ONCHANGE,
 459 };
 460
 461 enum action_id {
 462         ACTION_SAVE = 1,
 463         ACTION_TRACE,
 464         ACTION_SNAPSHOT,
 465 };
 466
 467 struct action_data {
 468         enum handler_id         handler;
 469         enum action_id          action;
 470         char                    *action_name;
 471         action_fn_t             fn;
 472
 473         unsigned int            n_params;
 474         char                    *params[SYNTH_FIELDS_MAX];
 475
 476         /*
 477          * When a histogram trigger is hit, the values of any
 478          * references to variables, including variables being passed
 479          * as parameters to synthetic events, are collected into a
 480          * var_ref_vals array.  This var_ref_idx array is an array of
 481          * indices into the var_ref_vals array, one for each synthetic
 482          * event param, and is passed to the synthetic event
 483          * invocation.
 484          */
 485         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 486         struct synth_event      *synth_event;
 487         bool                    use_trace_keyword;
 488         char                    *synth_event_name;
 489
 490         union {
 491                 struct {
 492                         char                    *event;
 493                         char                    *event_system;
 494                 } match_data;
 495
 496                 struct {
 497                         /*
 498                          * var_str contains the $-unstripped variable
 499                          * name referenced by var_ref, and used when
 500                          * printing the action.  Because var_ref
 501                          * creation is deferred to create_actions(),
 502                          * we need a per-action way to save it until
 503                          * then, thus var_str.
 504                          */
 505                         char                    *var_str;
 506
 507                         /*
 508                          * var_ref refers to the variable being
 509                          * tracked e.g onmax($var).
 510                          */
 511                         struct hist_field       *var_ref;
 512
 513                         /*
 514                          * track_var contains the 'invisible' tracking
 515                          * variable created to keep the current
 516                          * e.g. max value.
 517                          */
 518                         struct hist_field       *track_var;
 519
 520                         check_track_val_fn_t    check_val;
 521                         action_fn_t             save_data;
 522                 } track_data;
 523         };
 524 };
 525
 526 struct track_data {
 527         u64                             track_val;
 528         bool                            updated;
 529
 530         unsigned int                    key_len;
 531         void                            *key;
 532         struct tracing_map_elt          elt;
 533
 534         struct action_data              *action_data;
 535         struct hist_trigger_data        *hist_data;
 536 };
 537
 538 struct hist_elt_data {
 539         char *comm;
 540         u64 *var_ref_vals;
 541         char *field_var_str[SYNTH_FIELDS_MAX];
 542 };
 543
 544 struct snapshot_context {
 545         struct tracing_map_elt  *elt;
 546         void                    *key;
 547 };
 548
 549 static void track_data_free(struct track_data *track_data)
 550 {
 551         struct hist_elt_data *elt_data;
 552
 553         if (!track_data)
 554                 return;
 555
 556         kfree(track_data->key);
 557
 558         elt_data = track_data->elt.private_data;
 559         if (elt_data) {
 560                 kfree(elt_data->comm);
 561                 kfree(elt_data);
 562         }
 563
 564         kfree(track_data);
 565 }
 566
 567 static struct track_data *track_data_alloc(unsigned int key_len,
 568                                            struct action_data *action_data,
 569                                            struct hist_trigger_data *hist_data)
 570 {
 571         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 572         struct hist_elt_data *elt_data;
 573
 574         if (!data)
 575                 return ERR_PTR(-ENOMEM);
 576
 577         data->key = kzalloc(key_len, GFP_KERNEL);
 578         if (!data->key) {
 579                 track_data_free(data);
 580                 return ERR_PTR(-ENOMEM);
 581         }
 582
 583         data->key_len = key_len;
 584         data->action_data = action_data;
 585         data->hist_data = hist_data;
 586
 587         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 588         if (!elt_data) {
 589                 track_data_free(data);
 590                 return ERR_PTR(-ENOMEM);
 591         }
 592         data->elt.private_data = elt_data;
 593
 594         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 595         if (!elt_data->comm) {
 596                 track_data_free(data);
 597                 return ERR_PTR(-ENOMEM);
 598         }
 599
 600         return data;
 601 }
 602
 603 static char last_cmd[MAX_FILTER_STR_VAL];
 604 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 605
 606 static int errpos(char *str)
 607 {
 608         return err_pos(last_cmd, str);
 609 }
 610
 611 static void last_cmd_set(struct trace_event_file *file, char *str)
 612 {
 613         const char *system = NULL, *name = NULL;
 614         struct trace_event_call *call;
 615
 616         if (!str)
 617                 return;
 618
 619         strcpy(last_cmd, "hist:");
 620         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 621
 622         if (file) {
 623                 call = file->event_call;
 624
 625                 system = call->class->system;
 626                 if (system) {
 627                         name = trace_event_name(call);
 628                         if (!name)
 629                                 system = NULL;
 630                 }
 631         }
 632
 633         if (system)
 634                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 635 }
 636
 637 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 638 {
 639         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 640                         err_type, err_pos);
 641 }
 642
 643 static void hist_err_clear(void)
 644 {
 645         last_cmd[0] = '\0';
 646         last_cmd_loc[0] = '\0';
 647 }
 648
 649 struct synth_trace_event {
 650         struct trace_entry      ent;
 651         u64                     fields[];
 652 };
 653
 654 static int synth_event_define_fields(struct trace_event_call *call)
 655 {
 656         struct synth_trace_event trace;
 657         int offset = offsetof(typeof(trace), fields);
 658         struct synth_event *event = call->data;
 659         unsigned int i, size, n_u64;
 660         char *name, *type;
 661         bool is_signed;
 662         int ret = 0;
 663
 664         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 665                 size = event->fields[i]->size;
 666                 is_signed = event->fields[i]->is_signed;
 667                 type = event->fields[i]->type;
 668                 name = event->fields[i]->name;
 669                 ret = trace_define_field(call, type, name, offset, size,
 670                                          is_signed, FILTER_OTHER);
 671                 if (ret)
 672                         break;
 673
 674                 event->fields[i]->offset = n_u64;
 675
 676                 if (event->fields[i]->is_string) {
 677                         offset += STR_VAR_LEN_MAX;
 678                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 679                 } else {
 680                         offset += sizeof(u64);
 681                         n_u64++;
 682                 }
 683         }
 684
 685         event->n_u64 = n_u64;
 686
 687         return ret;
 688 }
 689
 690 static bool synth_field_signed(char *type)
 691 {
 692         if (str_has_prefix(type, "u"))
 693                 return false;
 694         if (strcmp(type, "gfp_t") == 0)
 695                 return false;
 696
 697         return true;
 698 }
 699
 700 static int synth_field_is_string(char *type)
 701 {
 702         if (strstr(type, "char[") != NULL)
 703                 return true;
 704
 705         return false;
 706 }
 707
 708 static int synth_field_string_size(char *type)
 709 {
 710         char buf[4], *end, *start;
 711         unsigned int len;
 712         int size, err;
 713
 714         start = strstr(type, "char[");
 715         if (start == NULL)
 716                 return -EINVAL;
 717         start += sizeof("char[") - 1;
 718
 719         end = strchr(type, ']');
 720         if (!end || end < start)
 721                 return -EINVAL;
 722
 723         len = end - start;
 724         if (len > 3)
 725                 return -EINVAL;
 726
 727         strncpy(buf, start, len);
 728         buf[len] = '\0';
 729
 730         err = kstrtouint(buf, 0, &size);
 731         if (err)
 732                 return err;
 733
 734         if (size > STR_VAR_LEN_MAX)
 735                 return -EINVAL;
 736
 737         return size;
 738 }
 739
 740 static int synth_field_size(char *type)
 741 {
 742         int size = 0;
 743
 744         if (strcmp(type, "s64") == 0)
 745                 size = sizeof(s64);
 746         else if (strcmp(type, "u64") == 0)
 747                 size = sizeof(u64);
 748         else if (strcmp(type, "s32") == 0)
 749                 size = sizeof(s32);
 750         else if (strcmp(type, "u32") == 0)
 751                 size = sizeof(u32);
 752         else if (strcmp(type, "s16") == 0)
 753                 size = sizeof(s16);
 754         else if (strcmp(type, "u16") == 0)
 755                 size = sizeof(u16);
 756         else if (strcmp(type, "s8") == 0)
 757                 size = sizeof(s8);
 758         else if (strcmp(type, "u8") == 0)
 759                 size = sizeof(u8);
 760         else if (strcmp(type, "char") == 0)
 761                 size = sizeof(char);
 762         else if (strcmp(type, "unsigned char") == 0)
 763                 size = sizeof(unsigned char);
 764         else if (strcmp(type, "int") == 0)
 765                 size = sizeof(int);
 766         else if (strcmp(type, "unsigned int") == 0)
 767                 size = sizeof(unsigned int);
 768         else if (strcmp(type, "long") == 0)
 769                 size = sizeof(long);
 770         else if (strcmp(type, "unsigned long") == 0)
 771                 size = sizeof(unsigned long);
 772         else if (strcmp(type, "pid_t") == 0)
 773                 size = sizeof(pid_t);
 774         else if (strcmp(type, "gfp_t") == 0)
 775                 size = sizeof(gfp_t);
 776         else if (synth_field_is_string(type))
 777                 size = synth_field_string_size(type);
 778
 779         return size;
 780 }
 781
 782 static const char *synth_field_fmt(char *type)
 783 {
 784         const char *fmt = "%llu";
 785
 786         if (strcmp(type, "s64") == 0)
 787                 fmt = "%lld";
 788         else if (strcmp(type, "u64") == 0)
 789                 fmt = "%llu";
 790         else if (strcmp(type, "s32") == 0)
 791                 fmt = "%d";
 792         else if (strcmp(type, "u32") == 0)
 793                 fmt = "%u";
 794         else if (strcmp(type, "s16") == 0)
 795                 fmt = "%d";
 796         else if (strcmp(type, "u16") == 0)
 797                 fmt = "%u";
 798         else if (strcmp(type, "s8") == 0)
 799                 fmt = "%d";
 800         else if (strcmp(type, "u8") == 0)
 801                 fmt = "%u";
 802         else if (strcmp(type, "char") == 0)
 803                 fmt = "%d";
 804         else if (strcmp(type, "unsigned char") == 0)
 805                 fmt = "%u";
 806         else if (strcmp(type, "int") == 0)
 807                 fmt = "%d";
 808         else if (strcmp(type, "unsigned int") == 0)
 809                 fmt = "%u";
 810         else if (strcmp(type, "long") == 0)
 811                 fmt = "%ld";
 812         else if (strcmp(type, "unsigned long") == 0)
 813                 fmt = "%lu";
 814         else if (strcmp(type, "pid_t") == 0)
 815                 fmt = "%d";
 816         else if (strcmp(type, "gfp_t") == 0)
 817                 fmt = "%x";
 818         else if (synth_field_is_string(type))
 819                 fmt = "%s";
 820
 821         return fmt;
 822 }
 823
 824 static void print_synth_event_num_val(struct trace_seq *s,
 825                                       char *print_fmt, char *name,
 826                                       int size, u64 val, char *space)
 827 {
 828         switch (size) {
 829         case 1:
 830                 trace_seq_printf(s, print_fmt, name, (u8)val, space);
 831                 break;
 832
 833         case 2:
 834                 trace_seq_printf(s, print_fmt, name, (u16)val, space);
 835                 break;
 836
 837         case 4:
 838                 trace_seq_printf(s, print_fmt, name, (u32)val, space);
 839                 break;
 840
 841         default:
 842                 trace_seq_printf(s, print_fmt, name, val, space);
 843                 break;
 844         }
 845 }
 846
 847 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 848                                            int flags,
 849                                            struct trace_event *event)
 850 {
 851         struct trace_array *tr = iter->tr;
 852         struct trace_seq *s = &iter->seq;
 853         struct synth_trace_event *entry;
 854         struct synth_event *se;
 855         unsigned int i, n_u64;
 856         char print_fmt[32];
 857         const char *fmt;
 858
 859         entry = (struct synth_trace_event *)iter->ent;
 860         se = container_of(event, struct synth_event, call.event);
 861
 862         trace_seq_printf(s, "%s: ", se->name);
 863
 864         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 865                 if (trace_seq_has_overflowed(s))
 866                         goto end;
 867
 868                 fmt = synth_field_fmt(se->fields[i]->type);
 869
 870                 /* parameter types */
 871                 if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
 872                         trace_seq_printf(s, "%s ", fmt);
 873
 874                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 875
 876                 /* parameter values */
 877                 if (se->fields[i]->is_string) {
 878                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 879                                          (char *)&entry->fields[n_u64],
 880                                          i == se->n_fields - 1 ? "" : " ");
 881                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 882                 } else {
 883                         struct trace_print_flags __flags[] = {
 884                             __def_gfpflag_names, {-1, NULL} };
 885                         char *space = (i == se->n_fields - 1 ? "" : " ");
 886
 887                         print_synth_event_num_val(s, print_fmt,
 888                                                   se->fields[i]->name,
 889                                                   se->fields[i]->size,
 890                                                   entry->fields[n_u64],
 891                                                   space);
 892
 893                         if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 894                                 trace_seq_puts(s, " (");
 895                                 trace_print_flags_seq(s, "|",
 896                                                       entry->fields[n_u64],
 897                                                       __flags);
 898                                 trace_seq_putc(s, ')');
 899                         }
 900                         n_u64++;
 901                 }
 902         }
 903 end:
 904         trace_seq_putc(s, '\n');
 905
 906         return trace_handle_return(s);
 907 }
 908
 909 static struct trace_event_functions synth_event_funcs = {
 910         .trace          = print_synth_event
 911 };
 912
 913 static notrace void trace_event_raw_event_synth(void *__data,
 914                                                 u64 *var_ref_vals,
 915                                                 unsigned int *var_ref_idx)
 916 {
 917         struct trace_event_file *trace_file = __data;
 918         struct synth_trace_event *entry;
 919         struct trace_event_buffer fbuffer;
 920         struct trace_buffer *buffer;
 921         struct synth_event *event;
 922         unsigned int i, n_u64, val_idx;
 923         int fields_size = 0;
 924
 925         event = trace_file->event_call->data;
 926
 927         if (trace_trigger_soft_disabled(trace_file))
 928                 return;
 929
 930         fields_size = event->n_u64 * sizeof(u64);
 931
 932         /*
 933          * Avoid ring buffer recursion detection, as this event
 934          * is being performed within another event.
 935          */
 936         buffer = trace_file->tr->array_buffer.buffer;
 937         ring_buffer_nest_start(buffer);
 938
 939         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 940                                            sizeof(*entry) + fields_size);
 941         if (!entry)
 942                 goto out;
 943
 944         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 945                 val_idx = var_ref_idx[i];
 946                 if (event->fields[i]->is_string) {
 947                         char *str_val = (char *)(long)var_ref_vals[val_idx];
 948                         char *str_field = (char *)&entry->fields[n_u64];
 949
 950                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 951                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 952                 } else {
 953                         struct synth_field *field = event->fields[i];
 954                         u64 val = var_ref_vals[val_idx];
 955
 956                         switch (field->size) {
 957                         case 1:
 958                                 *(u8 *)&entry->fields[n_u64] = (u8)val;
 959                                 break;
 960
 961                         case 2:
 962                                 *(u16 *)&entry->fields[n_u64] = (u16)val;
 963                                 break;
 964
 965                         case 4:
 966                                 *(u32 *)&entry->fields[n_u64] = (u32)val;
 967                                 break;
 968
 969                         default:
 970                                 entry->fields[n_u64] = val;
 971                                 break;
 972                         }
 973                         n_u64++;
 974                 }
 975         }
 976
 977         trace_event_buffer_commit(&fbuffer);
 978 out:
 979         ring_buffer_nest_end(buffer);
 980 }
 981
 982 static void free_synth_event_print_fmt(struct trace_event_call *call)
 983 {
 984         if (call) {
 985                 kfree(call->print_fmt);
 986                 call->print_fmt = NULL;
 987         }
 988 }
 989
 990 static int __set_synth_event_print_fmt(struct synth_event *event,
 991                                        char *buf, int len)
 992 {
 993         const char *fmt;
 994         int pos = 0;
 995         int i;
 996
 997         /* When len=0, we just calculate the needed length */
 998 #define LEN_OR_ZERO (len ? len - pos : 0)
 999
1000         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1001         for (i = 0; i < event->n_fields; i++) {
1002                 fmt = synth_field_fmt(event->fields[i]->type);
1003                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
1004                                 event->fields[i]->name, fmt,
1005                                 i == event->n_fields - 1 ? "" : ", ");
1006         }
1007         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1008
1009         for (i = 0; i < event->n_fields; i++) {
1010                 pos += snprintf(buf + pos, LEN_OR_ZERO,
1011                                 ", REC->%s", event->fields[i]->name);
1012         }
1013
1014 #undef LEN_OR_ZERO
1015
1016         /* return the length of print_fmt */
1017         return pos;
1018 }
1019
1020 static int set_synth_event_print_fmt(struct trace_event_call *call)
1021 {
1022         struct synth_event *event = call->data;
1023         char *print_fmt;
1024         int len;
1025
1026         /* First: called with 0 length to calculate the needed length */
1027         len = __set_synth_event_print_fmt(event, NULL, 0);
1028
1029         print_fmt = kmalloc(len + 1, GFP_KERNEL);
1030         if (!print_fmt)
1031                 return -ENOMEM;
1032
1033         /* Second: actually write the @print_fmt */
1034         __set_synth_event_print_fmt(event, print_fmt, len + 1);
1035         call->print_fmt = print_fmt;
1036
1037         return 0;
1038 }
1039
1040 static void free_synth_field(struct synth_field *field)
1041 {
1042         kfree(field->type);
1043         kfree(field->name);
1044         kfree(field);
1045 }
1046
1047 static struct synth_field *parse_synth_field(int argc, const char **argv,
1048                                              int *consumed)
1049 {
1050         struct synth_field *field;
1051         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
1052         int len, ret = 0;
1053
1054         if (field_type[0] == ';')
1055                 field_type++;
1056
1057         if (!strcmp(field_type, "unsigned")) {
1058                 if (argc < 3)
1059                         return ERR_PTR(-EINVAL);
1060                 prefix = "unsigned ";
1061                 field_type = argv[1];
1062                 field_name = argv[2];
1063                 *consumed = 3;
1064         } else {
1065                 field_name = argv[1];
1066                 *consumed = 2;
1067         }
1068
1069         field = kzalloc(sizeof(*field), GFP_KERNEL);
1070         if (!field)
1071                 return ERR_PTR(-ENOMEM);
1072
1073         len = strlen(field_name);
1074         array = strchr(field_name, '[');
1075         if (array)
1076                 len -= strlen(array);
1077         else if (field_name[len - 1] == ';')
1078                 len--;
1079
1080         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1081         if (!field->name) {
1082                 ret = -ENOMEM;
1083                 goto free;
1084         }
1085
1086         if (field_type[0] == ';')
1087                 field_type++;
1088         len = strlen(field_type) + 1;
1089         if (array)
1090                 len += strlen(array);
1091         if (prefix)
1092                 len += strlen(prefix);
1093
1094         field->type = kzalloc(len, GFP_KERNEL);
1095         if (!field->type) {
1096                 ret = -ENOMEM;
1097                 goto free;
1098         }
1099         if (prefix)
1100                 strcat(field->type, prefix);
1101         strcat(field->type, field_type);
1102         if (array) {
1103                 strcat(field->type, array);
1104                 if (field->type[len - 1] == ';')
1105                         field->type[len - 1] = '\0';
1106         }
1107
1108         field->size = synth_field_size(field->type);
1109         if (!field->size) {
1110                 ret = -EINVAL;
1111                 goto free;
1112         }
1113
1114         if (synth_field_is_string(field->type))
1115                 field->is_string = true;
1116
1117         field->is_signed = synth_field_signed(field->type);
1118
1119  out:
1120         return field;
1121  free:
1122         free_synth_field(field);
1123         field = ERR_PTR(ret);
1124         goto out;
1125 }
1126
1127 static void free_synth_tracepoint(struct tracepoint *tp)
1128 {
1129         if (!tp)
1130                 return;
1131
1132         kfree(tp->name);
1133         kfree(tp);
1134 }
1135
1136 static struct tracepoint *alloc_synth_tracepoint(char *name)
1137 {
1138         struct tracepoint *tp;
1139
1140         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1141         if (!tp)
1142                 return ERR_PTR(-ENOMEM);
1143
1144         tp->name = kstrdup(name, GFP_KERNEL);
1145         if (!tp->name) {
1146                 kfree(tp);
1147                 return ERR_PTR(-ENOMEM);
1148         }
1149
1150         return tp;
1151 }
1152
1153 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1154                                     unsigned int *var_ref_idx);
1155
1156 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1157                                unsigned int *var_ref_idx)
1158 {
1159         struct tracepoint *tp = event->tp;
1160
1161         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1162                 struct tracepoint_func *probe_func_ptr;
1163                 synth_probe_func_t probe_func;
1164                 void *__data;
1165
1166                 if (!(cpu_online(raw_smp_processor_id())))
1167                         return;
1168
1169                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1170                 if (probe_func_ptr) {
1171                         do {
1172                                 probe_func = probe_func_ptr->func;
1173                                 __data = probe_func_ptr->data;
1174                                 probe_func(__data, var_ref_vals, var_ref_idx);
1175                         } while ((++probe_func_ptr)->func);
1176                 }
1177         }
1178 }
1179
1180 static struct synth_event *find_synth_event(const char *name)
1181 {
1182         struct dyn_event *pos;
1183         struct synth_event *event;
1184
1185         for_each_dyn_event(pos) {
1186                 if (!is_synth_event(pos))
1187                         continue;
1188                 event = to_synth_event(pos);
1189                 if (strcmp(event->name, name) == 0)
1190                         return event;
1191         }
1192
1193         return NULL;
1194 }
1195
1196 static struct trace_event_fields synth_event_fields_array[] = {
1197         { .type = TRACE_FUNCTION_TYPE,
1198           .define_fields = synth_event_define_fields },
1199         {}
1200 };
1201
1202 static int register_synth_event(struct synth_event *event)
1203 {
1204         struct trace_event_call *call = &event->call;
1205         int ret = 0;
1206
1207         event->call.class = &event->class;
1208         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1209         if (!event->class.system) {
1210                 ret = -ENOMEM;
1211                 goto out;
1212         }
1213
1214         event->tp = alloc_synth_tracepoint(event->name);
1215         if (IS_ERR(event->tp)) {
1216                 ret = PTR_ERR(event->tp);
1217                 event->tp = NULL;
1218                 goto out;
1219         }
1220
1221         INIT_LIST_HEAD(&call->class->fields);
1222         call->event.funcs = &synth_event_funcs;
1223         call->class->fields_array = synth_event_fields_array;
1224
1225         ret = register_trace_event(&call->event);
1226         if (!ret) {
1227                 ret = -ENODEV;
1228                 goto out;
1229         }
1230         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1231         call->class->reg = trace_event_reg;
1232         call->class->probe = trace_event_raw_event_synth;
1233         call->data = event;
1234         call->tp = event->tp;
1235
1236         ret = trace_add_event_call(call);
1237         if (ret) {
1238                 pr_warn("Failed to register synthetic event: %s\n",
1239                         trace_event_name(call));
1240                 goto err;
1241         }
1242
1243         ret = set_synth_event_print_fmt(call);
1244         if (ret < 0) {
1245                 trace_remove_event_call(call);
1246                 goto err;
1247         }
1248  out:
1249         return ret;
1250  err:
1251         unregister_trace_event(&call->event);
1252         goto out;
1253 }
1254
1255 static int unregister_synth_event(struct synth_event *event)
1256 {
1257         struct trace_event_call *call = &event->call;
1258         int ret;
1259
1260         ret = trace_remove_event_call(call);
1261
1262         return ret;
1263 }
1264
1265 static void free_synth_event(struct synth_event *event)
1266 {
1267         unsigned int i;
1268
1269         if (!event)
1270                 return;
1271
1272         for (i = 0; i < event->n_fields; i++)
1273                 free_synth_field(event->fields[i]);
1274
1275         kfree(event->fields);
1276         kfree(event->name);
1277         kfree(event->class.system);
1278         free_synth_tracepoint(event->tp);
1279         free_synth_event_print_fmt(&event->call);
1280         kfree(event);
1281 }
1282
1283 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1284                                              struct synth_field **fields)
1285 {
1286         struct synth_event *event;
1287         unsigned int i;
1288
1289         event = kzalloc(sizeof(*event), GFP_KERNEL);
1290         if (!event) {
1291                 event = ERR_PTR(-ENOMEM);
1292                 goto out;
1293         }
1294
1295         event->name = kstrdup(name, GFP_KERNEL);
1296         if (!event->name) {
1297                 kfree(event);
1298                 event = ERR_PTR(-ENOMEM);
1299                 goto out;
1300         }
1301
1302         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1303         if (!event->fields) {
1304                 free_synth_event(event);
1305                 event = ERR_PTR(-ENOMEM);
1306                 goto out;
1307         }
1308
1309         dyn_event_init(&event->devent, &synth_event_ops);
1310
1311         for (i = 0; i < n_fields; i++)
1312                 event->fields[i] = fields[i];
1313
1314         event->n_fields = n_fields;
1315  out:
1316         return event;
1317 }
1318
1319 static void action_trace(struct hist_trigger_data *hist_data,
1320                          struct tracing_map_elt *elt, void *rec,
1321                          struct ring_buffer_event *rbe, void *key,
1322                          struct action_data *data, u64 *var_ref_vals)
1323 {
1324         struct synth_event *event = data->synth_event;
1325
1326         trace_synth(event, var_ref_vals, data->var_ref_idx);
1327 }
1328
1329 struct hist_var_data {
1330         struct list_head list;
1331         struct hist_trigger_data *hist_data;
1332 };
1333
1334 static int synth_event_check_arg_fn(void *data)
1335 {
1336         struct dynevent_arg_pair *arg_pair = data;
1337         int size;
1338
1339         size = synth_field_size((char *)arg_pair->lhs);
1340
1341         return size ? 0 : -EINVAL;
1342 }
1343
1344 /**
1345  * synth_event_add_field - Add a new field to a synthetic event cmd
1346  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1347  * @type: The type of the new field to add
1348  * @name: The name of the new field to add
1349  *
1350  * Add a new field to a synthetic event cmd object.  Field ordering is in
1351  * the same order the fields are added.
1352  *
1353  * See synth_field_size() for available types. If field_name contains
1354  * [n] the field is considered to be an array.
1355  *
1356  * Return: 0 if successful, error otherwise.
1357  */
1358 int synth_event_add_field(struct dynevent_cmd *cmd, const char *type,
1359                           const char *name)
1360 {
1361         struct dynevent_arg_pair arg_pair;
1362         int ret;
1363
1364         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1365                 return -EINVAL;
1366
1367         if (!type || !name)
1368                 return -EINVAL;
1369
1370         dynevent_arg_pair_init(&arg_pair, 0, ';');
1371
1372         arg_pair.lhs = type;
1373         arg_pair.rhs = name;
1374
1375         ret = dynevent_arg_pair_add(cmd, &arg_pair, synth_event_check_arg_fn);
1376         if (ret)
1377                 return ret;
1378
1379         if (++cmd->n_fields > SYNTH_FIELDS_MAX)
1380                 ret = -EINVAL;
1381
1382         return ret;
1383 }
1384 EXPORT_SYMBOL_GPL(synth_event_add_field);
1385
1386 /**
1387  * synth_event_add_field_str - Add a new field to a synthetic event cmd
1388  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1389  * @type_name: The type and name of the new field to add, as a single string
1390  *
1391  * Add a new field to a synthetic event cmd object, as a single
1392  * string.  The @type_name string is expected to be of the form 'type
1393  * name', which will be appended by ';'.  No sanity checking is done -
1394  * what's passed in is assumed to already be well-formed.  Field
1395  * ordering is in the same order the fields are added.
1396  *
1397  * See synth_field_size() for available types. If field_name contains
1398  * [n] the field is considered to be an array.
1399  *
1400  * Return: 0 if successful, error otherwise.
1401  */
1402 int synth_event_add_field_str(struct dynevent_cmd *cmd, const char *type_name)
1403 {
1404         struct dynevent_arg arg;
1405         int ret;
1406
1407         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1408                 return -EINVAL;
1409
1410         if (!type_name)
1411                 return -EINVAL;
1412
1413         dynevent_arg_init(&arg, ';');
1414
1415         arg.str = type_name;
1416
1417         ret = dynevent_arg_add(cmd, &arg, NULL);
1418         if (ret)
1419                 return ret;
1420
1421         if (++cmd->n_fields > SYNTH_FIELDS_MAX)
1422                 ret = -EINVAL;
1423
1424         return ret;
1425 }
1426 EXPORT_SYMBOL_GPL(synth_event_add_field_str);
1427
1428 /**
1429  * synth_event_add_fields - Add multiple fields to a synthetic event cmd
1430  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1431  * @fields: An array of type/name field descriptions
1432  * @n_fields: The number of field descriptions contained in the fields array
1433  *
1434  * Add a new set of fields to a synthetic event cmd object.  The event
1435  * fields that will be defined for the event should be passed in as an
1436  * array of struct synth_field_desc, and the number of elements in the
1437  * array passed in as n_fields.  Field ordering will retain the
1438  * ordering given in the fields array.
1439  *
1440  * See synth_field_size() for available types. If field_name contains
1441  * [n] the field is considered to be an array.
1442  *
1443  * Return: 0 if successful, error otherwise.
1444  */
1445 int synth_event_add_fields(struct dynevent_cmd *cmd,
1446                            struct synth_field_desc *fields,
1447                            unsigned int n_fields)
1448 {
1449         unsigned int i;
1450         int ret = 0;
1451
1452         for (i = 0; i < n_fields; i++) {
1453                 if (fields[i].type == NULL || fields[i].name == NULL) {
1454                         ret = -EINVAL;
1455                         break;
1456                 }
1457
1458                 ret = synth_event_add_field(cmd, fields[i].type, fields[i].name);
1459                 if (ret)
1460                         break;
1461         }
1462
1463         return ret;
1464 }
1465 EXPORT_SYMBOL_GPL(synth_event_add_fields);
1466
1467 /**
1468  * __synth_event_gen_cmd_start - Start a synthetic event command from arg list
1469  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1470  * @name: The name of the synthetic event
1471  * @mod: The module creating the event, NULL if not created from a module
1472  * @args: Variable number of arg (pairs), one pair for each field
1473  *
1474  * NOTE: Users normally won't want to call this function directly, but
1475  * rather use the synth_event_gen_cmd_start() wrapper, which
1476  * automatically adds a NULL to the end of the arg list.  If this
1477  * function is used directly, make sure the last arg in the variable
1478  * arg list is NULL.
1479  *
1480  * Generate a synthetic event command to be executed by
1481  * synth_event_gen_cmd_end().  This function can be used to generate
1482  * the complete command or only the first part of it; in the latter
1483  * case, synth_event_add_field(), synth_event_add_field_str(), or
1484  * synth_event_add_fields() can be used to add more fields following
1485  * this.
1486  *
1487  * There should be an even number variable args, each pair consisting
1488  * of a type followed by a field name.
1489  *
1490  * See synth_field_size() for available types. If field_name contains
1491  * [n] the field is considered to be an array.
1492  *
1493  * Return: 0 if successful, error otherwise.
1494  */
1495 int __synth_event_gen_cmd_start(struct dynevent_cmd *cmd, const char *name,
1496                                 struct module *mod, ...)
1497 {
1498         struct dynevent_arg arg;
1499         va_list args;
1500         int ret;
1501
1502         cmd->event_name = name;
1503         cmd->private_data = mod;
1504
1505         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1506                 return -EINVAL;
1507
1508         dynevent_arg_init(&arg, 0);
1509         arg.str = name;
1510         ret = dynevent_arg_add(cmd, &arg, NULL);
1511         if (ret)
1512                 return ret;
1513
1514         va_start(args, mod);
1515         for (;;) {
1516                 const char *type, *name;
1517
1518                 type = va_arg(args, const char *);
1519                 if (!type)
1520                         break;
1521                 name = va_arg(args, const char *);
1522                 if (!name)
1523                         break;
1524
1525                 if (++cmd->n_fields > SYNTH_FIELDS_MAX) {
1526                         ret = -EINVAL;
1527                         break;
1528                 }
1529
1530                 ret = synth_event_add_field(cmd, type, name);
1531                 if (ret)
1532                         break;
1533         }
1534         va_end(args);
1535
1536         return ret;
1537 }
1538 EXPORT_SYMBOL_GPL(__synth_event_gen_cmd_start);
1539
1540 /**
1541  * synth_event_gen_cmd_array_start - Start synthetic event command from an array
1542  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1543  * @name: The name of the synthetic event
1544  * @fields: An array of type/name field descriptions
1545  * @n_fields: The number of field descriptions contained in the fields array
1546  *
1547  * Generate a synthetic event command to be executed by
1548  * synth_event_gen_cmd_end().  This function can be used to generate
1549  * the complete command or only the first part of it; in the latter
1550  * case, synth_event_add_field(), synth_event_add_field_str(), or
1551  * synth_event_add_fields() can be used to add more fields following
1552  * this.
1553  *
1554  * The event fields that will be defined for the event should be
1555  * passed in as an array of struct synth_field_desc, and the number of
1556  * elements in the array passed in as n_fields.  Field ordering will
1557  * retain the ordering given in the fields array.
1558  *
1559  * See synth_field_size() for available types. If field_name contains
1560  * [n] the field is considered to be an array.
1561  *
1562  * Return: 0 if successful, error otherwise.
1563  */
1564 int synth_event_gen_cmd_array_start(struct dynevent_cmd *cmd, const char *name,
1565                                     struct module *mod,
1566                                     struct synth_field_desc *fields,
1567                                     unsigned int n_fields)
1568 {
1569         struct dynevent_arg arg;
1570         unsigned int i;
1571         int ret = 0;
1572
1573         cmd->event_name = name;
1574         cmd->private_data = mod;
1575
1576         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1577                 return -EINVAL;
1578
1579         if (n_fields > SYNTH_FIELDS_MAX)
1580                 return -EINVAL;
1581
1582         dynevent_arg_init(&arg, 0);
1583         arg.str = name;
1584         ret = dynevent_arg_add(cmd, &arg, NULL);
1585         if (ret)
1586                 return ret;
1587
1588         for (i = 0; i < n_fields; i++) {
1589                 if (fields[i].type == NULL || fields[i].name == NULL)
1590                         return -EINVAL;
1591
1592                 ret = synth_event_add_field(cmd, fields[i].type, fields[i].name);
1593                 if (ret)
1594                         break;
1595         }
1596
1597         return ret;
1598 }
1599 EXPORT_SYMBOL_GPL(synth_event_gen_cmd_array_start);
1600
1601 static int __create_synth_event(int argc, const char *name, const char **argv)
1602 {
1603         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1604         struct synth_event *event = NULL;
1605         int i, consumed = 0, n_fields = 0, ret = 0;
1606
1607         /*
1608          * Argument syntax:
1609          *  - Add synthetic event: <event_name> field[;field] ...
1610          *  - Remove synthetic event: !<event_name> field[;field] ...
1611          *      where 'field' = type field_name
1612          */
1613
1614         if (name[0] == '\0' || argc < 1)
1615                 return -EINVAL;
1616
1617         mutex_lock(&event_mutex);
1618
1619         event = find_synth_event(name);
1620         if (event) {
1621                 ret = -EEXIST;
1622                 goto out;
1623         }
1624
1625         for (i = 0; i < argc - 1; i++) {
1626                 if (strcmp(argv[i], ";") == 0)
1627                         continue;
1628                 if (n_fields == SYNTH_FIELDS_MAX) {
1629                         ret = -EINVAL;
1630                         goto err;
1631                 }
1632
1633                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1634                 if (IS_ERR(field)) {
1635                         ret = PTR_ERR(field);
1636                         goto err;
1637                 }
1638                 fields[n_fields++] = field;
1639                 i += consumed - 1;
1640         }
1641
1642         if (i < argc && strcmp(argv[i], ";") != 0) {
1643                 ret = -EINVAL;
1644                 goto err;
1645         }
1646
1647         event = alloc_synth_event(name, n_fields, fields);
1648         if (IS_ERR(event)) {
1649                 ret = PTR_ERR(event);
1650                 event = NULL;
1651                 goto err;
1652         }
1653         ret = register_synth_event(event);
1654         if (!ret)
1655                 dyn_event_add(&event->devent);
1656         else
1657                 free_synth_event(event);
1658  out:
1659         mutex_unlock(&event_mutex);
1660
1661         return ret;
1662  err:
1663         for (i = 0; i < n_fields; i++)
1664                 free_synth_field(fields[i]);
1665
1666         goto out;
1667 }
1668
1669 /**
1670  * synth_event_create - Create a new synthetic event
1671  * @name: The name of the new sythetic event
1672  * @fields: An array of type/name field descriptions
1673  * @n_fields: The number of field descriptions contained in the fields array
1674  * @mod: The module creating the event, NULL if not created from a module
1675  *
1676  * Create a new synthetic event with the given name under the
1677  * trace/events/synthetic/ directory.  The event fields that will be
1678  * defined for the event should be passed in as an array of struct
1679  * synth_field_desc, and the number elements in the array passed in as
1680  * n_fields. Field ordering will retain the ordering given in the
1681  * fields array.
1682  *
1683  * If the new synthetic event is being created from a module, the mod
1684  * param must be non-NULL.  This will ensure that the trace buffer
1685  * won't contain unreadable events.
1686  *
1687  * The new synth event should be deleted using synth_event_delete()
1688  * function.  The new synthetic event can be generated from modules or
1689  * other kernel code using trace_synth_event() and related functions.
1690  *
1691  * Return: 0 if successful, error otherwise.
1692  */
1693 int synth_event_create(const char *name, struct synth_field_desc *fields,
1694                        unsigned int n_fields, struct module *mod)
1695 {
1696         struct dynevent_cmd cmd;
1697         char *buf;
1698         int ret;
1699
1700         buf = kzalloc(MAX_DYNEVENT_CMD_LEN, GFP_KERNEL);
1701         if (!buf)
1702                 return -ENOMEM;
1703
1704         synth_event_cmd_init(&cmd, buf, MAX_DYNEVENT_CMD_LEN);
1705
1706         ret = synth_event_gen_cmd_array_start(&cmd, name, mod,
1707                                               fields, n_fields);
1708         if (ret)
1709                 goto out;
1710
1711         ret = synth_event_gen_cmd_end(&cmd);
1712  out:
1713         kfree(buf);
1714
1715         return ret;
1716 }
1717 EXPORT_SYMBOL_GPL(synth_event_create);
1718
1719 static int destroy_synth_event(struct synth_event *se)
1720 {
1721         int ret;
1722
1723         if (se->ref)
1724                 ret = -EBUSY;
1725         else {
1726                 ret = unregister_synth_event(se);
1727                 if (!ret) {
1728                         dyn_event_remove(&se->devent);
1729                         free_synth_event(se);
1730                 }
1731         }
1732
1733         return ret;
1734 }
1735
1736 /**
1737  * synth_event_delete - Delete a synthetic event
1738  * @event_name: The name of the new sythetic event
1739  *
1740  * Delete a synthetic event that was created with synth_event_create().
1741  *
1742  * Return: 0 if successful, error otherwise.
1743  */
1744 int synth_event_delete(const char *event_name)
1745 {
1746         struct synth_event *se = NULL;
1747         struct module *mod = NULL;
1748         int ret = -ENOENT;
1749
1750         mutex_lock(&event_mutex);
1751         se = find_synth_event(event_name);
1752         if (se) {
1753                 mod = se->mod;
1754                 ret = destroy_synth_event(se);
1755         }
1756         mutex_unlock(&event_mutex);
1757
1758         if (mod) {
1759                 mutex_lock(&trace_types_lock);
1760                 /*
1761                  * It is safest to reset the ring buffer if the module
1762                  * being unloaded registered any events that were
1763                  * used. The only worry is if a new module gets
1764                  * loaded, and takes on the same id as the events of
1765                  * this module. When printing out the buffer, traced
1766                  * events left over from this module may be passed to
1767                  * the new module events and unexpected results may
1768                  * occur.
1769                  */
1770                 tracing_reset_all_online_cpus();
1771                 mutex_unlock(&trace_types_lock);
1772         }
1773
1774         return ret;
1775 }
1776 EXPORT_SYMBOL_GPL(synth_event_delete);
1777
1778 static int create_or_delete_synth_event(int argc, char **argv)
1779 {
1780         const char *name = argv[0];
1781         int ret;
1782
1783         /* trace_run_command() ensures argc != 0 */
1784         if (name[0] == '!') {
1785                 ret = synth_event_delete(name + 1);
1786                 return ret;
1787         }
1788
1789         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1790         return ret == -ECANCELED ? -EINVAL : ret;
1791 }
1792
1793 static int synth_event_run_command(struct dynevent_cmd *cmd)
1794 {
1795         struct synth_event *se;
1796         int ret;
1797
1798         ret = trace_run_command(cmd->seq.buffer, create_or_delete_synth_event);
1799         if (ret)
1800                 return ret;
1801
1802         se = find_synth_event(cmd->event_name);
1803         if (WARN_ON(!se))
1804                 return -ENOENT;
1805
1806         se->mod = cmd->private_data;
1807
1808         return ret;
1809 }
1810
1811 /**
1812  * synth_event_cmd_init - Initialize a synthetic event command object
1813  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1814  * @buf: A pointer to the buffer used to build the command
1815  * @maxlen: The length of the buffer passed in @buf
1816  *
1817  * Initialize a synthetic event command object.  Use this before
1818  * calling any of the other dyenvent_cmd functions.
1819  */
1820 void synth_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen)
1821 {
1822         dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_SYNTH,
1823                           synth_event_run_command);
1824 }
1825 EXPORT_SYMBOL_GPL(synth_event_cmd_init);
1826
1827 static inline int
1828 __synth_event_trace_start(struct trace_event_file *file,
1829                           struct synth_event_trace_state *trace_state)
1830 {
1831         int entry_size, fields_size = 0;
1832         int ret = 0;
1833
1834         memset(trace_state, '\0', sizeof(*trace_state));
1835
1836         /*
1837          * Normal event tracing doesn't get called at all unless the
1838          * ENABLED bit is set (which attaches the probe thus allowing
1839          * this code to be called, etc).  Because this is called
1840          * directly by the user, we don't have that but we still need
1841          * to honor not logging when disabled.  For the the iterated
1842          * trace case, we save the enabed state upon start and just
1843          * ignore the following data calls.
1844          */
1845         if (!(file->flags & EVENT_FILE_FL_ENABLED) ||
1846             trace_trigger_soft_disabled(file)) {
1847                 trace_state->disabled = true;
1848                 ret = -ENOENT;
1849                 goto out;
1850         }
1851
1852         trace_state->event = file->event_call->data;
1853
1854         fields_size = trace_state->event->n_u64 * sizeof(u64);
1855
1856         /*
1857          * Avoid ring buffer recursion detection, as this event
1858          * is being performed within another event.
1859          */
1860         trace_state->buffer = file->tr->array_buffer.buffer;
1861         ring_buffer_nest_start(trace_state->buffer);
1862
1863         entry_size = sizeof(*trace_state->entry) + fields_size;
1864         trace_state->entry = trace_event_buffer_reserve(&trace_state->fbuffer,
1865                                                         file,
1866                                                         entry_size);
1867         if (!trace_state->entry) {
1868                 ring_buffer_nest_end(trace_state->buffer);
1869                 ret = -EINVAL;
1870         }
1871 out:
1872         return ret;
1873 }
1874
1875 static inline void
1876 __synth_event_trace_end(struct synth_event_trace_state *trace_state)
1877 {
1878         trace_event_buffer_commit(&trace_state->fbuffer);
1879
1880         ring_buffer_nest_end(trace_state->buffer);
1881 }
1882
1883 /**
1884  * synth_event_trace - Trace a synthetic event
1885  * @file: The trace_event_file representing the synthetic event
1886  * @n_vals: The number of values in vals
1887  * @args: Variable number of args containing the event values
1888  *
1889  * Trace a synthetic event using the values passed in the variable
1890  * argument list.
1891  *
1892  * The argument list should be a list 'n_vals' u64 values.  The number
1893  * of vals must match the number of field in the synthetic event, and
1894  * must be in the same order as the synthetic event fields.
1895  *
1896  * All vals should be cast to u64, and string vals are just pointers
1897  * to strings, cast to u64.  Strings will be copied into space
1898  * reserved in the event for the string, using these pointers.
1899  *
1900  * Return: 0 on success, err otherwise.
1901  */
1902 int synth_event_trace(struct trace_event_file *file, unsigned int n_vals, ...)
1903 {
1904         struct synth_event_trace_state state;
1905         unsigned int i, n_u64;
1906         va_list args;
1907         int ret;
1908
1909         ret = __synth_event_trace_start(file, &state);
1910         if (ret) {
1911                 if (ret == -ENOENT)
1912                         ret = 0; /* just disabled, not really an error */
1913                 return ret;
1914         }
1915
1916         if (n_vals != state.event->n_fields) {
1917                 ret = -EINVAL;
1918                 goto out;
1919         }
1920
1921         va_start(args, n_vals);
1922         for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) {
1923                 u64 val;
1924
1925                 val = va_arg(args, u64);
1926
1927                 if (state.event->fields[i]->is_string) {
1928                         char *str_val = (char *)(long)val;
1929                         char *str_field = (char *)&state.entry->fields[n_u64];
1930
1931                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
1932                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
1933                 } else {
1934                         struct synth_field *field = state.event->fields[i];
1935
1936                         switch (field->size) {
1937                         case 1:
1938                                 *(u8 *)&state.entry->fields[n_u64] = (u8)val;
1939                                 break;
1940
1941                         case 2:
1942                                 *(u16 *)&state.entry->fields[n_u64] = (u16)val;
1943                                 break;
1944
1945                         case 4:
1946                                 *(u32 *)&state.entry->fields[n_u64] = (u32)val;
1947                                 break;
1948
1949                         default:
1950                                 state.entry->fields[n_u64] = val;
1951                                 break;
1952                         }
1953                         n_u64++;
1954                 }
1955         }
1956         va_end(args);
1957 out:
1958         __synth_event_trace_end(&state);
1959
1960         return ret;
1961 }
1962 EXPORT_SYMBOL_GPL(synth_event_trace);
1963
1964 /**
1965  * synth_event_trace_array - Trace a synthetic event from an array
1966  * @file: The trace_event_file representing the synthetic event
1967  * @vals: Array of values
1968  * @n_vals: The number of values in vals
1969  *
1970  * Trace a synthetic event using the values passed in as 'vals'.
1971  *
1972  * The 'vals' array is just an array of 'n_vals' u64.  The number of
1973  * vals must match the number of field in the synthetic event, and
1974  * must be in the same order as the synthetic event fields.
1975  *
1976  * All vals should be cast to u64, and string vals are just pointers
1977  * to strings, cast to u64.  Strings will be copied into space
1978  * reserved in the event for the string, using these pointers.
1979  *
1980  * Return: 0 on success, err otherwise.
1981  */
1982 int synth_event_trace_array(struct trace_event_file *file, u64 *vals,
1983                             unsigned int n_vals)
1984 {
1985         struct synth_event_trace_state state;
1986         unsigned int i, n_u64;
1987         int ret;
1988
1989         ret = __synth_event_trace_start(file, &state);
1990         if (ret) {
1991                 if (ret == -ENOENT)
1992                         ret = 0; /* just disabled, not really an error */
1993                 return ret;
1994         }
1995
1996         if (n_vals != state.event->n_fields) {
1997                 ret = -EINVAL;
1998                 goto out;
1999         }
2000
2001         for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) {
2002                 if (state.event->fields[i]->is_string) {
2003                         char *str_val = (char *)(long)vals[i];
2004                         char *str_field = (char *)&state.entry->fields[n_u64];
2005
2006                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
2007                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
2008                 } else {
2009                         struct synth_field *field = state.event->fields[i];
2010                         u64 val = vals[i];
2011
2012                         switch (field->size) {
2013                         case 1:
2014                                 *(u8 *)&state.entry->fields[n_u64] = (u8)val;
2015                                 break;
2016
2017                         case 2:
2018                                 *(u16 *)&state.entry->fields[n_u64] = (u16)val;
2019                                 break;
2020
2021                         case 4:
2022                                 *(u32 *)&state.entry->fields[n_u64] = (u32)val;
2023                                 break;
2024
2025                         default:
2026                                 state.entry->fields[n_u64] = val;
2027                                 break;
2028                         }
2029                         n_u64++;
2030                 }
2031         }
2032 out:
2033         __synth_event_trace_end(&state);
2034
2035         return ret;
2036 }
2037 EXPORT_SYMBOL_GPL(synth_event_trace_array);
2038
2039 /**
2040  * synth_event_trace_start - Start piecewise synthetic event trace
2041  * @file: The trace_event_file representing the synthetic event
2042  * @trace_state: A pointer to object tracking the piecewise trace state
2043  *
2044  * Start the trace of a synthetic event field-by-field rather than all
2045  * at once.
2046  *
2047  * This function 'opens' an event trace, which means space is reserved
2048  * for the event in the trace buffer, after which the event's
2049  * individual field values can be set through either
2050  * synth_event_add_next_val() or synth_event_add_val().
2051  *
2052  * A pointer to a trace_state object is passed in, which will keep
2053  * track of the current event trace state until the event trace is
2054  * closed (and the event finally traced) using
2055  * synth_event_trace_end().
2056  *
2057  * Note that synth_event_trace_end() must be called after all values
2058  * have been added for each event trace, regardless of whether adding
2059  * all field values succeeded or not.
2060  *
2061  * Note also that for a given event trace, all fields must be added
2062  * using either synth_event_add_next_val() or synth_event_add_val()
2063  * but not both together or interleaved.
2064  *
2065  * Return: 0 on success, err otherwise.
2066  */
2067 int synth_event_trace_start(struct trace_event_file *file,
2068                             struct synth_event_trace_state *trace_state)
2069 {
2070         int ret;
2071
2072         if (!trace_state)
2073                 return -EINVAL;
2074
2075         ret = __synth_event_trace_start(file, trace_state);
2076         if (ret == -ENOENT)
2077                 ret = 0; /* just disabled, not really an error */
2078
2079         return ret;
2080 }
2081 EXPORT_SYMBOL_GPL(synth_event_trace_start);
2082
2083 static int __synth_event_add_val(const char *field_name, u64 val,
2084                                  struct synth_event_trace_state *trace_state)
2085 {
2086         struct synth_field *field = NULL;
2087         struct synth_trace_event *entry;
2088         struct synth_event *event;
2089         int i, ret = 0;
2090
2091         if (!trace_state) {
2092                 ret = -EINVAL;
2093                 goto out;
2094         }
2095
2096         /* can't mix add_next_synth_val() with add_synth_val() */
2097         if (field_name) {
2098                 if (trace_state->add_next) {
2099                         ret = -EINVAL;
2100                         goto out;
2101                 }
2102                 trace_state->add_name = true;
2103         } else {
2104                 if (trace_state->add_name) {
2105                         ret = -EINVAL;
2106                         goto out;
2107                 }
2108                 trace_state->add_next = true;
2109         }
2110
2111         if (trace_state->disabled)
2112                 goto out;
2113
2114         event = trace_state->event;
2115         if (trace_state->add_name) {
2116                 for (i = 0; i < event->n_fields; i++) {
2117                         field = event->fields[i];
2118                         if (strcmp(field->name, field_name) == 0)
2119                                 break;
2120                 }
2121                 if (!field) {
2122                         ret = -EINVAL;
2123                         goto out;
2124                 }
2125         } else {
2126                 if (trace_state->cur_field >= event->n_fields) {
2127                         ret = -EINVAL;
2128                         goto out;
2129                 }
2130                 field = event->fields[trace_state->cur_field++];
2131         }
2132
2133         entry = trace_state->entry;
2134         if (field->is_string) {
2135                 char *str_val = (char *)(long)val;
2136                 char *str_field;
2137
2138                 if (!str_val) {
2139                         ret = -EINVAL;
2140                         goto out;
2141                 }
2142
2143                 str_field = (char *)&entry->fields[field->offset];
2144                 strscpy(str_field, str_val, STR_VAR_LEN_MAX);
2145         } else {
2146                 switch (field->size) {
2147                 case 1:
2148                         *(u8 *)&trace_state->entry->fields[field->offset] = (u8)val;
2149                         break;
2150
2151                 case 2:
2152                         *(u16 *)&trace_state->entry->fields[field->offset] = (u16)val;
2153                         break;
2154
2155                 case 4:
2156                         *(u32 *)&trace_state->entry->fields[field->offset] = (u32)val;
2157                         break;
2158
2159                 default:
2160                         trace_state->entry->fields[field->offset] = val;
2161                         break;
2162                 }
2163         }
2164  out:
2165         return ret;
2166 }
2167
2168 /**
2169  * synth_event_add_next_val - Add the next field's value to an open synth trace
2170  * @val: The value to set the next field to
2171  * @trace_state: A pointer to object tracking the piecewise trace state
2172  *
2173  * Set the value of the next field in an event that's been opened by
2174  * synth_event_trace_start().
2175  *
2176  * The val param should be the value cast to u64.  If the value points
2177  * to a string, the val param should be a char * cast to u64.
2178  *
2179  * This function assumes all the fields in an event are to be set one
2180  * after another - successive calls to this function are made, one for
2181  * each field, in the order of the fields in the event, until all
2182  * fields have been set.  If you'd rather set each field individually
2183  * without regard to ordering, synth_event_add_val() can be used
2184  * instead.
2185  *
2186  * Note however that synth_event_add_next_val() and
2187  * synth_event_add_val() can't be intermixed for a given event trace -
2188  * one or the other but not both can be used at the same time.
2189  *
2190  * Note also that synth_event_trace_end() must be called after all
2191  * values have been added for each event trace, regardless of whether
2192  * adding all field values succeeded or not.
2193  *
2194  * Return: 0 on success, err otherwise.
2195  */
2196 int synth_event_add_next_val(u64 val,
2197                              struct synth_event_trace_state *trace_state)
2198 {
2199         return __synth_event_add_val(NULL, val, trace_state);
2200 }
2201 EXPORT_SYMBOL_GPL(synth_event_add_next_val);
2202
2203 /**
2204  * synth_event_add_val - Add a named field's value to an open synth trace
2205  * @field_name: The name of the synthetic event field value to set
2206  * @val: The value to set the next field to
2207  * @trace_state: A pointer to object tracking the piecewise trace state
2208  *
2209  * Set the value of the named field in an event that's been opened by
2210  * synth_event_trace_start().
2211  *
2212  * The val param should be the value cast to u64.  If the value points
2213  * to a string, the val param should be a char * cast to u64.
2214  *
2215  * This function looks up the field name, and if found, sets the field
2216  * to the specified value.  This lookup makes this function more
2217  * expensive than synth_event_add_next_val(), so use that or the
2218  * none-piecewise synth_event_trace() instead if efficiency is more
2219  * important.
2220  *
2221  * Note however that synth_event_add_next_val() and
2222  * synth_event_add_val() can't be intermixed for a given event trace -
2223  * one or the other but not both can be used at the same time.
2224  *
2225  * Note also that synth_event_trace_end() must be called after all
2226  * values have been added for each event trace, regardless of whether
2227  * adding all field values succeeded or not.
2228  *
2229  * Return: 0 on success, err otherwise.
2230  */
2231 int synth_event_add_val(const char *field_name, u64 val,
2232                         struct synth_event_trace_state *trace_state)
2233 {
2234         return __synth_event_add_val(field_name, val, trace_state);
2235 }
2236 EXPORT_SYMBOL_GPL(synth_event_add_val);
2237
2238 /**
2239  * synth_event_trace_end - End piecewise synthetic event trace
2240  * @trace_state: A pointer to object tracking the piecewise trace state
2241  *
2242  * End the trace of a synthetic event opened by
2243  * synth_event_trace__start().
2244  *
2245  * This function 'closes' an event trace, which basically means that
2246  * it commits the reserved event and cleans up other loose ends.
2247  *
2248  * A pointer to a trace_state object is passed in, which will keep
2249  * track of the current event trace state opened with
2250  * synth_event_trace_start().
2251  *
2252  * Note that this function must be called after all values have been
2253  * added for each event trace, regardless of whether adding all field
2254  * values succeeded or not.
2255  *
2256  * Return: 0 on success, err otherwise.
2257  */
2258 int synth_event_trace_end(struct synth_event_trace_state *trace_state)
2259 {
2260         if (!trace_state)
2261                 return -EINVAL;
2262
2263         __synth_event_trace_end(trace_state);
2264
2265         return 0;
2266 }
2267 EXPORT_SYMBOL_GPL(synth_event_trace_end);
2268
2269 static int create_synth_event(int argc, const char **argv)
2270 {
2271         const char *name = argv[0];
2272         int len;
2273
2274         if (name[0] != 's' || name[1] != ':')
2275                 return -ECANCELED;
2276         name += 2;
2277
2278         /* This interface accepts group name prefix */
2279         if (strchr(name, '/')) {
2280                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
2281                 if (len == 0)
2282                         return -EINVAL;
2283                 name += len;
2284         }
2285         return __create_synth_event(argc - 1, name, argv + 1);
2286 }
2287
2288 static int synth_event_release(struct dyn_event *ev)
2289 {
2290         struct synth_event *event = to_synth_event(ev);
2291         int ret;
2292
2293         if (event->ref)
2294                 return -EBUSY;
2295
2296         ret = unregister_synth_event(event);
2297         if (ret)
2298                 return ret;
2299
2300         dyn_event_remove(ev);
2301         free_synth_event(event);
2302         return 0;
2303 }
2304
2305 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
2306 {
2307         struct synth_field *field;
2308         unsigned int i;
2309
2310         seq_printf(m, "%s\t", event->name);
2311
2312         for (i = 0; i < event->n_fields; i++) {
2313                 field = event->fields[i];
2314
2315                 /* parameter values */
2316                 seq_printf(m, "%s %s%s", field->type, field->name,
2317                            i == event->n_fields - 1 ? "" : "; ");
2318         }
2319
2320         seq_putc(m, '\n');
2321
2322         return 0;
2323 }
2324
2325 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
2326 {
2327         struct synth_event *event = to_synth_event(ev);
2328
2329         seq_printf(m, "s:%s/", event->class.system);
2330
2331         return __synth_event_show(m, event);
2332 }
2333
2334 static int synth_events_seq_show(struct seq_file *m, void *v)
2335 {
2336         struct dyn_event *ev = v;
2337
2338         if (!is_synth_event(ev))
2339                 return 0;
2340
2341         return __synth_event_show(m, to_synth_event(ev));
2342 }
2343
2344 static const struct seq_operations synth_events_seq_op = {
2345         .start  = dyn_event_seq_start,
2346         .next   = dyn_event_seq_next,
2347         .stop   = dyn_event_seq_stop,
2348         .show   = synth_events_seq_show,
2349 };
2350
2351 static int synth_events_open(struct inode *inode, struct file *file)
2352 {
2353         int ret;
2354
2355         ret = security_locked_down(LOCKDOWN_TRACEFS);
2356         if (ret)
2357                 return ret;
2358
2359         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
2360                 ret = dyn_events_release_all(&synth_event_ops);
2361                 if (ret < 0)
2362                         return ret;
2363         }
2364
2365         return seq_open(file, &synth_events_seq_op);
2366 }
2367
2368 static ssize_t synth_events_write(struct file *file,
2369                                   const char __user *buffer,
2370                                   size_t count, loff_t *ppos)
2371 {
2372         return trace_parse_run_command(file, buffer, count, ppos,
2373                                        create_or_delete_synth_event);
2374 }
2375
2376 static const struct file_operations synth_events_fops = {
2377         .open           = synth_events_open,
2378         .write          = synth_events_write,
2379         .read           = seq_read,
2380         .llseek         = seq_lseek,
2381         .release        = seq_release,
2382 };
2383
2384 static u64 hist_field_timestamp(struct hist_field *hist_field,
2385                                 struct tracing_map_elt *elt,
2386                                 struct ring_buffer_event *rbe,
2387                                 void *event)
2388 {
2389         struct hist_trigger_data *hist_data = hist_field->hist_data;
2390         struct trace_array *tr = hist_data->event_file->tr;
2391
2392         u64 ts = ring_buffer_event_time_stamp(rbe);
2393
2394         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
2395                 ts = ns2usecs(ts);
2396
2397         return ts;
2398 }
2399
2400 static u64 hist_field_cpu(struct hist_field *hist_field,
2401                           struct tracing_map_elt *elt,
2402                           struct ring_buffer_event *rbe,
2403                           void *event)
2404 {
2405         int cpu = smp_processor_id();
2406
2407         return cpu;
2408 }
2409
2410 /**
2411  * check_field_for_var_ref - Check if a VAR_REF field references a variable
2412  * @hist_field: The VAR_REF field to check
2413  * @var_data: The hist trigger that owns the variable
2414  * @var_idx: The trigger variable identifier
2415  *
2416  * Check the given VAR_REF field to see whether or not it references
2417  * the given variable associated with the given trigger.
2418  *
2419  * Return: The VAR_REF field if it does reference the variable, NULL if not
2420  */
2421 static struct hist_field *
2422 check_field_for_var_ref(struct hist_field *hist_field,
2423                         struct hist_trigger_data *var_data,
2424                         unsigned int var_idx)
2425 {
2426         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
2427
2428         if (hist_field && hist_field->var.idx == var_idx &&
2429             hist_field->var.hist_data == var_data)
2430                 return hist_field;
2431
2432         return NULL;
2433 }
2434
2435 /**
2436  * find_var_ref - Check if a trigger has a reference to a trigger variable
2437  * @hist_data: The hist trigger that might have a reference to the variable
2438  * @var_data: The hist trigger that owns the variable
2439  * @var_idx: The trigger variable identifier
2440  *
2441  * Check the list of var_refs[] on the first hist trigger to see
2442  * whether any of them are references to the variable on the second
2443  * trigger.
2444  *
2445  * Return: The VAR_REF field referencing the variable if so, NULL if not
2446  */
2447 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
2448                                        struct hist_trigger_data *var_data,
2449                                        unsigned int var_idx)
2450 {
2451         struct hist_field *hist_field;
2452         unsigned int i;
2453
2454         for (i = 0; i < hist_data->n_var_refs; i++) {
2455                 hist_field = hist_data->var_refs[i];
2456                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
2457                         return hist_field;
2458         }
2459
2460         return NULL;
2461 }
2462
2463 /**
2464  * find_any_var_ref - Check if there is a reference to a given trigger variable
2465  * @hist_data: The hist trigger
2466  * @var_idx: The trigger variable identifier
2467  *
2468  * Check to see whether the given variable is currently referenced by
2469  * any other trigger.
2470  *
2471  * The trigger the variable is defined on is explicitly excluded - the
2472  * assumption being that a self-reference doesn't prevent a trigger
2473  * from being removed.
2474  *
2475  * Return: The VAR_REF field referencing the variable if so, NULL if not
2476  */
2477 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
2478                                            unsigned int var_idx)
2479 {
2480         struct trace_array *tr = hist_data->event_file->tr;
2481         struct hist_field *found = NULL;
2482         struct hist_var_data *var_data;
2483
2484         list_for_each_entry(var_data, &tr->hist_vars, list) {
2485                 if (var_data->hist_data == hist_data)
2486                         continue;
2487                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
2488                 if (found)
2489                         break;
2490         }
2491
2492         return found;
2493 }
2494
2495 /**
2496  * check_var_refs - Check if there is a reference to any of trigger's variables
2497  * @hist_data: The hist trigger
2498  *
2499  * A trigger can define one or more variables.  If any one of them is
2500  * currently referenced by any other trigger, this function will
2501  * determine that.
2502
2503  * Typically used to determine whether or not a trigger can be removed
2504  * - if there are any references to a trigger's variables, it cannot.
2505  *
2506  * Return: True if there is a reference to any of trigger's variables
2507  */
2508 static bool check_var_refs(struct hist_trigger_data *hist_data)
2509 {
2510         struct hist_field *field;
2511         bool found = false;
2512         int i;
2513
2514         for_each_hist_field(i, hist_data) {
2515                 field = hist_data->fields[i];
2516                 if (field && field->flags & HIST_FIELD_FL_VAR) {
2517                         if (find_any_var_ref(hist_data, field->var.idx)) {
2518                                 found = true;
2519                                 break;
2520                         }
2521                 }
2522         }
2523
2524         return found;
2525 }
2526
2527 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
2528 {
2529         struct trace_array *tr = hist_data->event_file->tr;
2530         struct hist_var_data *var_data, *found = NULL;
2531
2532         list_for_each_entry(var_data, &tr->hist_vars, list) {
2533                 if (var_data->hist_data == hist_data) {
2534                         found = var_data;
2535                         break;
2536                 }
2537         }
2538
2539         return found;
2540 }
2541
2542 static bool field_has_hist_vars(struct hist_field *hist_field,
2543                                 unsigned int level)
2544 {
2545         int i;
2546
2547         if (level > 3)
2548                 return false;
2549
2550         if (!hist_field)
2551                 return false;
2552
2553         if (hist_field->flags & HIST_FIELD_FL_VAR ||
2554             hist_field->flags & HIST_FIELD_FL_VAR_REF)
2555                 return true;
2556
2557         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
2558                 struct hist_field *operand;
2559
2560                 operand = hist_field->operands[i];
2561                 if (field_has_hist_vars(operand, level + 1))
2562                         return true;
2563         }
2564
2565         return false;
2566 }
2567
2568 static bool has_hist_vars(struct hist_trigger_data *hist_data)
2569 {
2570         struct hist_field *hist_field;
2571         int i;
2572
2573         for_each_hist_field(i, hist_data) {
2574                 hist_field = hist_data->fields[i];
2575                 if (field_has_hist_vars(hist_field, 0))
2576                         return true;
2577         }
2578
2579         return false;
2580 }
2581
2582 static int save_hist_vars(struct hist_trigger_data *hist_data)
2583 {
2584         struct trace_array *tr = hist_data->event_file->tr;
2585         struct hist_var_data *var_data;
2586
2587         var_data = find_hist_vars(hist_data);
2588         if (var_data)
2589                 return 0;
2590
2591         if (tracing_check_open_get_tr(tr))
2592                 return -ENODEV;
2593
2594         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
2595         if (!var_data) {
2596                 trace_array_put(tr);
2597                 return -ENOMEM;
2598         }
2599
2600         var_data->hist_data = hist_data;
2601         list_add(&var_data->list, &tr->hist_vars);
2602
2603         return 0;
2604 }
2605
2606 static void remove_hist_vars(struct hist_trigger_data *hist_data)
2607 {
2608         struct trace_array *tr = hist_data->event_file->tr;
2609         struct hist_var_data *var_data;
2610
2611         var_data = find_hist_vars(hist_data);
2612         if (!var_data)
2613                 return;
2614
2615         if (WARN_ON(check_var_refs(hist_data)))
2616                 return;
2617
2618         list_del(&var_data->list);
2619
2620         kfree(var_data);
2621
2622         trace_array_put(tr);
2623 }
2624
2625 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
2626                                          const char *var_name)
2627 {
2628         struct hist_field *hist_field, *found = NULL;
2629         int i;
2630
2631         for_each_hist_field(i, hist_data) {
2632                 hist_field = hist_data->fields[i];
2633                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
2634                     strcmp(hist_field->var.name, var_name) == 0) {
2635                         found = hist_field;
2636                         break;
2637                 }
2638         }
2639
2640         return found;
2641 }
2642
2643 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
2644                                    struct trace_event_file *file,
2645                                    const char *var_name)
2646 {
2647         struct hist_trigger_data *test_data;
2648         struct event_trigger_data *test;
2649         struct hist_field *hist_field;
2650
2651         lockdep_assert_held(&event_mutex);
2652
2653         hist_field = find_var_field(hist_data, var_name);
2654         if (hist_field)
2655                 return hist_field;
2656
2657         list_for_each_entry(test, &file->triggers, list) {
2658                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2659                         test_data = test->private_data;
2660                         hist_field = find_var_field(test_data, var_name);
2661                         if (hist_field)
2662                                 return hist_field;
2663                 }
2664         }
2665
2666         return NULL;
2667 }
2668
2669 static struct trace_event_file *find_var_file(struct trace_array *tr,
2670                                               char *system,
2671                                               char *event_name,
2672                                               char *var_name)
2673 {
2674         struct hist_trigger_data *var_hist_data;
2675         struct hist_var_data *var_data;
2676         struct trace_event_file *file, *found = NULL;
2677
2678         if (system)
2679                 return find_event_file(tr, system, event_name);
2680
2681         list_for_each_entry(var_data, &tr->hist_vars, list) {
2682                 var_hist_data = var_data->hist_data;
2683                 file = var_hist_data->event_file;
2684                 if (file == found)
2685                         continue;
2686
2687                 if (find_var_field(var_hist_data, var_name)) {
2688                         if (found) {
2689                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
2690                                 return NULL;
2691                         }
2692
2693                         found = file;
2694                 }
2695         }
2696
2697         return found;
2698 }
2699
2700 static struct hist_field *find_file_var(struct trace_event_file *file,
2701                                         const char *var_name)
2702 {
2703         struct hist_trigger_data *test_data;
2704         struct event_trigger_data *test;
2705         struct hist_field *hist_field;
2706
2707         lockdep_assert_held(&event_mutex);
2708
2709         list_for_each_entry(test, &file->triggers, list) {
2710                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2711                         test_data = test->private_data;
2712                         hist_field = find_var_field(test_data, var_name);
2713                         if (hist_field)
2714                                 return hist_field;
2715                 }
2716         }
2717
2718         return NULL;
2719 }
2720
2721 static struct hist_field *
2722 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
2723 {
2724         struct trace_array *tr = hist_data->event_file->tr;
2725         struct hist_field *hist_field, *found = NULL;
2726         struct trace_event_file *file;
2727         unsigned int i;
2728
2729         for (i = 0; i < hist_data->n_actions; i++) {
2730                 struct action_data *data = hist_data->actions[i];
2731
2732                 if (data->handler == HANDLER_ONMATCH) {
2733                         char *system = data->match_data.event_system;
2734                         char *event_name = data->match_data.event;
2735
2736                         file = find_var_file(tr, system, event_name, var_name);
2737                         if (!file)
2738                                 continue;
2739                         hist_field = find_file_var(file, var_name);
2740                         if (hist_field) {
2741                                 if (found) {
2742                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
2743                                                  errpos(var_name));
2744                                         return ERR_PTR(-EINVAL);
2745                                 }
2746
2747                                 found = hist_field;
2748                         }
2749                 }
2750         }
2751         return found;
2752 }
2753
2754 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
2755                                          char *system,
2756                                          char *event_name,
2757                                          char *var_name)
2758 {
2759         struct trace_array *tr = hist_data->event_file->tr;
2760         struct hist_field *hist_field = NULL;
2761         struct trace_event_file *file;
2762
2763         if (!system || !event_name) {
2764                 hist_field = find_match_var(hist_data, var_name);
2765                 if (IS_ERR(hist_field))
2766                         return NULL;
2767                 if (hist_field)
2768                         return hist_field;
2769         }
2770
2771         file = find_var_file(tr, system, event_name, var_name);
2772         if (!file)
2773                 return NULL;
2774
2775         hist_field = find_file_var(file, var_name);
2776
2777         return hist_field;
2778 }
2779
2780 static u64 hist_field_var_ref(struct hist_field *hist_field,
2781                               struct tracing_map_elt *elt,
2782                               struct ring_buffer_event *rbe,
2783                               void *event)
2784 {
2785         struct hist_elt_data *elt_data;
2786         u64 var_val = 0;
2787
2788         if (WARN_ON_ONCE(!elt))
2789                 return var_val;
2790
2791         elt_data = elt->private_data;
2792         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
2793
2794         return var_val;
2795 }
2796
2797 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
2798                              u64 *var_ref_vals, bool self)
2799 {
2800         struct hist_trigger_data *var_data;
2801         struct tracing_map_elt *var_elt;
2802         struct hist_field *hist_field;
2803         unsigned int i, var_idx;
2804         bool resolved = true;
2805         u64 var_val = 0;
2806
2807         for (i = 0; i < hist_data->n_var_refs; i++) {
2808                 hist_field = hist_data->var_refs[i];
2809                 var_idx = hist_field->var.idx;
2810                 var_data = hist_field->var.hist_data;
2811
2812                 if (var_data == NULL) {
2813                         resolved = false;
2814                         break;
2815                 }
2816
2817                 if ((self && var_data != hist_data) ||
2818                     (!self && var_data == hist_data))
2819                         continue;
2820
2821                 var_elt = tracing_map_lookup(var_data->map, key);
2822                 if (!var_elt) {
2823                         resolved = false;
2824                         break;
2825                 }
2826
2827                 if (!tracing_map_var_set(var_elt, var_idx)) {
2828                         resolved = false;
2829                         break;
2830                 }
2831
2832                 if (self || !hist_field->read_once)
2833                         var_val = tracing_map_read_var(var_elt, var_idx);
2834                 else
2835                         var_val = tracing_map_read_var_once(var_elt, var_idx);
2836
2837                 var_ref_vals[i] = var_val;
2838         }
2839
2840         return resolved;
2841 }
2842
2843 static const char *hist_field_name(struct hist_field *field,
2844                                    unsigned int level)
2845 {
2846         const char *field_name = "";
2847
2848         if (level > 1)
2849                 return field_name;
2850
2851         if (field->field)
2852                 field_name = field->field->name;
2853         else if (field->flags & HIST_FIELD_FL_LOG2 ||
2854                  field->flags & HIST_FIELD_FL_ALIAS)
2855                 field_name = hist_field_name(field->operands[0], ++level);
2856         else if (field->flags & HIST_FIELD_FL_CPU)
2857                 field_name = "cpu";
2858         else if (field->flags & HIST_FIELD_FL_EXPR ||
2859                  field->flags & HIST_FIELD_FL_VAR_REF) {
2860                 if (field->system) {
2861                         static char full_name[MAX_FILTER_STR_VAL];
2862
2863                         strcat(full_name, field->system);
2864                         strcat(full_name, ".");
2865                         strcat(full_name, field->event_name);
2866                         strcat(full_name, ".");
2867                         strcat(full_name, field->name);
2868                         field_name = full_name;
2869                 } else
2870                         field_name = field->name;
2871         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
2872                 field_name = "common_timestamp";
2873
2874         if (field_name == NULL)
2875                 field_name = "";
2876
2877         return field_name;
2878 }
2879
2880 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
2881 {
2882         hist_field_fn_t fn = NULL;
2883
2884         switch (field_size) {
2885         case 8:
2886                 if (field_is_signed)
2887                         fn = hist_field_s64;
2888                 else
2889                         fn = hist_field_u64;
2890                 break;
2891         case 4:
2892                 if (field_is_signed)
2893                         fn = hist_field_s32;
2894                 else
2895                         fn = hist_field_u32;
2896                 break;
2897         case 2:
2898                 if (field_is_signed)
2899                         fn = hist_field_s16;
2900                 else
2901                         fn = hist_field_u16;
2902                 break;
2903         case 1:
2904                 if (field_is_signed)
2905                         fn = hist_field_s8;
2906                 else
2907                         fn = hist_field_u8;
2908                 break;
2909         }
2910
2911         return fn;
2912 }
2913
2914 static int parse_map_size(char *str)
2915 {
2916         unsigned long size, map_bits;
2917         int ret;
2918
2919         ret = kstrtoul(str, 0, &size);
2920         if (ret)
2921                 goto out;
2922
2923         map_bits = ilog2(roundup_pow_of_two(size));
2924         if (map_bits < TRACING_MAP_BITS_MIN ||
2925             map_bits > TRACING_MAP_BITS_MAX)
2926                 ret = -EINVAL;
2927         else
2928                 ret = map_bits;
2929  out:
2930         return ret;
2931 }
2932
2933 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2934 {
2935         unsigned int i;
2936
2937         if (!attrs)
2938                 return;
2939
2940         for (i = 0; i < attrs->n_assignments; i++)
2941                 kfree(attrs->assignment_str[i]);
2942
2943         for (i = 0; i < attrs->n_actions; i++)
2944                 kfree(attrs->action_str[i]);
2945
2946         kfree(attrs->name);
2947         kfree(attrs->sort_key_str);
2948         kfree(attrs->keys_str);
2949         kfree(attrs->vals_str);
2950         kfree(attrs->clock);
2951         kfree(attrs);
2952 }
2953
2954 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2955 {
2956         int ret = -EINVAL;
2957
2958         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2959                 return ret;
2960
2961         if ((str_has_prefix(str, "onmatch(")) ||
2962             (str_has_prefix(str, "onmax(")) ||
2963             (str_has_prefix(str, "onchange("))) {
2964                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2965                 if (!attrs->action_str[attrs->n_actions]) {
2966                         ret = -ENOMEM;
2967                         return ret;
2968                 }
2969                 attrs->n_actions++;
2970                 ret = 0;
2971         }
2972         return ret;
2973 }
2974
2975 static int parse_assignment(struct trace_array *tr,
2976                             char *str, struct hist_trigger_attrs *attrs)
2977 {
2978         int len, ret = 0;
2979
2980         if ((len = str_has_prefix(str, "key=")) ||
2981             (len = str_has_prefix(str, "keys="))) {
2982                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
2983                 if (!attrs->keys_str) {
2984                         ret = -ENOMEM;
2985                         goto out;
2986                 }
2987         } else if ((len = str_has_prefix(str, "val=")) ||
2988                    (len = str_has_prefix(str, "vals=")) ||
2989                    (len = str_has_prefix(str, "values="))) {
2990                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
2991                 if (!attrs->vals_str) {
2992                         ret = -ENOMEM;
2993                         goto out;
2994                 }
2995         } else if ((len = str_has_prefix(str, "sort="))) {
2996                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
2997                 if (!attrs->sort_key_str) {
2998                         ret = -ENOMEM;
2999                         goto out;
3000                 }
3001         } else if (str_has_prefix(str, "name=")) {
3002                 attrs->name = kstrdup(str, GFP_KERNEL);
3003                 if (!attrs->name) {
3004                         ret = -ENOMEM;
3005                         goto out;
3006                 }
3007         } else if ((len = str_has_prefix(str, "clock="))) {
3008                 str += len;
3009
3010                 str = strstrip(str);
3011                 attrs->clock = kstrdup(str, GFP_KERNEL);
3012                 if (!attrs->clock) {
3013                         ret = -ENOMEM;
3014                         goto out;
3015                 }
3016         } else if ((len = str_has_prefix(str, "size="))) {
3017                 int map_bits = parse_map_size(str + len);
3018
3019                 if (map_bits < 0) {
3020                         ret = map_bits;
3021                         goto out;
3022                 }
3023                 attrs->map_bits = map_bits;
3024         } else {
3025                 char *assignment;
3026
3027                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
3028                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
3029                         ret = -EINVAL;
3030                         goto out;
3031                 }
3032
3033                 assignment = kstrdup(str, GFP_KERNEL);
3034                 if (!assignment) {
3035                         ret = -ENOMEM;
3036                         goto out;
3037                 }
3038
3039                 attrs->assignment_str[attrs->n_assignments++] = assignment;
3040         }
3041  out:
3042         return ret;
3043 }
3044
3045 static struct hist_trigger_attrs *
3046 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
3047 {
3048         struct hist_trigger_attrs *attrs;
3049         int ret = 0;
3050
3051         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3052         if (!attrs)
3053                 return ERR_PTR(-ENOMEM);
3054
3055         while (trigger_str) {
3056                 char *str = strsep(&trigger_str, ":");
3057                 char *rhs;
3058
3059                 rhs = strchr(str, '=');
3060                 if (rhs) {
3061                         if (!strlen(++rhs)) {
3062                                 ret = -EINVAL;
3063                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
3064                                 goto free;
3065                         }
3066                         ret = parse_assignment(tr, str, attrs);
3067                         if (ret)
3068                                 goto free;
3069                 } else if (strcmp(str, "pause") == 0)
3070                         attrs->pause = true;
3071                 else if ((strcmp(str, "cont") == 0) ||
3072                          (strcmp(str, "continue") == 0))
3073                         attrs->cont = true;
3074                 else if (strcmp(str, "clear") == 0)
3075                         attrs->clear = true;
3076                 else {
3077                         ret = parse_action(str, attrs);
3078                         if (ret)
3079                                 goto free;
3080                 }
3081         }
3082
3083         if (!attrs->keys_str) {
3084                 ret = -EINVAL;
3085                 goto free;
3086         }
3087
3088         if (!attrs->clock) {
3089                 attrs->clock = kstrdup("global", GFP_KERNEL);
3090                 if (!attrs->clock) {
3091                         ret = -ENOMEM;
3092                         goto free;
3093                 }
3094         }
3095
3096         return attrs;
3097  free:
3098         destroy_hist_trigger_attrs(attrs);
3099
3100         return ERR_PTR(ret);
3101 }
3102
3103 static inline void save_comm(char *comm, struct task_struct *task)
3104 {
3105         if (!task->pid) {
3106                 strcpy(comm, "<idle>");
3107                 return;
3108         }
3109
3110         if (WARN_ON_ONCE(task->pid < 0)) {
3111                 strcpy(comm, "<XXX>");
3112                 return;
3113         }
3114
3115         strncpy(comm, task->comm, TASK_COMM_LEN);
3116 }
3117
3118 static void hist_elt_data_free(struct hist_elt_data *elt_data)
3119 {
3120         unsigned int i;
3121
3122         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
3123                 kfree(elt_data->field_var_str[i]);
3124
3125         kfree(elt_data->comm);
3126         kfree(elt_data);
3127 }
3128
3129 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
3130 {
3131         struct hist_elt_data *elt_data = elt->private_data;
3132
3133         hist_elt_data_free(elt_data);
3134 }
3135
3136 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
3137 {
3138         struct hist_trigger_data *hist_data = elt->map->private_data;
3139         unsigned int size = TASK_COMM_LEN;
3140         struct hist_elt_data *elt_data;
3141         struct hist_field *key_field;
3142         unsigned int i, n_str;
3143
3144         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
3145         if (!elt_data)
3146                 return -ENOMEM;
3147
3148         for_each_hist_key_field(i, hist_data) {
3149                 key_field = hist_data->fields[i];
3150
3151                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
3152                         elt_data->comm = kzalloc(size, GFP_KERNEL);
3153                         if (!elt_data->comm) {
3154                                 kfree(elt_data);
3155                                 return -ENOMEM;
3156                         }
3157                         break;
3158                 }
3159         }
3160
3161         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
3162
3163         size = STR_VAR_LEN_MAX;
3164
3165         for (i = 0; i < n_str; i++) {
3166                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
3167                 if (!elt_data->field_var_str[i]) {
3168                         hist_elt_data_free(elt_data);
3169                         return -ENOMEM;
3170                 }
3171         }
3172
3173         elt->private_data = elt_data;
3174
3175         return 0;
3176 }
3177
3178 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
3179 {
3180         struct hist_elt_data *elt_data = elt->private_data;
3181
3182         if (elt_data->comm)
3183                 save_comm(elt_data->comm, current);
3184 }
3185
3186 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
3187         .elt_alloc      = hist_trigger_elt_data_alloc,
3188         .elt_free       = hist_trigger_elt_data_free,
3189         .elt_init       = hist_trigger_elt_data_init,
3190 };
3191
3192 static const char *get_hist_field_flags(struct hist_field *hist_field)
3193 {
3194         const char *flags_str = NULL;
3195
3196         if (hist_field->flags & HIST_FIELD_FL_HEX)
3197                 flags_str = "hex";
3198         else if (hist_field->flags & HIST_FIELD_FL_SYM)
3199                 flags_str = "sym";
3200         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
3201                 flags_str = "sym-offset";
3202         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
3203                 flags_str = "execname";
3204         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
3205                 flags_str = "syscall";
3206         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
3207                 flags_str = "log2";
3208         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
3209                 flags_str = "usecs";
3210
3211         return flags_str;
3212 }
3213
3214 static void expr_field_str(struct hist_field *field, char *expr)
3215 {
3216         if (field->flags & HIST_FIELD_FL_VAR_REF)
3217                 strcat(expr, "$");
3218
3219         strcat(expr, hist_field_name(field, 0));
3220
3221         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
3222                 const char *flags_str = get_hist_field_flags(field);
3223
3224                 if (flags_str) {
3225                         strcat(expr, ".");
3226                         strcat(expr, flags_str);
3227                 }
3228         }
3229 }
3230
3231 static char *expr_str(struct hist_field *field, unsigned int level)
3232 {
3233         char *expr;
3234
3235         if (level > 1)
3236                 return NULL;
3237
3238         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3239         if (!expr)
3240                 return NULL;
3241
3242         if (!field->operands[0]) {
3243                 expr_field_str(field, expr);
3244                 return expr;
3245         }
3246
3247         if (field->operator == FIELD_OP_UNARY_MINUS) {
3248                 char *subexpr;
3249
3250                 strcat(expr, "-(");
3251                 subexpr = expr_str(field->operands[0], ++level);
3252                 if (!subexpr) {
3253                         kfree(expr);
3254                         return NULL;
3255                 }
3256                 strcat(expr, subexpr);
3257                 strcat(expr, ")");
3258
3259                 kfree(subexpr);
3260
3261                 return expr;
3262         }
3263
3264         expr_field_str(field->operands[0], expr);
3265
3266         switch (field->operator) {
3267         case FIELD_OP_MINUS:
3268                 strcat(expr, "-");
3269                 break;
3270         case FIELD_OP_PLUS:
3271                 strcat(expr, "+");
3272                 break;
3273         default:
3274                 kfree(expr);
3275                 return NULL;
3276         }
3277
3278         expr_field_str(field->operands[1], expr);
3279
3280         return expr;
3281 }
3282
3283 static int contains_operator(char *str)
3284 {
3285         enum field_op_id field_op = FIELD_OP_NONE;
3286         char *op;
3287
3288         op = strpbrk(str, "+-");
3289         if (!op)
3290                 return FIELD_OP_NONE;
3291
3292         switch (*op) {
3293         case '-':
3294                 if (*str == '-')
3295                         field_op = FIELD_OP_UNARY_MINUS;
3296                 else
3297                         field_op = FIELD_OP_MINUS;
3298                 break;
3299         case '+':
3300                 field_op = FIELD_OP_PLUS;
3301                 break;
3302         default:
3303                 break;
3304         }
3305
3306         return field_op;
3307 }
3308
3309 static void get_hist_field(struct hist_field *hist_field)
3310 {
3311         hist_field->ref++;
3312 }
3313
3314 static void __destroy_hist_field(struct hist_field *hist_field)
3315 {
3316         if (--hist_field->ref > 1)
3317                 return;
3318
3319         kfree(hist_field->var.name);
3320         kfree(hist_field->name);
3321         kfree(hist_field->type);
3322
3323         kfree(hist_field);
3324 }
3325
3326 static void destroy_hist_field(struct hist_field *hist_field,
3327                                unsigned int level)
3328 {
3329         unsigned int i;
3330
3331         if (level > 3)
3332                 return;
3333
3334         if (!hist_field)
3335                 return;
3336
3337         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
3338                 return; /* var refs will be destroyed separately */
3339
3340         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
3341                 destroy_hist_field(hist_field->operands[i], level + 1);
3342
3343         __destroy_hist_field(hist_field);
3344 }
3345
3346 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
3347                                             struct ftrace_event_field *field,
3348                                             unsigned long flags,
3349                                             char *var_name)
3350 {
3351         struct hist_field *hist_field;
3352
3353         if (field && is_function_field(field))
3354                 return NULL;
3355
3356         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3357         if (!hist_field)
3358                 return NULL;
3359
3360         hist_field->ref = 1;
3361
3362         hist_field->hist_data = hist_data;
3363
3364         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
3365                 goto out; /* caller will populate */
3366
3367         if (flags & HIST_FIELD_FL_VAR_REF) {
3368                 hist_field->fn = hist_field_var_ref;
3369                 goto out;
3370         }
3371
3372         if (flags & HIST_FIELD_FL_HITCOUNT) {
3373                 hist_field->fn = hist_field_counter;
3374                 hist_field->size = sizeof(u64);
3375                 hist_field->type = kstrdup("u64", GFP_KERNEL);
3376                 if (!hist_field->type)
3377                         goto free;
3378                 goto out;
3379         }
3380
3381         if (flags & HIST_FIELD_FL_STACKTRACE) {
3382                 hist_field->fn = hist_field_none;
3383                 goto out;
3384         }
3385
3386         if (flags & HIST_FIELD_FL_LOG2) {
3387                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
3388                 hist_field->fn = hist_field_log2;
3389                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
3390                 hist_field->size = hist_field->operands[0]->size;
3391                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
3392                 if (!hist_field->type)
3393                         goto free;
3394                 goto out;
3395         }
3396
3397         if (flags & HIST_FIELD_FL_TIMESTAMP) {
3398                 hist_field->fn = hist_field_timestamp;
3399                 hist_field->size = sizeof(u64);
3400                 hist_field->type = kstrdup("u64", GFP_KERNEL);
3401                 if (!hist_field->type)
3402                         goto free;
3403                 goto out;
3404         }
3405
3406         if (flags & HIST_FIELD_FL_CPU) {
3407                 hist_field->fn = hist_field_cpu;
3408                 hist_field->size = sizeof(int);
3409                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
3410                 if (!hist_field->type)
3411                         goto free;
3412                 goto out;
3413         }
3414
3415         if (WARN_ON_ONCE(!field))
3416                 goto out;
3417
3418         if (is_string_field(field)) {
3419                 flags |= HIST_FIELD_FL_STRING;
3420
3421                 hist_field->size = MAX_FILTER_STR_VAL;
3422                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
3423                 if (!hist_field->type)
3424                         goto free;
3425
3426                 if (field->filter_type == FILTER_STATIC_STRING)
3427                         hist_field->fn = hist_field_string;
3428                 else if (field->filter_type == FILTER_DYN_STRING)
3429                         hist_field->fn = hist_field_dynstring;
3430                 else
3431                         hist_field->fn = hist_field_pstring;
3432         } else {
3433                 hist_field->size = field->size;
3434                 hist_field->is_signed = field->is_signed;
3435                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
3436                 if (!hist_field->type)
3437                         goto free;
3438
3439                 hist_field->fn = select_value_fn(field->size,
3440                                                  field->is_signed);
3441                 if (!hist_field->fn) {
3442                         destroy_hist_field(hist_field, 0);
3443                         return NULL;
3444                 }
3445         }
3446  out:
3447         hist_field->field = field;
3448         hist_field->flags = flags;
3449
3450         if (var_name) {
3451                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
3452                 if (!hist_field->var.name)
3453                         goto free;
3454         }
3455
3456         return hist_field;
3457  free:
3458         destroy_hist_field(hist_field, 0);
3459         return NULL;
3460 }
3461
3462 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
3463 {
3464         unsigned int i;
3465
3466         for (i = 0; i < HIST_FIELDS_MAX; i++) {
3467                 if (hist_data->fields[i]) {
3468                         destroy_hist_field(hist_data->fields[i], 0);
3469                         hist_data->fields[i] = NULL;
3470                 }
3471         }
3472
3473         for (i = 0; i < hist_data->n_var_refs; i++) {
3474                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
3475                 __destroy_hist_field(hist_data->var_refs[i]);
3476                 hist_data->var_refs[i] = NULL;
3477         }
3478 }
3479
3480 static int init_var_ref(struct hist_field *ref_field,
3481                         struct hist_field *var_field,
3482                         char *system, char *event_name)
3483 {
3484         int err = 0;
3485
3486         ref_field->var.idx = var_field->var.idx;
3487         ref_field->var.hist_data = var_field->hist_data;
3488         ref_field->size = var_field->size;
3489         ref_field->is_signed = var_field->is_signed;
3490         ref_field->flags |= var_field->flags &
3491                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3492
3493         if (system) {
3494                 ref_field->system = kstrdup(system, GFP_KERNEL);
3495                 if (!ref_field->system)
3496                         return -ENOMEM;
3497         }
3498
3499         if (event_name) {
3500                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
3501                 if (!ref_field->event_name) {
3502                         err = -ENOMEM;
3503                         goto free;
3504                 }
3505         }
3506
3507         if (var_field->var.name) {
3508                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
3509                 if (!ref_field->name) {
3510                         err = -ENOMEM;
3511                         goto free;
3512                 }
3513         } else if (var_field->name) {
3514                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
3515                 if (!ref_field->name) {
3516                         err = -ENOMEM;
3517                         goto free;
3518                 }
3519         }
3520
3521         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
3522         if (!ref_field->type) {
3523                 err = -ENOMEM;
3524                 goto free;
3525         }
3526  out:
3527         return err;
3528  free:
3529         kfree(ref_field->system);
3530         kfree(ref_field->event_name);
3531         kfree(ref_field->name);
3532
3533         goto out;
3534 }
3535
3536 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
3537                             struct hist_field *var_field)
3538 {
3539         struct hist_field *ref_field;
3540         int i;
3541
3542         for (i = 0; i < hist_data->n_var_refs; i++) {
3543                 ref_field = hist_data->var_refs[i];
3544                 if (ref_field->var.idx == var_field->var.idx &&
3545                     ref_field->var.hist_data == var_field->hist_data)
3546                         return i;
3547         }
3548
3549         return -ENOENT;
3550 }
3551
3552 /**
3553  * create_var_ref - Create a variable reference and attach it to trigger
3554  * @hist_data: The trigger that will be referencing the variable
3555  * @var_field: The VAR field to create a reference to
3556  * @system: The optional system string
3557  * @event_name: The optional event_name string
3558  *
3559  * Given a variable hist_field, create a VAR_REF hist_field that
3560  * represents a reference to it.
3561  *
3562  * This function also adds the reference to the trigger that
3563  * now references the variable.
3564  *
3565  * Return: The VAR_REF field if successful, NULL if not
3566  */
3567 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
3568                                          struct hist_field *var_field,
3569                                          char *system, char *event_name)
3570 {
3571         unsigned long flags = HIST_FIELD_FL_VAR_REF;
3572         struct hist_field *ref_field;
3573         int i;
3574
3575         /* Check if the variable already exists */
3576         for (i = 0; i < hist_data->n_var_refs; i++) {
3577                 ref_field = hist_data->var_refs[i];
3578                 if (ref_field->var.idx == var_field->var.idx &&
3579                     ref_field->var.hist_data == var_field->hist_data) {
3580                         get_hist_field(ref_field);
3581                         return ref_field;
3582                 }
3583         }
3584
3585         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
3586         if (ref_field) {
3587                 if (init_var_ref(ref_field, var_field, system, event_name)) {
3588                         destroy_hist_field(ref_field, 0);
3589                         return NULL;
3590                 }
3591
3592                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3593                 ref_field->var_ref_idx = hist_data->n_var_refs++;
3594         }
3595
3596         return ref_field;
3597 }
3598
3599 static bool is_var_ref(char *var_name)
3600 {
3601         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
3602                 return false;
3603
3604         return true;
3605 }
3606
3607 static char *field_name_from_var(struct hist_trigger_data *hist_data,
3608                                  char *var_name)
3609 {
3610         char *name, *field;
3611         unsigned int i;
3612
3613         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3614                 name = hist_data->attrs->var_defs.name[i];
3615
3616                 if (strcmp(var_name, name) == 0) {
3617                         field = hist_data->attrs->var_defs.expr[i];
3618                         if (contains_operator(field) || is_var_ref(field))
3619                                 continue;
3620                         return field;
3621                 }
3622         }
3623
3624         return NULL;
3625 }
3626
3627 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
3628                                  char *system, char *event_name,
3629                                  char *var_name)
3630 {
3631         struct trace_event_call *call;
3632
3633         if (system && event_name) {
3634                 call = hist_data->event_file->event_call;
3635
3636                 if (strcmp(system, call->class->system) != 0)
3637                         return NULL;
3638
3639                 if (strcmp(event_name, trace_event_name(call)) != 0)
3640                         return NULL;
3641         }
3642
3643         if (!!system != !!event_name)
3644                 return NULL;
3645
3646         if (!is_var_ref(var_name))
3647                 return NULL;
3648
3649         var_name++;
3650
3651         return field_name_from_var(hist_data, var_name);
3652 }
3653
3654 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
3655                                         char *system, char *event_name,
3656                                         char *var_name)
3657 {
3658         struct hist_field *var_field = NULL, *ref_field = NULL;
3659         struct trace_array *tr = hist_data->event_file->tr;
3660
3661         if (!is_var_ref(var_name))
3662                 return NULL;
3663
3664         var_name++;
3665
3666         var_field = find_event_var(hist_data, system, event_name, var_name);
3667         if (var_field)
3668                 ref_field = create_var_ref(hist_data, var_field,
3669                                            system, event_name);
3670
3671         if (!ref_field)
3672                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
3673
3674         return ref_field;
3675 }
3676
3677 static struct ftrace_event_field *
3678 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
3679             char *field_str, unsigned long *flags)
3680 {
3681         struct ftrace_event_field *field = NULL;
3682         char *field_name, *modifier, *str;
3683         struct trace_array *tr = file->tr;
3684
3685         modifier = str = kstrdup(field_str, GFP_KERNEL);
3686         if (!modifier)
3687                 return ERR_PTR(-ENOMEM);
3688
3689         field_name = strsep(&modifier, ".");
3690         if (modifier) {
3691                 if (strcmp(modifier, "hex") == 0)
3692                         *flags |= HIST_FIELD_FL_HEX;
3693                 else if (strcmp(modifier, "sym") == 0)
3694                         *flags |= HIST_FIELD_FL_SYM;
3695                 else if (strcmp(modifier, "sym-offset") == 0)
3696                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
3697                 else if ((strcmp(modifier, "execname") == 0) &&
3698                          (strcmp(field_name, "common_pid") == 0))
3699                         *flags |= HIST_FIELD_FL_EXECNAME;
3700                 else if (strcmp(modifier, "syscall") == 0)
3701                         *flags |= HIST_FIELD_FL_SYSCALL;
3702                 else if (strcmp(modifier, "log2") == 0)
3703                         *flags |= HIST_FIELD_FL_LOG2;
3704                 else if (strcmp(modifier, "usecs") == 0)
3705                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
3706                 else {
3707                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
3708                         field = ERR_PTR(-EINVAL);
3709                         goto out;
3710                 }
3711         }
3712
3713         if (strcmp(field_name, "common_timestamp") == 0) {
3714                 *flags |= HIST_FIELD_FL_TIMESTAMP;
3715                 hist_data->enable_timestamps = true;
3716                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
3717                         hist_data->attrs->ts_in_usecs = true;
3718         } else if (strcmp(field_name, "cpu") == 0)
3719                 *flags |= HIST_FIELD_FL_CPU;
3720         else {
3721                 field = trace_find_event_field(file->event_call, field_name);
3722                 if (!field || !field->size) {
3723                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
3724                         field = ERR_PTR(-EINVAL);
3725                         goto out;
3726                 }
3727         }
3728  out:
3729         kfree(str);
3730
3731         return field;
3732 }
3733
3734 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
3735                                        struct hist_field *var_ref,
3736                                        char *var_name)
3737 {
3738         struct hist_field *alias = NULL;
3739         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
3740
3741         alias = create_hist_field(hist_data, NULL, flags, var_name);
3742         if (!alias)
3743                 return NULL;
3744
3745         alias->fn = var_ref->fn;
3746         alias->operands[0] = var_ref;
3747
3748         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
3749                 destroy_hist_field(alias, 0);
3750                 return NULL;
3751         }
3752
3753         alias->var_ref_idx = var_ref->var_ref_idx;
3754
3755         return alias;
3756 }
3757
3758 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
3759                                      struct trace_event_file *file, char *str,
3760                                      unsigned long *flags, char *var_name)
3761 {
3762         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
3763         struct ftrace_event_field *field = NULL;
3764         struct hist_field *hist_field = NULL;
3765         int ret = 0;
3766
3767         s = strchr(str, '.');
3768         if (s) {
3769                 s = strchr(++s, '.');
3770                 if (s) {
3771                         ref_system = strsep(&str, ".");
3772                         if (!str) {
3773                                 ret = -EINVAL;
3774                                 goto out;
3775                         }
3776                         ref_event = strsep(&str, ".");
3777                         if (!str) {
3778                                 ret = -EINVAL;
3779                                 goto out;
3780                         }
3781                         ref_var = str;
3782                 }
3783         }
3784
3785         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
3786         if (!s) {
3787                 hist_field = parse_var_ref(hist_data, ref_system,
3788                                            ref_event, ref_var);
3789                 if (hist_field) {
3790                         if (var_name) {
3791                                 hist_field = create_alias(hist_data, hist_field, var_name);
3792                                 if (!hist_field) {
3793                                         ret = -ENOMEM;
3794                                         goto out;
3795                                 }
3796                         }
3797                         return hist_field;
3798                 }
3799         } else
3800                 str = s;
3801
3802         field = parse_field(hist_data, file, str, flags);
3803         if (IS_ERR(field)) {
3804                 ret = PTR_ERR(field);
3805                 goto out;
3806         }
3807
3808         hist_field = create_hist_field(hist_data, field, *flags, var_name);
3809         if (!hist_field) {
3810                 ret = -ENOMEM;
3811                 goto out;
3812         }
3813
3814         return hist_field;
3815  out:
3816         return ERR_PTR(ret);
3817 }
3818
3819 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3820                                      struct trace_event_file *file,
3821                                      char *str, unsigned long flags,
3822                                      char *var_name, unsigned int level);
3823
3824 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
3825                                       struct trace_event_file *file,
3826                                       char *str, unsigned long flags,
3827                                       char *var_name, unsigned int level)
3828 {
3829         struct hist_field *operand1, *expr = NULL;
3830         unsigned long operand_flags;
3831         int ret = 0;
3832         char *s;
3833
3834         /* we support only -(xxx) i.e. explicit parens required */
3835
3836         if (level > 3) {
3837                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3838                 ret = -EINVAL;
3839                 goto free;
3840         }
3841
3842         str++; /* skip leading '-' */
3843
3844         s = strchr(str, '(');
3845         if (s)
3846                 str++;
3847         else {
3848                 ret = -EINVAL;
3849                 goto free;
3850         }
3851
3852         s = strrchr(str, ')');
3853         if (s)
3854                 *s = '\0';
3855         else {
3856                 ret = -EINVAL; /* no closing ')' */
3857                 goto free;
3858         }
3859
3860         flags |= HIST_FIELD_FL_EXPR;
3861         expr = create_hist_field(hist_data, NULL, flags, var_name);
3862         if (!expr) {
3863                 ret = -ENOMEM;
3864                 goto free;
3865         }
3866
3867         operand_flags = 0;
3868         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3869         if (IS_ERR(operand1)) {
3870                 ret = PTR_ERR(operand1);
3871                 goto free;
3872         }
3873
3874         expr->flags |= operand1->flags &
3875                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3876         expr->fn = hist_field_unary_minus;
3877         expr->operands[0] = operand1;
3878         expr->operator = FIELD_OP_UNARY_MINUS;
3879         expr->name = expr_str(expr, 0);
3880         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3881         if (!expr->type) {
3882                 ret = -ENOMEM;
3883                 goto free;
3884         }
3885
3886         return expr;
3887  free:
3888         destroy_hist_field(expr, 0);
3889         return ERR_PTR(ret);
3890 }
3891
3892 static int check_expr_operands(struct trace_array *tr,
3893                                struct hist_field *operand1,
3894                                struct hist_field *operand2)
3895 {
3896         unsigned long operand1_flags = operand1->flags;
3897         unsigned long operand2_flags = operand2->flags;
3898
3899         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
3900             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
3901                 struct hist_field *var;
3902
3903                 var = find_var_field(operand1->var.hist_data, operand1->name);
3904                 if (!var)
3905                         return -EINVAL;
3906                 operand1_flags = var->flags;
3907         }
3908
3909         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
3910             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
3911                 struct hist_field *var;
3912
3913                 var = find_var_field(operand2->var.hist_data, operand2->name);
3914                 if (!var)
3915                         return -EINVAL;
3916                 operand2_flags = var->flags;
3917         }
3918
3919         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
3920             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
3921                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
3922                 return -EINVAL;
3923         }
3924
3925         return 0;
3926 }
3927
3928 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3929                                      struct trace_event_file *file,
3930                                      char *str, unsigned long flags,
3931                                      char *var_name, unsigned int level)
3932 {
3933         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
3934         unsigned long operand_flags;
3935         int field_op, ret = -EINVAL;
3936         char *sep, *operand1_str;
3937
3938         if (level > 3) {
3939                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3940                 return ERR_PTR(-EINVAL);
3941         }
3942
3943         field_op = contains_operator(str);
3944
3945         if (field_op == FIELD_OP_NONE)
3946                 return parse_atom(hist_data, file, str, &flags, var_name);
3947
3948         if (field_op == FIELD_OP_UNARY_MINUS)
3949                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
3950
3951         switch (field_op) {
3952         case FIELD_OP_MINUS:
3953                 sep = "-";
3954                 break;
3955         case FIELD_OP_PLUS:
3956                 sep = "+";
3957                 break;
3958         default:
3959                 goto free;
3960         }
3961
3962         operand1_str = strsep(&str, sep);
3963         if (!operand1_str || !str)
3964                 goto free;
3965
3966         operand_flags = 0;
3967         operand1 = parse_atom(hist_data, file, operand1_str,
3968                               &operand_flags, NULL);
3969         if (IS_ERR(operand1)) {
3970                 ret = PTR_ERR(operand1);
3971                 operand1 = NULL;
3972                 goto free;
3973         }
3974
3975         /* rest of string could be another expression e.g. b+c in a+b+c */
3976         operand_flags = 0;
3977         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3978         if (IS_ERR(operand2)) {
3979                 ret = PTR_ERR(operand2);
3980                 operand2 = NULL;
3981                 goto free;
3982         }
3983
3984         ret = check_expr_operands(file->tr, operand1, operand2);
3985         if (ret)
3986                 goto free;
3987
3988         flags |= HIST_FIELD_FL_EXPR;
3989
3990         flags |= operand1->flags &
3991                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3992
3993         expr = create_hist_field(hist_data, NULL, flags, var_name);
3994         if (!expr) {
3995                 ret = -ENOMEM;
3996                 goto free;
3997         }
3998
3999         operand1->read_once = true;
4000         operand2->read_once = true;
4001
4002         expr->operands[0] = operand1;
4003         expr->operands[1] = operand2;
4004         expr->operator = field_op;
4005         expr->name = expr_str(expr, 0);
4006         expr->type = kstrdup(operand1->type, GFP_KERNEL);
4007         if (!expr->type) {
4008                 ret = -ENOMEM;
4009                 goto free;
4010         }
4011
4012         switch (field_op) {
4013         case FIELD_OP_MINUS:
4014                 expr->fn = hist_field_minus;
4015                 break;
4016         case FIELD_OP_PLUS:
4017                 expr->fn = hist_field_plus;
4018                 break;
4019         default:
4020                 ret = -EINVAL;
4021                 goto free;
4022         }
4023
4024         return expr;
4025  free:
4026         destroy_hist_field(operand1, 0);
4027         destroy_hist_field(operand2, 0);
4028         destroy_hist_field(expr, 0);
4029
4030         return ERR_PTR(ret);
4031 }
4032
4033 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
4034                                  struct trace_event_file *file)
4035 {
4036         struct event_trigger_data *test;
4037
4038         lockdep_assert_held(&event_mutex);
4039
4040         list_for_each_entry(test, &file->triggers, list) {
4041                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
4042                         if (test->private_data == hist_data)
4043                                 return test->filter_str;
4044                 }
4045         }
4046
4047         return NULL;
4048 }
4049
4050 static struct event_command trigger_hist_cmd;
4051 static int event_hist_trigger_func(struct event_command *cmd_ops,
4052                                    struct trace_event_file *file,
4053                                    char *glob, char *cmd, char *param);
4054
4055 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
4056                             struct hist_trigger_data *hist_data,
4057                             unsigned int n_keys)
4058 {
4059         struct hist_field *target_hist_field, *hist_field;
4060         unsigned int n, i, j;
4061
4062         if (hist_data->n_fields - hist_data->n_vals != n_keys)
4063                 return false;
4064
4065         i = hist_data->n_vals;
4066         j = target_hist_data->n_vals;
4067
4068         for (n = 0; n < n_keys; n++) {
4069                 hist_field = hist_data->fields[i + n];
4070                 target_hist_field = target_hist_data->fields[j + n];
4071
4072                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
4073                         return false;
4074                 if (hist_field->size != target_hist_field->size)
4075                         return false;
4076                 if (hist_field->is_signed != target_hist_field->is_signed)
4077                         return false;
4078         }
4079
4080         return true;
4081 }
4082
4083 static struct hist_trigger_data *
4084 find_compatible_hist(struct hist_trigger_data *target_hist_data,
4085                      struct trace_event_file *file)
4086 {
4087         struct hist_trigger_data *hist_data;
4088         struct event_trigger_data *test;
4089         unsigned int n_keys;
4090
4091         lockdep_assert_held(&event_mutex);
4092
4093         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
4094
4095         list_for_each_entry(test, &file->triggers, list) {
4096                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
4097                         hist_data = test->private_data;
4098
4099                         if (compatible_keys(target_hist_data, hist_data, n_keys))
4100                                 return hist_data;
4101                 }
4102         }
4103
4104         return NULL;
4105 }
4106
4107 static struct trace_event_file *event_file(struct trace_array *tr,
4108                                            char *system, char *event_name)
4109 {
4110         struct trace_event_file *file;
4111
4112         file = __find_event_file(tr, system, event_name);
4113         if (!file)
4114                 return ERR_PTR(-EINVAL);
4115
4116         return file;
4117 }
4118
4119 static struct hist_field *
4120 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
4121                          char *system, char *event_name, char *field_name)
4122 {
4123         struct hist_field *event_var;
4124         char *synthetic_name;
4125
4126         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
4127         if (!synthetic_name)
4128                 return ERR_PTR(-ENOMEM);
4129
4130         strcpy(synthetic_name, "synthetic_");
4131         strcat(synthetic_name, field_name);
4132
4133         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
4134
4135         kfree(synthetic_name);
4136
4137         return event_var;
4138 }
4139
4140 /**
4141  * create_field_var_hist - Automatically create a histogram and var for a field
4142  * @target_hist_data: The target hist trigger
4143  * @subsys_name: Optional subsystem name
4144  * @event_name: Optional event name
4145  * @field_name: The name of the field (and the resulting variable)
4146  *
4147  * Hist trigger actions fetch data from variables, not directly from
4148  * events.  However, for convenience, users are allowed to directly
4149  * specify an event field in an action, which will be automatically
4150  * converted into a variable on their behalf.
4151
4152  * If a user specifies a field on an event that isn't the event the
4153  * histogram currently being defined (the target event histogram), the
4154  * only way that can be accomplished is if a new hist trigger is
4155  * created and the field variable defined on that.
4156  *
4157  * This function creates a new histogram compatible with the target
4158  * event (meaning a histogram with the same key as the target
4159  * histogram), and creates a variable for the specified field, but
4160  * with 'synthetic_' prepended to the variable name in order to avoid
4161  * collision with normal field variables.
4162  *
4163  * Return: The variable created for the field.
4164  */
4165 static struct hist_field *
4166 create_field_var_hist(struct hist_trigger_data *target_hist_data,
4167                       char *subsys_name, char *event_name, char *field_name)
4168 {
4169         struct trace_array *tr = target_hist_data->event_file->tr;
4170         struct hist_field *event_var = ERR_PTR(-EINVAL);
4171         struct hist_trigger_data *hist_data;
4172         unsigned int i, n, first = true;
4173         struct field_var_hist *var_hist;
4174         struct trace_event_file *file;
4175         struct hist_field *key_field;
4176         char *saved_filter;
4177         char *cmd;
4178         int ret;
4179
4180         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
4181                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
4182                 return ERR_PTR(-EINVAL);
4183         }
4184
4185         file = event_file(tr, subsys_name, event_name);
4186
4187         if (IS_ERR(file)) {
4188                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
4189                 ret = PTR_ERR(file);
4190                 return ERR_PTR(ret);
4191         }
4192
4193         /*
4194          * Look for a histogram compatible with target.  We'll use the
4195          * found histogram specification to create a new matching
4196          * histogram with our variable on it.  target_hist_data is not
4197          * yet a registered histogram so we can't use that.
4198          */
4199         hist_data = find_compatible_hist(target_hist_data, file);
4200         if (!hist_data) {
4201                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
4202                 return ERR_PTR(-EINVAL);
4203         }
4204
4205         /* See if a synthetic field variable has already been created */
4206         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
4207                                              event_name, field_name);
4208         if (!IS_ERR_OR_NULL(event_var))
4209                 return event_var;
4210
4211         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
4212         if (!var_hist)
4213                 return ERR_PTR(-ENOMEM);
4214
4215         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
4216         if (!cmd) {
4217                 kfree(var_hist);
4218                 return ERR_PTR(-ENOMEM);
4219         }
4220
4221         /* Use the same keys as the compatible histogram */
4222         strcat(cmd, "keys=");
4223
4224         for_each_hist_key_field(i, hist_data) {
4225                 key_field = hist_data->fields[i];
4226                 if (!first)
4227                         strcat(cmd, ",");
4228                 strcat(cmd, key_field->field->name);
4229                 first = false;
4230         }
4231
4232         /* Create the synthetic field variable specification */
4233         strcat(cmd, ":synthetic_");
4234         strcat(cmd, field_name);
4235         strcat(cmd, "=");
4236         strcat(cmd, field_name);
4237
4238         /* Use the same filter as the compatible histogram */
4239         saved_filter = find_trigger_filter(hist_data, file);
4240         if (saved_filter) {
4241                 strcat(cmd, " if ");
4242                 strcat(cmd, saved_filter);
4243         }
4244
4245         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
4246         if (!var_hist->cmd) {
4247                 kfree(cmd);
4248                 kfree(var_hist);
4249                 return ERR_PTR(-ENOMEM);
4250         }
4251
4252         /* Save the compatible histogram information */
4253         var_hist->hist_data = hist_data;
4254
4255         /* Create the new histogram with our variable */
4256         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
4257                                       "", "hist", cmd);
4258         if (ret) {
4259                 kfree(cmd);
4260                 kfree(var_hist->cmd);
4261                 kfree(var_hist);
4262                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
4263                 return ERR_PTR(ret);
4264         }
4265
4266         kfree(cmd);
4267
4268         /* If we can't find the variable, something went wrong */
4269         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
4270                                              event_name, field_name);
4271         if (IS_ERR_OR_NULL(event_var)) {
4272                 kfree(var_hist->cmd);
4273                 kfree(var_hist);
4274                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
4275                 return ERR_PTR(-EINVAL);
4276         }
4277
4278         n = target_hist_data->n_field_var_hists;
4279         target_hist_data->field_var_hists[n] = var_hist;
4280         target_hist_data->n_field_var_hists++;
4281
4282         return event_var;
4283 }
4284
4285 static struct hist_field *
4286 find_target_event_var(struct hist_trigger_data *hist_data,
4287                       char *subsys_name, char *event_name, char *var_name)
4288 {
4289         struct trace_event_file *file = hist_data->event_file;
4290         struct hist_field *hist_field = NULL;
4291
4292         if (subsys_name) {
4293                 struct trace_event_call *call;
4294
4295                 if (!event_name)
4296                         return NULL;
4297
4298                 call = file->event_call;
4299
4300                 if (strcmp(subsys_name, call->class->system) != 0)
4301                         return NULL;
4302
4303                 if (strcmp(event_name, trace_event_name(call)) != 0)
4304                         return NULL;
4305         }
4306
4307         hist_field = find_var_field(hist_data, var_name);
4308
4309         return hist_field;
4310 }
4311
4312 static inline void __update_field_vars(struct tracing_map_elt *elt,
4313                                        struct ring_buffer_event *rbe,
4314                                        void *rec,
4315                                        struct field_var **field_vars,
4316                                        unsigned int n_field_vars,
4317                                        unsigned int field_var_str_start)
4318 {
4319         struct hist_elt_data *elt_data = elt->private_data;
4320         unsigned int i, j, var_idx;
4321         u64 var_val;
4322
4323         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
4324                 struct field_var *field_var = field_vars[i];
4325                 struct hist_field *var = field_var->var;
4326                 struct hist_field *val = field_var->val;
4327
4328                 var_val = val->fn(val, elt, rbe, rec);
4329                 var_idx = var->var.idx;
4330
4331                 if (val->flags & HIST_FIELD_FL_STRING) {
4332                         char *str = elt_data->field_var_str[j++];
4333                         char *val_str = (char *)(uintptr_t)var_val;
4334
4335                         strscpy(str, val_str, STR_VAR_LEN_MAX);
4336                         var_val = (u64)(uintptr_t)str;
4337                 }
4338                 tracing_map_set_var(elt, var_idx, var_val);
4339         }
4340 }
4341
4342 static void update_field_vars(struct hist_trigger_data *hist_data,
4343                               struct tracing_map_elt *elt,
4344                               struct ring_buffer_event *rbe,
4345                               void *rec)
4346 {
4347         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
4348                             hist_data->n_field_vars, 0);
4349 }
4350
4351 static void save_track_data_vars(struct hist_trigger_data *hist_data,
4352                                  struct tracing_map_elt *elt, void *rec,
4353                                  struct ring_buffer_event *rbe, void *key,
4354                                  struct action_data *data, u64 *var_ref_vals)
4355 {
4356         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
4357                             hist_data->n_save_vars, hist_data->n_field_var_str);
4358 }
4359
4360 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
4361                                      struct trace_event_file *file,
4362                                      char *name, int size, const char *type)
4363 {
4364         struct hist_field *var;
4365         int idx;
4366
4367         if (find_var(hist_data, file, name) && !hist_data->remove) {
4368                 var = ERR_PTR(-EINVAL);
4369                 goto out;
4370         }
4371
4372         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
4373         if (!var) {
4374                 var = ERR_PTR(-ENOMEM);
4375                 goto out;
4376         }
4377
4378         idx = tracing_map_add_var(hist_data->map);
4379         if (idx < 0) {
4380                 kfree(var);
4381                 var = ERR_PTR(-EINVAL);
4382                 goto out;
4383         }
4384
4385         var->flags = HIST_FIELD_FL_VAR;
4386         var->var.idx = idx;
4387         var->var.hist_data = var->hist_data = hist_data;
4388         var->size = size;
4389         var->var.name = kstrdup(name, GFP_KERNEL);
4390         var->type = kstrdup(type, GFP_KERNEL);
4391         if (!var->var.name || !var->type) {
4392                 kfree(var->var.name);
4393                 kfree(var->type);
4394                 kfree(var);
4395                 var = ERR_PTR(-ENOMEM);
4396         }
4397  out:
4398         return var;
4399 }
4400
4401 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
4402                                           struct trace_event_file *file,
4403                                           char *field_name)
4404 {
4405         struct hist_field *val = NULL, *var = NULL;
4406         unsigned long flags = HIST_FIELD_FL_VAR;
4407         struct trace_array *tr = file->tr;
4408         struct field_var *field_var;
4409         int ret = 0;
4410
4411         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
4412                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
4413                 ret = -EINVAL;
4414                 goto err;
4415         }
4416
4417         val = parse_atom(hist_data, file, field_name, &flags, NULL);
4418         if (IS_ERR(val)) {
4419                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
4420                 ret = PTR_ERR(val);
4421                 goto err;
4422         }
4423
4424         var = create_var(hist_data, file, field_name, val->size, val->type);
4425         if (IS_ERR(var)) {
4426                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
4427                 kfree(val);
4428                 ret = PTR_ERR(var);
4429                 goto err;
4430         }
4431
4432         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
4433         if (!field_var) {
4434                 kfree(val);
4435                 kfree(var);
4436                 ret =  -ENOMEM;
4437                 goto err;
4438         }
4439
4440         field_var->var = var;
4441         field_var->val = val;
4442  out:
4443         return field_var;
4444  err:
4445         field_var = ERR_PTR(ret);
4446         goto out;
4447 }
4448
4449 /**
4450  * create_target_field_var - Automatically create a variable for a field
4451  * @target_hist_data: The target hist trigger
4452  * @subsys_name: Optional subsystem name
4453  * @event_name: Optional event name
4454  * @var_name: The name of the field (and the resulting variable)
4455  *
4456  * Hist trigger actions fetch data from variables, not directly from
4457  * events.  However, for convenience, users are allowed to directly
4458  * specify an event field in an action, which will be automatically
4459  * converted into a variable on their behalf.
4460
4461  * This function creates a field variable with the name var_name on
4462  * the hist trigger currently being defined on the target event.  If
4463  * subsys_name and event_name are specified, this function simply
4464  * verifies that they do in fact match the target event subsystem and
4465  * event name.
4466  *
4467  * Return: The variable created for the field.
4468  */
4469 static struct field_var *
4470 create_target_field_var(struct hist_trigger_data *target_hist_data,
4471                         char *subsys_name, char *event_name, char *var_name)
4472 {
4473         struct trace_event_file *file = target_hist_data->event_file;
4474
4475         if (subsys_name) {
4476                 struct trace_event_call *call;
4477
4478                 if (!event_name)
4479                         return NULL;
4480
4481                 call = file->event_call;
4482
4483                 if (strcmp(subsys_name, call->class->system) != 0)
4484                         return NULL;
4485
4486                 if (strcmp(event_name, trace_event_name(call)) != 0)
4487                         return NULL;
4488         }
4489
4490         return create_field_var(target_hist_data, file, var_name);
4491 }
4492
4493 static bool check_track_val_max(u64 track_val, u64 var_val)
4494 {
4495         if (var_val <= track_val)
4496                 return false;
4497
4498         return true;
4499 }
4500
4501 static bool check_track_val_changed(u64 track_val, u64 var_val)
4502 {
4503         if (var_val == track_val)
4504                 return false;
4505
4506         return true;
4507 }
4508
4509 static u64 get_track_val(struct hist_trigger_data *hist_data,
4510                          struct tracing_map_elt *elt,
4511                          struct action_data *data)
4512 {
4513         unsigned int track_var_idx = data->track_data.track_var->var.idx;
4514         u64 track_val;
4515
4516         track_val = tracing_map_read_var(elt, track_var_idx);
4517
4518         return track_val;
4519 }
4520
4521 static void save_track_val(struct hist_trigger_data *hist_data,
4522                            struct tracing_map_elt *elt,
4523                            struct action_data *data, u64 var_val)
4524 {
4525         unsigned int track_var_idx = data->track_data.track_var->var.idx;
4526
4527         tracing_map_set_var(elt, track_var_idx, var_val);
4528 }
4529
4530 static void save_track_data(struct hist_trigger_data *hist_data,
4531                             struct tracing_map_elt *elt, void *rec,
4532                             struct ring_buffer_event *rbe, void *key,
4533                             struct action_data *data, u64 *var_ref_vals)
4534 {
4535         if (data->track_data.save_data)
4536                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4537 }
4538
4539 static bool check_track_val(struct tracing_map_elt *elt,
4540                             struct action_data *data,
4541                             u64 var_val)
4542 {
4543         struct hist_trigger_data *hist_data;
4544         u64 track_val;
4545
4546         hist_data = data->track_data.track_var->hist_data;
4547         track_val = get_track_val(hist_data, elt, data);
4548
4549         return data->track_data.check_val(track_val, var_val);
4550 }
4551
4552 #ifdef CONFIG_TRACER_SNAPSHOT
4553 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
4554 {
4555         /* called with tr->max_lock held */
4556         struct track_data *track_data = tr->cond_snapshot->cond_data;
4557         struct hist_elt_data *elt_data, *track_elt_data;
4558         struct snapshot_context *context = cond_data;
4559         struct action_data *action;
4560         u64 track_val;
4561
4562         if (!track_data)
4563                 return false;
4564
4565         action = track_data->action_data;
4566
4567         track_val = get_track_val(track_data->hist_data, context->elt,
4568                                   track_data->action_data);
4569
4570         if (!action->track_data.check_val(track_data->track_val, track_val))
4571                 return false;
4572
4573         track_data->track_val = track_val;
4574         memcpy(track_data->key, context->key, track_data->key_len);
4575
4576         elt_data = context->elt->private_data;
4577         track_elt_data = track_data->elt.private_data;
4578         if (elt_data->comm)
4579                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
4580
4581         track_data->updated = true;
4582
4583         return true;
4584 }
4585
4586 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
4587                                      struct tracing_map_elt *elt, void *rec,
4588                                      struct ring_buffer_event *rbe, void *key,
4589                                      struct action_data *data,
4590                                      u64 *var_ref_vals)
4591 {
4592         struct trace_event_file *file = hist_data->event_file;
4593         struct snapshot_context context;
4594
4595         context.elt = elt;
4596         context.key = key;
4597
4598         tracing_snapshot_cond(file->tr, &context);
4599 }
4600
4601 static void hist_trigger_print_key(struct seq_file *m,
4602                                    struct hist_trigger_data *hist_data,
4603                                    void *key,
4604                                    struct tracing_map_elt *elt);
4605
4606 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
4607 {
4608         unsigned int i;
4609
4610         if (!hist_data->n_actions)
4611                 return NULL;
4612
4613         for (i = 0; i < hist_data->n_actions; i++) {
4614                 struct action_data *data = hist_data->actions[i];
4615
4616                 if (data->action == ACTION_SNAPSHOT)
4617                         return data;
4618         }
4619
4620         return NULL;
4621 }
4622
4623 static void track_data_snapshot_print(struct seq_file *m,
4624                                       struct hist_trigger_data *hist_data)
4625 {
4626         struct trace_event_file *file = hist_data->event_file;
4627         struct track_data *track_data;
4628         struct action_data *action;
4629
4630         track_data = tracing_cond_snapshot_data(file->tr);
4631         if (!track_data)
4632                 return;
4633
4634         if (!track_data->updated)
4635                 return;
4636
4637         action = snapshot_action(hist_data);
4638         if (!action)
4639                 return;
4640
4641         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
4642         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
4643                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
4644                    action->track_data.var_str, track_data->track_val);
4645
4646         seq_puts(m, "\ttriggered by event with key: ");
4647         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
4648         seq_putc(m, '\n');
4649 }
4650 #else
4651 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
4652 {
4653         return false;
4654 }
4655 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
4656                                      struct tracing_map_elt *elt, void *rec,
4657                                      struct ring_buffer_event *rbe, void *key,
4658                                      struct action_data *data,
4659                                      u64 *var_ref_vals) {}
4660 static void track_data_snapshot_print(struct seq_file *m,
4661                                       struct hist_trigger_data *hist_data) {}
4662 #endif /* CONFIG_TRACER_SNAPSHOT */
4663
4664 static void track_data_print(struct seq_file *m,
4665                              struct hist_trigger_data *hist_data,
4666                              struct tracing_map_elt *elt,
4667                              struct action_data *data)
4668 {
4669         u64 track_val = get_track_val(hist_data, elt, data);
4670         unsigned int i, save_var_idx;
4671
4672         if (data->handler == HANDLER_ONMAX)
4673                 seq_printf(m, "\n\tmax: %10llu", track_val);
4674         else if (data->handler == HANDLER_ONCHANGE)
4675                 seq_printf(m, "\n\tchanged: %10llu", track_val);
4676
4677         if (data->action == ACTION_SNAPSHOT)
4678                 return;
4679
4680         for (i = 0; i < hist_data->n_save_vars; i++) {
4681                 struct hist_field *save_val = hist_data->save_vars[i]->val;
4682                 struct hist_field *save_var = hist_data->save_vars[i]->var;
4683                 u64 val;
4684
4685                 save_var_idx = save_var->var.idx;
4686
4687                 val = tracing_map_read_var(elt, save_var_idx);
4688
4689                 if (save_val->flags & HIST_FIELD_FL_STRING) {
4690                         seq_printf(m, "  %s: %-32s", save_var->var.name,
4691                                    (char *)(uintptr_t)(val));
4692                 } else
4693                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
4694         }
4695 }
4696
4697 static void ontrack_action(struct hist_trigger_data *hist_data,
4698                            struct tracing_map_elt *elt, void *rec,
4699                            struct ring_buffer_event *rbe, void *key,
4700                            struct action_data *data, u64 *var_ref_vals)
4701 {
4702         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
4703
4704         if (check_track_val(elt, data, var_val)) {
4705                 save_track_val(hist_data, elt, data, var_val);
4706                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4707         }
4708 }
4709
4710 static void action_data_destroy(struct action_data *data)
4711 {
4712         unsigned int i;
4713
4714         lockdep_assert_held(&event_mutex);
4715
4716         kfree(data->action_name);
4717
4718         for (i = 0; i < data->n_params; i++)
4719                 kfree(data->params[i]);
4720
4721         if (data->synth_event)
4722                 data->synth_event->ref--;
4723
4724         kfree(data->synth_event_name);
4725
4726         kfree(data);
4727 }
4728
4729 static void track_data_destroy(struct hist_trigger_data *hist_data,
4730                                struct action_data *data)
4731 {
4732         struct trace_event_file *file = hist_data->event_file;
4733
4734         destroy_hist_field(data->track_data.track_var, 0);
4735
4736         if (data->action == ACTION_SNAPSHOT) {
4737                 struct track_data *track_data;
4738
4739                 track_data = tracing_cond_snapshot_data(file->tr);
4740                 if (track_data && track_data->hist_data == hist_data) {
4741                         tracing_snapshot_cond_disable(file->tr);
4742                         track_data_free(track_data);
4743                 }
4744         }
4745
4746         kfree(data->track_data.var_str);
4747
4748         action_data_destroy(data);
4749 }
4750
4751 static int action_create(struct hist_trigger_data *hist_data,
4752                          struct action_data *data);
4753
4754 static int track_data_create(struct hist_trigger_data *hist_data,
4755                              struct action_data *data)
4756 {
4757         struct hist_field *var_field, *ref_field, *track_var = NULL;
4758         struct trace_event_file *file = hist_data->event_file;
4759         struct trace_array *tr = file->tr;
4760         char *track_data_var_str;
4761         int ret = 0;
4762
4763         track_data_var_str = data->track_data.var_str;
4764         if (track_data_var_str[0] != '$') {
4765                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
4766                 return -EINVAL;
4767         }
4768         track_data_var_str++;
4769
4770         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
4771         if (!var_field) {
4772                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
4773                 return -EINVAL;
4774         }
4775
4776         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
4777         if (!ref_field)
4778                 return -ENOMEM;
4779
4780         data->track_data.var_ref = ref_field;
4781
4782         if (data->handler == HANDLER_ONMAX)
4783                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
4784         if (IS_ERR(track_var)) {
4785                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
4786                 ret = PTR_ERR(track_var);
4787                 goto out;
4788         }
4789
4790         if (data->handler == HANDLER_ONCHANGE)
4791                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
4792         if (IS_ERR(track_var)) {
4793                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
4794                 ret = PTR_ERR(track_var);
4795                 goto out;
4796         }
4797         data->track_data.track_var = track_var;
4798
4799         ret = action_create(hist_data, data);
4800  out:
4801         return ret;
4802 }
4803
4804 static int parse_action_params(struct trace_array *tr, char *params,
4805                                struct action_data *data)
4806 {
4807         char *param, *saved_param;
4808         bool first_param = true;
4809         int ret = 0;
4810
4811         while (params) {
4812                 if (data->n_params >= SYNTH_FIELDS_MAX) {
4813                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
4814                         goto out;
4815                 }
4816
4817                 param = strsep(&params, ",");
4818                 if (!param) {
4819                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
4820                         ret = -EINVAL;
4821                         goto out;
4822                 }
4823
4824                 param = strstrip(param);
4825                 if (strlen(param) < 2) {
4826                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
4827                         ret = -EINVAL;
4828                         goto out;
4829                 }
4830
4831                 saved_param = kstrdup(param, GFP_KERNEL);
4832                 if (!saved_param) {
4833                         ret = -ENOMEM;
4834                         goto out;
4835                 }
4836
4837                 if (first_param && data->use_trace_keyword) {
4838                         data->synth_event_name = saved_param;
4839                         first_param = false;
4840                         continue;
4841                 }
4842                 first_param = false;
4843
4844                 data->params[data->n_params++] = saved_param;
4845         }
4846  out:
4847         return ret;
4848 }
4849
4850 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
4851                         enum handler_id handler)
4852 {
4853         char *action_name;
4854         int ret = 0;
4855
4856         strsep(&str, ".");
4857         if (!str) {
4858                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
4859                 ret = -EINVAL;
4860                 goto out;
4861         }
4862
4863         action_name = strsep(&str, "(");
4864         if (!action_name || !str) {
4865                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
4866                 ret = -EINVAL;
4867                 goto out;
4868         }
4869
4870         if (str_has_prefix(action_name, "save")) {
4871                 char *params = strsep(&str, ")");
4872
4873                 if (!params) {
4874                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
4875                         ret = -EINVAL;
4876                         goto out;
4877                 }
4878
4879                 ret = parse_action_params(tr, params, data);
4880                 if (ret)
4881                         goto out;
4882
4883                 if (handler == HANDLER_ONMAX)
4884                         data->track_data.check_val = check_track_val_max;
4885                 else if (handler == HANDLER_ONCHANGE)
4886                         data->track_data.check_val = check_track_val_changed;
4887                 else {
4888                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
4889                         ret = -EINVAL;
4890                         goto out;
4891                 }
4892
4893                 data->track_data.save_data = save_track_data_vars;
4894                 data->fn = ontrack_action;
4895                 data->action = ACTION_SAVE;
4896         } else if (str_has_prefix(action_name, "snapshot")) {
4897                 char *params = strsep(&str, ")");
4898
4899                 if (!str) {
4900                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
4901                         ret = -EINVAL;
4902                         goto out;
4903                 }
4904
4905                 if (handler == HANDLER_ONMAX)
4906                         data->track_data.check_val = check_track_val_max;
4907                 else if (handler == HANDLER_ONCHANGE)
4908                         data->track_data.check_val = check_track_val_changed;
4909                 else {
4910                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
4911                         ret = -EINVAL;
4912                         goto out;
4913                 }
4914
4915                 data->track_data.save_data = save_track_data_snapshot;
4916                 data->fn = ontrack_action;
4917                 data->action = ACTION_SNAPSHOT;
4918         } else {
4919                 char *params = strsep(&str, ")");
4920
4921                 if (str_has_prefix(action_name, "trace"))
4922                         data->use_trace_keyword = true;
4923
4924                 if (params) {
4925                         ret = parse_action_params(tr, params, data);
4926                         if (ret)
4927                                 goto out;
4928                 }
4929
4930                 if (handler == HANDLER_ONMAX)
4931                         data->track_data.check_val = check_track_val_max;
4932                 else if (handler == HANDLER_ONCHANGE)
4933                         data->track_data.check_val = check_track_val_changed;
4934
4935                 if (handler != HANDLER_ONMATCH) {
4936                         data->track_data.save_data = action_trace;
4937                         data->fn = ontrack_action;
4938                 } else
4939                         data->fn = action_trace;
4940
4941                 data->action = ACTION_TRACE;
4942         }
4943
4944         data->action_name = kstrdup(action_name, GFP_KERNEL);
4945         if (!data->action_name) {
4946                 ret = -ENOMEM;
4947                 goto out;
4948         }
4949
4950         data->handler = handler;
4951  out:
4952         return ret;
4953 }
4954
4955 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4956                                             char *str, enum handler_id handler)
4957 {
4958         struct action_data *data;
4959         int ret = -EINVAL;
4960         char *var_str;
4961
4962         data = kzalloc(sizeof(*data), GFP_KERNEL);
4963         if (!data)
4964                 return ERR_PTR(-ENOMEM);
4965
4966         var_str = strsep(&str, ")");
4967         if (!var_str || !str) {
4968                 ret = -EINVAL;
4969                 goto free;
4970         }
4971
4972         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4973         if (!data->track_data.var_str) {
4974                 ret = -ENOMEM;
4975                 goto free;
4976         }
4977
4978         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4979         if (ret)
4980                 goto free;
4981  out:
4982         return data;
4983  free:
4984         track_data_destroy(hist_data, data);
4985         data = ERR_PTR(ret);
4986         goto out;
4987 }
4988
4989 static void onmatch_destroy(struct action_data *data)
4990 {
4991         kfree(data->match_data.event);
4992         kfree(data->match_data.event_system);
4993
4994         action_data_destroy(data);
4995 }
4996
4997 static void destroy_field_var(struct field_var *field_var)
4998 {
4999         if (!field_var)
5000                 return;
5001
5002         destroy_hist_field(field_var->var, 0);
5003         destroy_hist_field(field_var->val, 0);
5004
5005         kfree(field_var);
5006 }
5007
5008 static void destroy_field_vars(struct hist_trigger_data *hist_data)
5009 {
5010         unsigned int i;
5011
5012         for (i = 0; i < hist_data->n_field_vars; i++)
5013                 destroy_field_var(hist_data->field_vars[i]);
5014 }
5015
5016 static void save_field_var(struct hist_trigger_data *hist_data,
5017                            struct field_var *field_var)
5018 {
5019         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
5020
5021         if (field_var->val->flags & HIST_FIELD_FL_STRING)
5022                 hist_data->n_field_var_str++;
5023 }
5024
5025
5026 static int check_synth_field(struct synth_event *event,
5027                              struct hist_field *hist_field,
5028                              unsigned int field_pos)
5029 {
5030         struct synth_field *field;
5031
5032         if (field_pos >= event->n_fields)
5033                 return -EINVAL;
5034
5035         field = event->fields[field_pos];
5036
5037         if (strcmp(field->type, hist_field->type) != 0) {
5038                 if (field->size != hist_field->size ||
5039                     field->is_signed != hist_field->is_signed)
5040                         return -EINVAL;
5041         }
5042
5043         return 0;
5044 }
5045
5046 static struct hist_field *
5047 trace_action_find_var(struct hist_trigger_data *hist_data,
5048                       struct action_data *data,
5049                       char *system, char *event, char *var)
5050 {
5051         struct trace_array *tr = hist_data->event_file->tr;
5052         struct hist_field *hist_field;
5053
5054         var++; /* skip '$' */
5055
5056         hist_field = find_target_event_var(hist_data, system, event, var);
5057         if (!hist_field) {
5058                 if (!system && data->handler == HANDLER_ONMATCH) {
5059                         system = data->match_data.event_system;
5060                         event = data->match_data.event;
5061                 }
5062
5063                 hist_field = find_event_var(hist_data, system, event, var);
5064         }
5065
5066         if (!hist_field)
5067                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
5068
5069         return hist_field;
5070 }
5071
5072 static struct hist_field *
5073 trace_action_create_field_var(struct hist_trigger_data *hist_data,
5074                               struct action_data *data, char *system,
5075                               char *event, char *var)
5076 {
5077         struct hist_field *hist_field = NULL;
5078         struct field_var *field_var;
5079
5080         /*
5081          * First try to create a field var on the target event (the
5082          * currently being defined).  This will create a variable for
5083          * unqualified fields on the target event, or if qualified,
5084          * target fields that have qualified names matching the target.
5085          */
5086         field_var = create_target_field_var(hist_data, system, event, var);
5087
5088         if (field_var && !IS_ERR(field_var)) {
5089                 save_field_var(hist_data, field_var);
5090                 hist_field = field_var->var;
5091         } else {
5092                 field_var = NULL;
5093                 /*
5094                  * If no explicit system.event is specfied, default to
5095                  * looking for fields on the onmatch(system.event.xxx)
5096                  * event.
5097                  */
5098                 if (!system && data->handler == HANDLER_ONMATCH) {
5099                         system = data->match_data.event_system;
5100                         event = data->match_data.event;
5101                 }
5102
5103                 /*
5104                  * At this point, we're looking at a field on another
5105                  * event.  Because we can't modify a hist trigger on
5106                  * another event to add a variable for a field, we need
5107                  * to create a new trigger on that event and create the
5108                  * variable at the same time.
5109                  */
5110                 hist_field = create_field_var_hist(hist_data, system, event, var);
5111                 if (IS_ERR(hist_field))
5112                         goto free;
5113         }
5114  out:
5115         return hist_field;
5116  free:
5117         destroy_field_var(field_var);
5118         hist_field = NULL;
5119         goto out;
5120 }
5121
5122 static int trace_action_create(struct hist_trigger_data *hist_data,
5123                                struct action_data *data)
5124 {
5125         struct trace_array *tr = hist_data->event_file->tr;
5126         char *event_name, *param, *system = NULL;
5127         struct hist_field *hist_field, *var_ref;
5128         unsigned int i;
5129         unsigned int field_pos = 0;
5130         struct synth_event *event;
5131         char *synth_event_name;
5132         int var_ref_idx, ret = 0;
5133
5134         lockdep_assert_held(&event_mutex);
5135
5136         if (data->use_trace_keyword)
5137                 synth_event_name = data->synth_event_name;
5138         else
5139                 synth_event_name = data->action_name;
5140
5141         event = find_synth_event(synth_event_name);
5142         if (!event) {
5143                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
5144                 return -EINVAL;
5145         }
5146
5147         event->ref++;
5148
5149         for (i = 0; i < data->n_params; i++) {
5150                 char *p;
5151
5152                 p = param = kstrdup(data->params[i], GFP_KERNEL);
5153                 if (!param) {
5154                         ret = -ENOMEM;
5155                         goto err;
5156                 }
5157
5158                 system = strsep(&param, ".");
5159                 if (!param) {
5160                         param = (char *)system;
5161                         system = event_name = NULL;
5162                 } else {
5163                         event_name = strsep(&param, ".");
5164                         if (!param) {
5165                                 kfree(p);
5166                                 ret = -EINVAL;
5167                                 goto err;
5168                         }
5169                 }
5170
5171                 if (param[0] == '$')
5172                         hist_field = trace_action_find_var(hist_data, data,
5173                                                            system, event_name,
5174                                                            param);
5175                 else
5176                         hist_field = trace_action_create_field_var(hist_data,
5177                                                                    data,
5178                                                                    system,
5179                                                                    event_name,
5180                                                                    param);
5181
5182                 if (!hist_field) {
5183                         kfree(p);
5184                         ret = -EINVAL;
5185                         goto err;
5186                 }
5187
5188                 if (check_synth_field(event, hist_field, field_pos) == 0) {
5189                         var_ref = create_var_ref(hist_data, hist_field,
5190                                                  system, event_name);
5191                         if (!var_ref) {
5192                                 kfree(p);
5193                                 ret = -ENOMEM;
5194                                 goto err;
5195                         }
5196
5197                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
5198                         if (WARN_ON(var_ref_idx < 0)) {
5199                                 ret = var_ref_idx;
5200                                 goto err;
5201                         }
5202
5203                         data->var_ref_idx[i] = var_ref_idx;
5204
5205                         field_pos++;
5206                         kfree(p);
5207                         continue;
5208                 }
5209
5210                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
5211                 kfree(p);
5212                 ret = -EINVAL;
5213                 goto err;
5214         }
5215
5216         if (field_pos != event->n_fields) {
5217                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
5218                 ret = -EINVAL;
5219                 goto err;
5220         }
5221
5222         data->synth_event = event;
5223  out:
5224         return ret;
5225  err:
5226         event->ref--;
5227
5228         goto out;
5229 }
5230
5231 static int action_create(struct hist_trigger_data *hist_data,
5232                          struct action_data *data)
5233 {
5234         struct trace_event_file *file = hist_data->event_file;
5235         struct trace_array *tr = file->tr;
5236         struct track_data *track_data;
5237         struct field_var *field_var;
5238         unsigned int i;
5239         char *param;
5240         int ret = 0;
5241
5242         if (data->action == ACTION_TRACE)
5243                 return trace_action_create(hist_data, data);
5244
5245         if (data->action == ACTION_SNAPSHOT) {
5246                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
5247                 if (IS_ERR(track_data)) {
5248                         ret = PTR_ERR(track_data);
5249                         goto out;
5250                 }
5251
5252                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
5253                                                    cond_snapshot_update);
5254                 if (ret)
5255                         track_data_free(track_data);
5256
5257                 goto out;
5258         }
5259
5260         if (data->action == ACTION_SAVE) {
5261                 if (hist_data->n_save_vars) {
5262                         ret = -EEXIST;
5263                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
5264                         goto out;
5265                 }
5266
5267                 for (i = 0; i < data->n_params; i++) {
5268                         param = kstrdup(data->params[i], GFP_KERNEL);
5269                         if (!param) {
5270                                 ret = -ENOMEM;
5271                                 goto out;
5272                         }
5273
5274                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
5275                         if (IS_ERR(field_var)) {
5276                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
5277                                          errpos(param));
5278                                 ret = PTR_ERR(field_var);
5279                                 kfree(param);
5280                                 goto out;
5281                         }
5282
5283                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
5284                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
5285                                 hist_data->n_save_var_str++;
5286                         kfree(param);
5287                 }
5288         }
5289  out:
5290         return ret;
5291 }
5292
5293 static int onmatch_create(struct hist_trigger_data *hist_data,
5294                           struct action_data *data)
5295 {
5296         return action_create(hist_data, data);
5297 }
5298
5299 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
5300 {
5301         char *match_event, *match_event_system;
5302         struct action_data *data;
5303         int ret = -EINVAL;
5304
5305         data = kzalloc(sizeof(*data), GFP_KERNEL);
5306         if (!data)
5307                 return ERR_PTR(-ENOMEM);
5308
5309         match_event = strsep(&str, ")");
5310         if (!match_event || !str) {
5311                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
5312                 goto free;
5313         }
5314
5315         match_event_system = strsep(&match_event, ".");
5316         if (!match_event) {
5317                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
5318                 goto free;
5319         }
5320
5321         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
5322                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
5323                 goto free;
5324         }
5325
5326         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
5327         if (!data->match_data.event) {
5328                 ret = -ENOMEM;
5329                 goto free;
5330         }
5331
5332         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
5333         if (!data->match_data.event_system) {
5334                 ret = -ENOMEM;
5335                 goto free;
5336         }
5337
5338         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
5339         if (ret)
5340                 goto free;
5341  out:
5342         return data;
5343  free:
5344         onmatch_destroy(data);
5345         data = ERR_PTR(ret);
5346         goto out;
5347 }
5348
5349 static int create_hitcount_val(struct hist_trigger_data *hist_data)
5350 {
5351         hist_data->fields[HITCOUNT_IDX] =
5352                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
5353         if (!hist_data->fields[HITCOUNT_IDX])
5354                 return -ENOMEM;
5355
5356         hist_data->n_vals++;
5357         hist_data->n_fields++;
5358
5359         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
5360                 return -EINVAL;
5361
5362         return 0;
5363 }
5364
5365 static int __create_val_field(struct hist_trigger_data *hist_data,
5366                               unsigned int val_idx,
5367                               struct trace_event_file *file,
5368                               char *var_name, char *field_str,
5369                               unsigned long flags)
5370 {
5371         struct hist_field *hist_field;
5372         int ret = 0;
5373
5374         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
5375         if (IS_ERR(hist_field)) {
5376                 ret = PTR_ERR(hist_field);
5377                 goto out;
5378         }
5379
5380         hist_data->fields[val_idx] = hist_field;
5381
5382         ++hist_data->n_vals;
5383         ++hist_data->n_fields;
5384
5385         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
5386                 ret = -EINVAL;
5387  out:
5388         return ret;
5389 }
5390
5391 static int create_val_field(struct hist_trigger_data *hist_data,
5392                             unsigned int val_idx,
5393                             struct trace_event_file *file,
5394                             char *field_str)
5395 {
5396         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
5397                 return -EINVAL;
5398
5399         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
5400 }
5401
5402 static int create_var_field(struct hist_trigger_data *hist_data,
5403                             unsigned int val_idx,
5404                             struct trace_event_file *file,
5405                             char *var_name, char *expr_str)
5406 {
5407         struct trace_array *tr = hist_data->event_file->tr;
5408         unsigned long flags = 0;
5409
5410         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
5411                 return -EINVAL;
5412
5413         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
5414                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
5415                 return -EINVAL;
5416         }
5417
5418         flags |= HIST_FIELD_FL_VAR;
5419         hist_data->n_vars++;
5420         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
5421                 return -EINVAL;
5422
5423         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
5424 }
5425
5426 static int create_val_fields(struct hist_trigger_data *hist_data,
5427                              struct trace_event_file *file)
5428 {
5429         char *fields_str, *field_str;
5430         unsigned int i, j = 1;
5431         int ret;
5432
5433         ret = create_hitcount_val(hist_data);
5434         if (ret)
5435                 goto out;
5436
5437         fields_str = hist_data->attrs->vals_str;
5438         if (!fields_str)
5439                 goto out;
5440
5441         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
5442                      j < TRACING_MAP_VALS_MAX; i++) {
5443                 field_str = strsep(&fields_str, ",");
5444                 if (!field_str)
5445                         break;
5446
5447                 if (strcmp(field_str, "hitcount") == 0)
5448                         continue;
5449
5450                 ret = create_val_field(hist_data, j++, file, field_str);
5451                 if (ret)
5452                         goto out;
5453         }
5454
5455         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
5456                 ret = -EINVAL;
5457  out:
5458         return ret;
5459 }
5460
5461 static int create_key_field(struct hist_trigger_data *hist_data,
5462                             unsigned int key_idx,
5463                             unsigned int key_offset,
5464                             struct trace_event_file *file,
5465                             char *field_str)
5466 {
5467         struct trace_array *tr = hist_data->event_file->tr;
5468         struct hist_field *hist_field = NULL;
5469         unsigned long flags = 0;
5470         unsigned int key_size;
5471         int ret = 0;
5472
5473         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
5474                 return -EINVAL;
5475
5476         flags |= HIST_FIELD_FL_KEY;
5477
5478         if (strcmp(field_str, "stacktrace") == 0) {
5479                 flags |= HIST_FIELD_FL_STACKTRACE;
5480                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
5481                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
5482         } else {
5483                 hist_field = parse_expr(hist_data, file, field_str, flags,
5484                                         NULL, 0);
5485                 if (IS_ERR(hist_field)) {
5486                         ret = PTR_ERR(hist_field);
5487                         goto out;
5488                 }
5489
5490                 if (field_has_hist_vars(hist_field, 0)) {
5491                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
5492                         destroy_hist_field(hist_field, 0);
5493                         ret = -EINVAL;
5494                         goto out;
5495                 }
5496
5497                 key_size = hist_field->size;
5498         }
5499
5500         hist_data->fields[key_idx] = hist_field;
5501
5502         key_size = ALIGN(key_size, sizeof(u64));
5503         hist_data->fields[key_idx]->size = key_size;
5504         hist_data->fields[key_idx]->offset = key_offset;
5505
5506         hist_data->key_size += key_size;
5507
5508         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
5509                 ret = -EINVAL;
5510                 goto out;
5511         }
5512
5513         hist_data->n_keys++;
5514         hist_data->n_fields++;
5515
5516         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
5517                 return -EINVAL;
5518
5519         ret = key_size;
5520  out:
5521         return ret;
5522 }
5523
5524 static int create_key_fields(struct hist_trigger_data *hist_data,
5525                              struct trace_event_file *file)
5526 {
5527         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
5528         char *fields_str, *field_str;
5529         int ret = -EINVAL;
5530
5531         fields_str = hist_data->attrs->keys_str;
5532         if (!fields_str)
5533                 goto out;
5534
5535         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
5536                 field_str = strsep(&fields_str, ",");
5537                 if (!field_str)
5538                         break;
5539                 ret = create_key_field(hist_data, i, key_offset,
5540                                        file, field_str);
5541                 if (ret < 0)
5542                         goto out;
5543                 key_offset += ret;
5544         }
5545         if (fields_str) {
5546                 ret = -EINVAL;
5547                 goto out;
5548         }
5549         ret = 0;
5550  out:
5551         return ret;
5552 }
5553
5554 static int create_var_fields(struct hist_trigger_data *hist_data,
5555                              struct trace_event_file *file)
5556 {
5557         unsigned int i, j = hist_data->n_vals;
5558         int ret = 0;
5559
5560         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
5561
5562         for (i = 0; i < n_vars; i++) {
5563                 char *var_name = hist_data->attrs->var_defs.name[i];
5564                 char *expr = hist_data->attrs->var_defs.expr[i];
5565
5566                 ret = create_var_field(hist_data, j++, file, var_name, expr);
5567                 if (ret)
5568                         goto out;
5569         }
5570  out:
5571         return ret;
5572 }
5573
5574 static void free_var_defs(struct hist_trigger_data *hist_data)
5575 {
5576         unsigned int i;
5577
5578         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
5579                 kfree(hist_data->attrs->var_defs.name[i]);
5580                 kfree(hist_data->attrs->var_defs.expr[i]);
5581         }
5582
5583         hist_data->attrs->var_defs.n_vars = 0;
5584 }
5585
5586 static int parse_var_defs(struct hist_trigger_data *hist_data)
5587 {
5588         struct trace_array *tr = hist_data->event_file->tr;
5589         char *s, *str, *var_name, *field_str;
5590         unsigned int i, j, n_vars = 0;
5591         int ret = 0;
5592
5593         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
5594                 str = hist_data->attrs->assignment_str[i];
5595                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
5596                         field_str = strsep(&str, ",");
5597                         if (!field_str)
5598                                 break;
5599
5600                         var_name = strsep(&field_str, "=");
5601                         if (!var_name || !field_str) {
5602                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
5603                                          errpos(var_name));
5604                                 ret = -EINVAL;
5605                                 goto free;
5606                         }
5607
5608                         if (n_vars == TRACING_MAP_VARS_MAX) {
5609                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
5610                                 ret = -EINVAL;
5611                                 goto free;
5612                         }
5613
5614                         s = kstrdup(var_name, GFP_KERNEL);
5615                         if (!s) {
5616                                 ret = -ENOMEM;
5617                                 goto free;
5618                         }
5619                         hist_data->attrs->var_defs.name[n_vars] = s;
5620
5621                         s = kstrdup(field_str, GFP_KERNEL);
5622                         if (!s) {
5623                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
5624                                 ret = -ENOMEM;
5625                                 goto free;
5626                         }
5627                         hist_data->attrs->var_defs.expr[n_vars++] = s;
5628
5629                         hist_data->attrs->var_defs.n_vars = n_vars;
5630                 }
5631         }
5632
5633         return ret;
5634  free:
5635         free_var_defs(hist_data);
5636
5637         return ret;
5638 }
5639
5640 static int create_hist_fields(struct hist_trigger_data *hist_data,
5641                               struct trace_event_file *file)
5642 {
5643         int ret;
5644
5645         ret = parse_var_defs(hist_data);
5646         if (ret)
5647                 goto out;
5648
5649         ret = create_val_fields(hist_data, file);
5650         if (ret)
5651                 goto out;
5652
5653         ret = create_var_fields(hist_data, file);
5654         if (ret)
5655                 goto out;
5656
5657         ret = create_key_fields(hist_data, file);
5658         if (ret)
5659                 goto out;
5660  out:
5661         free_var_defs(hist_data);
5662
5663         return ret;
5664 }
5665
5666 static int is_descending(struct trace_array *tr, const char *str)
5667 {
5668         if (!str)
5669                 return 0;
5670
5671         if (strcmp(str, "descending") == 0)
5672                 return 1;
5673
5674         if (strcmp(str, "ascending") == 0)
5675                 return 0;
5676
5677         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
5678
5679         return -EINVAL;
5680 }
5681
5682 static int create_sort_keys(struct hist_trigger_data *hist_data)
5683 {
5684         struct trace_array *tr = hist_data->event_file->tr;
5685         char *fields_str = hist_data->attrs->sort_key_str;
5686         struct tracing_map_sort_key *sort_key;
5687         int descending, ret = 0;
5688         unsigned int i, j, k;
5689
5690         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
5691
5692         if (!fields_str)
5693                 goto out;
5694
5695         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
5696                 struct hist_field *hist_field;
5697                 char *field_str, *field_name;
5698                 const char *test_name;
5699
5700                 sort_key = &hist_data->sort_keys[i];
5701
5702                 field_str = strsep(&fields_str, ",");
5703                 if (!field_str)
5704                         break;
5705
5706                 if (!*field_str) {
5707                         ret = -EINVAL;
5708                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
5709                         break;
5710                 }
5711
5712                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
5713                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
5714                         ret = -EINVAL;
5715                         break;
5716                 }
5717
5718                 field_name = strsep(&field_str, ".");
5719                 if (!field_name || !*field_name) {
5720                         ret = -EINVAL;
5721                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
5722                         break;
5723                 }
5724
5725                 if (strcmp(field_name, "hitcount") == 0) {
5726                         descending = is_descending(tr, field_str);
5727                         if (descending < 0) {
5728                                 ret = descending;
5729                                 break;
5730                         }
5731                         sort_key->descending = descending;
5732                         continue;
5733                 }
5734
5735                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
5736                         unsigned int idx;
5737
5738                         hist_field = hist_data->fields[j];
5739                         if (hist_field->flags & HIST_FIELD_FL_VAR)
5740                                 continue;
5741
5742                         idx = k++;
5743
5744                         test_name = hist_field_name(hist_field, 0);
5745
5746                         if (strcmp(field_name, test_name) == 0) {
5747                                 sort_key->field_idx = idx;
5748                                 descending = is_descending(tr, field_str);
5749                                 if (descending < 0) {
5750                                         ret = descending;
5751                                         goto out;
5752                                 }
5753                                 sort_key->descending = descending;
5754                                 break;
5755                         }
5756                 }
5757                 if (j == hist_data->n_fields) {
5758                         ret = -EINVAL;
5759                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
5760                         break;
5761                 }
5762         }
5763
5764         hist_data->n_sort_keys = i;
5765  out:
5766         return ret;
5767 }
5768
5769 static void destroy_actions(struct hist_trigger_data *hist_data)
5770 {
5771         unsigned int i;
5772
5773         for (i = 0; i < hist_data->n_actions; i++) {
5774                 struct action_data *data = hist_data->actions[i];
5775
5776                 if (data->handler == HANDLER_ONMATCH)
5777                         onmatch_destroy(data);
5778                 else if (data->handler == HANDLER_ONMAX ||
5779                          data->handler == HANDLER_ONCHANGE)
5780                         track_data_destroy(hist_data, data);
5781                 else
5782                         kfree(data);
5783         }
5784 }
5785
5786 static int parse_actions(struct hist_trigger_data *hist_data)
5787 {
5788         struct trace_array *tr = hist_data->event_file->tr;
5789         struct action_data *data;
5790         unsigned int i;
5791         int ret = 0;
5792         char *str;
5793         int len;
5794
5795         for (i = 0; i < hist_data->attrs->n_actions; i++) {
5796                 str = hist_data->attrs->action_str[i];
5797
5798                 if ((len = str_has_prefix(str, "onmatch("))) {
5799                         char *action_str = str + len;
5800
5801                         data = onmatch_parse(tr, action_str);
5802                         if (IS_ERR(data)) {
5803                                 ret = PTR_ERR(data);
5804                                 break;
5805                         }
5806                 } else if ((len = str_has_prefix(str, "onmax("))) {
5807                         char *action_str = str + len;
5808
5809                         data = track_data_parse(hist_data, action_str,
5810                                                 HANDLER_ONMAX);
5811                         if (IS_ERR(data)) {
5812                                 ret = PTR_ERR(data);
5813                                 break;
5814                         }
5815                 } else if ((len = str_has_prefix(str, "onchange("))) {
5816                         char *action_str = str + len;
5817
5818                         data = track_data_parse(hist_data, action_str,
5819                                                 HANDLER_ONCHANGE);
5820                         if (IS_ERR(data)) {
5821                                 ret = PTR_ERR(data);
5822                                 break;
5823                         }
5824                 } else {
5825                         ret = -EINVAL;
5826                         break;
5827                 }
5828
5829                 hist_data->actions[hist_data->n_actions++] = data;
5830         }
5831
5832         return ret;
5833 }
5834
5835 static int create_actions(struct hist_trigger_data *hist_data)
5836 {
5837         struct action_data *data;
5838         unsigned int i;
5839         int ret = 0;
5840
5841         for (i = 0; i < hist_data->attrs->n_actions; i++) {
5842                 data = hist_data->actions[i];
5843
5844                 if (data->handler == HANDLER_ONMATCH) {
5845                         ret = onmatch_create(hist_data, data);
5846                         if (ret)
5847                                 break;
5848                 } else if (data->handler == HANDLER_ONMAX ||
5849                            data->handler == HANDLER_ONCHANGE) {
5850                         ret = track_data_create(hist_data, data);
5851                         if (ret)
5852                                 break;
5853                 } else {
5854                         ret = -EINVAL;
5855                         break;
5856                 }
5857         }
5858
5859         return ret;
5860 }
5861
5862 static void print_actions(struct seq_file *m,
5863                           struct hist_trigger_data *hist_data,
5864                           struct tracing_map_elt *elt)
5865 {
5866         unsigned int i;
5867
5868         for (i = 0; i < hist_data->n_actions; i++) {
5869                 struct action_data *data = hist_data->actions[i];
5870
5871                 if (data->action == ACTION_SNAPSHOT)
5872                         continue;
5873
5874                 if (data->handler == HANDLER_ONMAX ||
5875                     data->handler == HANDLER_ONCHANGE)
5876                         track_data_print(m, hist_data, elt, data);
5877         }
5878 }
5879
5880 static void print_action_spec(struct seq_file *m,
5881                               struct hist_trigger_data *hist_data,
5882                               struct action_data *data)
5883 {
5884         unsigned int i;
5885
5886         if (data->action == ACTION_SAVE) {
5887                 for (i = 0; i < hist_data->n_save_vars; i++) {
5888                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
5889                         if (i < hist_data->n_save_vars - 1)
5890                                 seq_puts(m, ",");
5891                 }
5892         } else if (data->action == ACTION_TRACE) {
5893                 if (data->use_trace_keyword)
5894                         seq_printf(m, "%s", data->synth_event_name);
5895                 for (i = 0; i < data->n_params; i++) {
5896                         if (i || data->use_trace_keyword)
5897                                 seq_puts(m, ",");
5898                         seq_printf(m, "%s", data->params[i]);
5899                 }
5900         }
5901 }
5902
5903 static void print_track_data_spec(struct seq_file *m,
5904                                   struct hist_trigger_data *hist_data,
5905                                   struct action_data *data)
5906 {
5907         if (data->handler == HANDLER_ONMAX)
5908                 seq_puts(m, ":onmax(");
5909         else if (data->handler == HANDLER_ONCHANGE)
5910                 seq_puts(m, ":onchange(");
5911         seq_printf(m, "%s", data->track_data.var_str);
5912         seq_printf(m, ").%s(", data->action_name);
5913
5914         print_action_spec(m, hist_data, data);
5915
5916         seq_puts(m, ")");
5917 }
5918
5919 static void print_onmatch_spec(struct seq_file *m,
5920                                struct hist_trigger_data *hist_data,
5921                                struct action_data *data)
5922 {
5923         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
5924                    data->match_data.event);
5925
5926         seq_printf(m, "%s(", data->action_name);
5927
5928         print_action_spec(m, hist_data, data);
5929
5930         seq_puts(m, ")");
5931 }
5932
5933 static bool actions_match(struct hist_trigger_data *hist_data,
5934                           struct hist_trigger_data *hist_data_test)
5935 {
5936         unsigned int i, j;
5937
5938         if (hist_data->n_actions != hist_data_test->n_actions)
5939                 return false;
5940
5941         for (i = 0; i < hist_data->n_actions; i++) {
5942                 struct action_data *data = hist_data->actions[i];
5943                 struct action_data *data_test = hist_data_test->actions[i];
5944                 char *action_name, *action_name_test;
5945
5946                 if (data->handler != data_test->handler)
5947                         return false;
5948                 if (data->action != data_test->action)
5949                         return false;
5950
5951                 if (data->n_params != data_test->n_params)
5952                         return false;
5953
5954                 for (j = 0; j < data->n_params; j++) {
5955                         if (strcmp(data->params[j], data_test->params[j]) != 0)
5956                                 return false;
5957                 }
5958
5959                 if (data->use_trace_keyword)
5960                         action_name = data->synth_event_name;
5961                 else
5962                         action_name = data->action_name;
5963
5964                 if (data_test->use_trace_keyword)
5965                         action_name_test = data_test->synth_event_name;
5966                 else
5967                         action_name_test = data_test->action_name;
5968
5969                 if (strcmp(action_name, action_name_test) != 0)
5970                         return false;
5971
5972                 if (data->handler == HANDLER_ONMATCH) {
5973                         if (strcmp(data->match_data.event_system,
5974                                    data_test->match_data.event_system) != 0)
5975                                 return false;
5976                         if (strcmp(data->match_data.event,
5977                                    data_test->match_data.event) != 0)
5978                                 return false;
5979                 } else if (data->handler == HANDLER_ONMAX ||
5980                            data->handler == HANDLER_ONCHANGE) {
5981                         if (strcmp(data->track_data.var_str,
5982                                    data_test->track_data.var_str) != 0)
5983                                 return false;
5984                 }
5985         }
5986
5987         return true;
5988 }
5989
5990
5991 static void print_actions_spec(struct seq_file *m,
5992                                struct hist_trigger_data *hist_data)
5993 {
5994         unsigned int i;
5995
5996         for (i = 0; i < hist_data->n_actions; i++) {
5997                 struct action_data *data = hist_data->actions[i];
5998
5999                 if (data->handler == HANDLER_ONMATCH)
6000                         print_onmatch_spec(m, hist_data, data);
6001                 else if (data->handler == HANDLER_ONMAX ||
6002                          data->handler == HANDLER_ONCHANGE)
6003                         print_track_data_spec(m, hist_data, data);
6004         }
6005 }
6006
6007 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
6008 {
6009         unsigned int i;
6010
6011         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6012                 kfree(hist_data->field_var_hists[i]->cmd);
6013                 kfree(hist_data->field_var_hists[i]);
6014         }
6015 }
6016
6017 static void destroy_hist_data(struct hist_trigger_data *hist_data)
6018 {
6019         if (!hist_data)
6020                 return;
6021
6022         destroy_hist_trigger_attrs(hist_data->attrs);
6023         destroy_hist_fields(hist_data);
6024         tracing_map_destroy(hist_data->map);
6025
6026         destroy_actions(hist_data);
6027         destroy_field_vars(hist_data);
6028         destroy_field_var_hists(hist_data);
6029
6030         kfree(hist_data);
6031 }
6032
6033 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
6034 {
6035         struct tracing_map *map = hist_data->map;
6036         struct ftrace_event_field *field;
6037         struct hist_field *hist_field;
6038         int i, idx = 0;
6039
6040         for_each_hist_field(i, hist_data) {
6041                 hist_field = hist_data->fields[i];
6042                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
6043                         tracing_map_cmp_fn_t cmp_fn;
6044
6045                         field = hist_field->field;
6046
6047                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
6048                                 cmp_fn = tracing_map_cmp_none;
6049                         else if (!field)
6050                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
6051                                                              hist_field->is_signed);
6052                         else if (is_string_field(field))
6053                                 cmp_fn = tracing_map_cmp_string;
6054                         else
6055                                 cmp_fn = tracing_map_cmp_num(field->size,
6056                                                              field->is_signed);
6057                         idx = tracing_map_add_key_field(map,
6058                                                         hist_field->offset,
6059                                                         cmp_fn);
6060                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
6061                         idx = tracing_map_add_sum_field(map);
6062
6063                 if (idx < 0)
6064                         return idx;
6065
6066                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6067                         idx = tracing_map_add_var(map);
6068                         if (idx < 0)
6069                                 return idx;
6070                         hist_field->var.idx = idx;
6071                         hist_field->var.hist_data = hist_data;
6072                 }
6073         }
6074
6075         return 0;
6076 }
6077
6078 static struct hist_trigger_data *
6079 create_hist_data(unsigned int map_bits,
6080                  struct hist_trigger_attrs *attrs,
6081                  struct trace_event_file *file,
6082                  bool remove)
6083 {
6084         const struct tracing_map_ops *map_ops = NULL;
6085         struct hist_trigger_data *hist_data;
6086         int ret = 0;
6087
6088         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
6089         if (!hist_data)
6090                 return ERR_PTR(-ENOMEM);
6091
6092         hist_data->attrs = attrs;
6093         hist_data->remove = remove;
6094         hist_data->event_file = file;
6095
6096         ret = parse_actions(hist_data);
6097         if (ret)
6098                 goto free;
6099
6100         ret = create_hist_fields(hist_data, file);
6101         if (ret)
6102                 goto free;
6103
6104         ret = create_sort_keys(hist_data);
6105         if (ret)
6106                 goto free;
6107
6108         map_ops = &hist_trigger_elt_data_ops;
6109
6110         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6111                                             map_ops, hist_data);
6112         if (IS_ERR(hist_data->map)) {
6113                 ret = PTR_ERR(hist_data->map);
6114                 hist_data->map = NULL;
6115                 goto free;
6116         }
6117
6118         ret = create_tracing_map_fields(hist_data);
6119         if (ret)
6120                 goto free;
6121  out:
6122         return hist_data;
6123  free:
6124         hist_data->attrs = NULL;
6125
6126         destroy_hist_data(hist_data);
6127
6128         hist_data = ERR_PTR(ret);
6129
6130         goto out;
6131 }
6132
6133 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
6134                                     struct tracing_map_elt *elt, void *rec,
6135                                     struct ring_buffer_event *rbe,
6136                                     u64 *var_ref_vals)
6137 {
6138         struct hist_elt_data *elt_data;
6139         struct hist_field *hist_field;
6140         unsigned int i, var_idx;
6141         u64 hist_val;
6142
6143         elt_data = elt->private_data;
6144         elt_data->var_ref_vals = var_ref_vals;
6145
6146         for_each_hist_val_field(i, hist_data) {
6147                 hist_field = hist_data->fields[i];
6148                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
6149                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6150                         var_idx = hist_field->var.idx;
6151                         tracing_map_set_var(elt, var_idx, hist_val);
6152                         continue;
6153                 }
6154                 tracing_map_update_sum(elt, i, hist_val);
6155         }
6156
6157         for_each_hist_key_field(i, hist_data) {
6158                 hist_field = hist_data->fields[i];
6159                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6160                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
6161                         var_idx = hist_field->var.idx;
6162                         tracing_map_set_var(elt, var_idx, hist_val);
6163                 }
6164         }
6165
6166         update_field_vars(hist_data, elt, rbe, rec);
6167 }
6168
6169 static inline void add_to_key(char *compound_key, void *key,
6170                               struct hist_field *key_field, void *rec)
6171 {
6172         size_t size = key_field->size;
6173
6174         if (key_field->flags & HIST_FIELD_FL_STRING) {
6175                 struct ftrace_event_field *field;
6176
6177                 field = key_field->field;
6178                 if (field->filter_type == FILTER_DYN_STRING)
6179                         size = *(u32 *)(rec + field->offset) >> 16;
6180                 else if (field->filter_type == FILTER_PTR_STRING)
6181                         size = strlen(key);
6182                 else if (field->filter_type == FILTER_STATIC_STRING)
6183                         size = field->size;
6184
6185                 /* ensure NULL-termination */
6186                 if (size > key_field->size - 1)
6187                         size = key_field->size - 1;
6188
6189                 strncpy(compound_key + key_field->offset, (char *)key, size);
6190         } else
6191                 memcpy(compound_key + key_field->offset, key, size);
6192 }
6193
6194 static void
6195 hist_trigger_actions(struct hist_trigger_data *hist_data,
6196                      struct tracing_map_elt *elt, void *rec,
6197                      struct ring_buffer_event *rbe, void *key,
6198                      u64 *var_ref_vals)
6199 {
6200         struct action_data *data;
6201         unsigned int i;
6202
6203         for (i = 0; i < hist_data->n_actions; i++) {
6204                 data = hist_data->actions[i];
6205                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
6206         }
6207 }
6208
6209 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
6210                                struct ring_buffer_event *rbe)
6211 {
6212         struct hist_trigger_data *hist_data = data->private_data;
6213         bool use_compound_key = (hist_data->n_keys > 1);
6214         unsigned long entries[HIST_STACKTRACE_DEPTH];
6215         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
6216         char compound_key[HIST_KEY_SIZE_MAX];
6217         struct tracing_map_elt *elt = NULL;
6218         struct hist_field *key_field;
6219         u64 field_contents;
6220         void *key = NULL;
6221         unsigned int i;
6222
6223         memset(compound_key, 0, hist_data->key_size);
6224
6225         for_each_hist_key_field(i, hist_data) {
6226                 key_field = hist_data->fields[i];
6227
6228                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
6229                         memset(entries, 0, HIST_STACKTRACE_SIZE);
6230                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
6231                                          HIST_STACKTRACE_SKIP);
6232                         key = entries;
6233                 } else {
6234                         field_contents = key_field->fn(key_field, elt, rbe, rec);
6235                         if (key_field->flags & HIST_FIELD_FL_STRING) {
6236                                 key = (void *)(unsigned long)field_contents;
6237                                 use_compound_key = true;
6238                         } else
6239                                 key = (void *)&field_contents;
6240                 }
6241
6242                 if (use_compound_key)
6243                         add_to_key(compound_key, key, key_field, rec);
6244         }
6245
6246         if (use_compound_key)
6247                 key = compound_key;
6248
6249         if (hist_data->n_var_refs &&
6250             !resolve_var_refs(hist_data, key, var_ref_vals, false))
6251                 return;
6252
6253         elt = tracing_map_insert(hist_data->map, key);
6254         if (!elt)
6255                 return;
6256
6257         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
6258
6259         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
6260                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
6261 }
6262
6263 static void hist_trigger_stacktrace_print(struct seq_file *m,
6264                                           unsigned long *stacktrace_entries,
6265                                           unsigned int max_entries)
6266 {
6267         char str[KSYM_SYMBOL_LEN];
6268         unsigned int spaces = 8;
6269         unsigned int i;
6270
6271         for (i = 0; i < max_entries; i++) {
6272                 if (!stacktrace_entries[i])
6273                         return;
6274
6275                 seq_printf(m, "%*c", 1 + spaces, ' ');
6276                 sprint_symbol(str, stacktrace_entries[i]);
6277                 seq_printf(m, "%s\n", str);
6278         }
6279 }
6280
6281 static void hist_trigger_print_key(struct seq_file *m,
6282                                    struct hist_trigger_data *hist_data,
6283                                    void *key,
6284                                    struct tracing_map_elt *elt)
6285 {
6286         struct hist_field *key_field;
6287         char str[KSYM_SYMBOL_LEN];
6288         bool multiline = false;
6289         const char *field_name;
6290         unsigned int i;
6291         u64 uval;
6292
6293         seq_puts(m, "{ ");
6294
6295         for_each_hist_key_field(i, hist_data) {
6296                 key_field = hist_data->fields[i];
6297
6298                 if (i > hist_data->n_vals)
6299                         seq_puts(m, ", ");
6300
6301                 field_name = hist_field_name(key_field, 0);
6302
6303                 if (key_field->flags & HIST_FIELD_FL_HEX) {
6304                         uval = *(u64 *)(key + key_field->offset);
6305                         seq_printf(m, "%s: %llx", field_name, uval);
6306                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
6307                         uval = *(u64 *)(key + key_field->offset);
6308                         sprint_symbol_no_offset(str, uval);
6309                         seq_printf(m, "%s: [%llx] %-45s", field_name,
6310                                    uval, str);
6311                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
6312                         uval = *(u64 *)(key + key_field->offset);
6313                         sprint_symbol(str, uval);
6314                         seq_printf(m, "%s: [%llx] %-55s", field_name,
6315                                    uval, str);
6316                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
6317                         struct hist_elt_data *elt_data = elt->private_data;
6318                         char *comm;
6319
6320                         if (WARN_ON_ONCE(!elt_data))
6321                                 return;
6322
6323                         comm = elt_data->comm;
6324
6325                         uval = *(u64 *)(key + key_field->offset);
6326                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
6327                                    comm, uval);
6328                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
6329                         const char *syscall_name;
6330
6331                         uval = *(u64 *)(key + key_field->offset);
6332                         syscall_name = get_syscall_name(uval);
6333                         if (!syscall_name)
6334                                 syscall_name = "unknown_syscall";
6335
6336                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
6337                                    syscall_name, uval);
6338                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
6339                         seq_puts(m, "stacktrace:\n");
6340                         hist_trigger_stacktrace_print(m,
6341                                                       key + key_field->offset,
6342                                                       HIST_STACKTRACE_DEPTH);
6343                         multiline = true;
6344                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
6345                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
6346                                    *(u64 *)(key + key_field->offset));
6347                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
6348                         seq_printf(m, "%s: %-50s", field_name,
6349                                    (char *)(key + key_field->offset));
6350                 } else {
6351                         uval = *(u64 *)(key + key_field->offset);
6352                         seq_printf(m, "%s: %10llu", field_name, uval);
6353                 }
6354         }
6355
6356         if (!multiline)
6357                 seq_puts(m, " ");
6358
6359         seq_puts(m, "}");
6360 }
6361
6362 static void hist_trigger_entry_print(struct seq_file *m,
6363                                      struct hist_trigger_data *hist_data,
6364                                      void *key,
6365                                      struct tracing_map_elt *elt)
6366 {
6367         const char *field_name;
6368         unsigned int i;
6369
6370         hist_trigger_print_key(m, hist_data, key, elt);
6371
6372         seq_printf(m, " hitcount: %10llu",
6373                    tracing_map_read_sum(elt, HITCOUNT_IDX));
6374
6375         for (i = 1; i < hist_data->n_vals; i++) {
6376                 field_name = hist_field_name(hist_data->fields[i], 0);
6377
6378                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
6379                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
6380                         continue;
6381
6382                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
6383                         seq_printf(m, "  %s: %10llx", field_name,
6384                                    tracing_map_read_sum(elt, i));
6385                 } else {
6386                         seq_printf(m, "  %s: %10llu", field_name,
6387                                    tracing_map_read_sum(elt, i));
6388                 }
6389         }
6390
6391         print_actions(m, hist_data, elt);
6392
6393         seq_puts(m, "\n");
6394 }
6395
6396 static int print_entries(struct seq_file *m,
6397                          struct hist_trigger_data *hist_data)
6398 {
6399         struct tracing_map_sort_entry **sort_entries = NULL;
6400         struct tracing_map *map = hist_data->map;
6401         int i, n_entries;
6402
6403         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
6404                                              hist_data->n_sort_keys,
6405                                              &sort_entries);
6406         if (n_entries < 0)
6407                 return n_entries;
6408
6409         for (i = 0; i < n_entries; i++)
6410                 hist_trigger_entry_print(m, hist_data,
6411                                          sort_entries[i]->key,
6412                                          sort_entries[i]->elt);
6413
6414         tracing_map_destroy_sort_entries(sort_entries, n_entries);
6415
6416         return n_entries;
6417 }
6418
6419 static void hist_trigger_show(struct seq_file *m,
6420                               struct event_trigger_data *data, int n)
6421 {
6422         struct hist_trigger_data *hist_data;
6423         int n_entries;
6424
6425         if (n > 0)
6426                 seq_puts(m, "\n\n");
6427
6428         seq_puts(m, "# event histogram\n#\n# trigger info: ");
6429         data->ops->print(m, data->ops, data);
6430         seq_puts(m, "#\n\n");
6431
6432         hist_data = data->private_data;
6433         n_entries = print_entries(m, hist_data);
6434         if (n_entries < 0)
6435                 n_entries = 0;
6436
6437         track_data_snapshot_print(m, hist_data);
6438
6439         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
6440                    (u64)atomic64_read(&hist_data->map->hits),
6441                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
6442 }
6443
6444 static int hist_show(struct seq_file *m, void *v)
6445 {
6446         struct event_trigger_data *data;
6447         struct trace_event_file *event_file;
6448         int n = 0, ret = 0;
6449
6450         mutex_lock(&event_mutex);
6451
6452         event_file = event_file_data(m->private);
6453         if (unlikely(!event_file)) {
6454                 ret = -ENODEV;
6455                 goto out_unlock;
6456         }
6457
6458         list_for_each_entry(data, &event_file->triggers, list) {
6459                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
6460                         hist_trigger_show(m, data, n++);
6461         }
6462
6463  out_unlock:
6464         mutex_unlock(&event_mutex);
6465
6466         return ret;
6467 }
6468
6469 static int event_hist_open(struct inode *inode, struct file *file)
6470 {
6471         int ret;
6472
6473         ret = security_locked_down(LOCKDOWN_TRACEFS);
6474         if (ret)
6475                 return ret;
6476
6477         return single_open(file, hist_show, file);
6478 }
6479
6480 const struct file_operations event_hist_fops = {
6481         .open = event_hist_open,
6482         .read = seq_read,
6483         .llseek = seq_lseek,
6484         .release = single_release,
6485 };
6486
6487 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
6488 {
6489         const char *field_name = hist_field_name(hist_field, 0);
6490
6491         if (hist_field->var.name)
6492                 seq_printf(m, "%s=", hist_field->var.name);
6493
6494         if (hist_field->flags & HIST_FIELD_FL_CPU)
6495                 seq_puts(m, "cpu");
6496         else if (field_name) {
6497                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
6498                     hist_field->flags & HIST_FIELD_FL_ALIAS)
6499                         seq_putc(m, '$');
6500                 seq_printf(m, "%s", field_name);
6501         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
6502                 seq_puts(m, "common_timestamp");
6503
6504         if (hist_field->flags) {
6505                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
6506                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
6507                         const char *flags = get_hist_field_flags(hist_field);
6508
6509                         if (flags)
6510                                 seq_printf(m, ".%s", flags);
6511                 }
6512         }
6513 }
6514
6515 static int event_hist_trigger_print(struct seq_file *m,
6516                                     struct event_trigger_ops *ops,
6517                                     struct event_trigger_data *data)
6518 {
6519         struct hist_trigger_data *hist_data = data->private_data;
6520         struct hist_field *field;
6521         bool have_var = false;
6522         unsigned int i;
6523
6524         seq_puts(m, "hist:");
6525
6526         if (data->name)
6527                 seq_printf(m, "%s:", data->name);
6528
6529         seq_puts(m, "keys=");
6530
6531         for_each_hist_key_field(i, hist_data) {
6532                 field = hist_data->fields[i];
6533
6534                 if (i > hist_data->n_vals)
6535                         seq_puts(m, ",");
6536
6537                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
6538                         seq_puts(m, "stacktrace");
6539                 else
6540                         hist_field_print(m, field);
6541         }
6542
6543         seq_puts(m, ":vals=");
6544
6545         for_each_hist_val_field(i, hist_data) {
6546                 field = hist_data->fields[i];
6547                 if (field->flags & HIST_FIELD_FL_VAR) {
6548                         have_var = true;
6549                         continue;
6550                 }
6551
6552                 if (i == HITCOUNT_IDX)
6553                         seq_puts(m, "hitcount");
6554                 else {
6555                         seq_puts(m, ",");
6556                         hist_field_print(m, field);
6557                 }
6558         }
6559
6560         if (have_var) {
6561                 unsigned int n = 0;
6562
6563                 seq_puts(m, ":");
6564
6565                 for_each_hist_val_field(i, hist_data) {
6566                         field = hist_data->fields[i];
6567
6568                         if (field->flags & HIST_FIELD_FL_VAR) {
6569                                 if (n++)
6570                                         seq_puts(m, ",");
6571                                 hist_field_print(m, field);
6572                         }
6573                 }
6574         }
6575
6576         seq_puts(m, ":sort=");
6577
6578         for (i = 0; i < hist_data->n_sort_keys; i++) {
6579                 struct tracing_map_sort_key *sort_key;
6580                 unsigned int idx, first_key_idx;
6581
6582                 /* skip VAR vals */
6583                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
6584
6585                 sort_key = &hist_data->sort_keys[i];
6586                 idx = sort_key->field_idx;
6587
6588                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
6589                         return -EINVAL;
6590
6591                 if (i > 0)
6592                         seq_puts(m, ",");
6593
6594                 if (idx == HITCOUNT_IDX)
6595                         seq_puts(m, "hitcount");
6596                 else {
6597                         if (idx >= first_key_idx)
6598                                 idx += hist_data->n_vars;
6599                         hist_field_print(m, hist_data->fields[idx]);
6600                 }
6601
6602                 if (sort_key->descending)
6603                         seq_puts(m, ".descending");
6604         }
6605         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6606         if (hist_data->enable_timestamps)
6607                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6608
6609         print_actions_spec(m, hist_data);
6610
6611         if (data->filter_str)
6612                 seq_printf(m, " if %s", data->filter_str);
6613
6614         if (data->paused)
6615                 seq_puts(m, " [paused]");
6616         else
6617                 seq_puts(m, " [active]");
6618
6619         seq_putc(m, '\n');
6620
6621         return 0;
6622 }
6623
6624 static int event_hist_trigger_init(struct event_trigger_ops *ops,
6625                                    struct event_trigger_data *data)
6626 {
6627         struct hist_trigger_data *hist_data = data->private_data;
6628
6629         if (!data->ref && hist_data->attrs->name)
6630                 save_named_trigger(hist_data->attrs->name, data);
6631
6632         data->ref++;
6633
6634         return 0;
6635 }
6636
6637 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6638 {
6639         struct trace_event_file *file;
6640         unsigned int i;
6641         char *cmd;
6642         int ret;
6643
6644         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6645                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6646                 cmd = hist_data->field_var_hists[i]->cmd;
6647                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
6648                                               "!hist", "hist", cmd);
6649         }
6650 }
6651
6652 static void event_hist_trigger_free(struct event_trigger_ops *ops,
6653                                     struct event_trigger_data *data)
6654 {
6655         struct hist_trigger_data *hist_data = data->private_data;
6656
6657         if (WARN_ON_ONCE(data->ref <= 0))
6658                 return;
6659
6660         data->ref--;
6661         if (!data->ref) {
6662                 if (data->name)
6663                         del_named_trigger(data);
6664
6665                 trigger_data_free(data);
6666
6667                 remove_hist_vars(hist_data);
6668
6669                 unregister_field_var_hists(hist_data);
6670
6671                 destroy_hist_data(hist_data);
6672         }
6673 }
6674
6675 static struct event_trigger_ops event_hist_trigger_ops = {
6676         .func                   = event_hist_trigger,
6677         .print                  = event_hist_trigger_print,
6678         .init                   = event_hist_trigger_init,
6679         .free                   = event_hist_trigger_free,
6680 };
6681
6682 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
6683                                          struct event_trigger_data *data)
6684 {
6685         data->ref++;
6686
6687         save_named_trigger(data->named_data->name, data);
6688
6689         event_hist_trigger_init(ops, data->named_data);
6690
6691         return 0;
6692 }
6693
6694 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
6695                                           struct event_trigger_data *data)
6696 {
6697         if (WARN_ON_ONCE(data->ref <= 0))
6698                 return;
6699
6700         event_hist_trigger_free(ops, data->named_data);
6701
6702         data->ref--;
6703         if (!data->ref) {
6704                 del_named_trigger(data);
6705                 trigger_data_free(data);
6706         }
6707 }
6708
6709 static struct event_trigger_ops event_hist_trigger_named_ops = {
6710         .func                   = event_hist_trigger,
6711         .print                  = event_hist_trigger_print,
6712         .init                   = event_hist_trigger_named_init,
6713         .free                   = event_hist_trigger_named_free,
6714 };
6715
6716 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6717                                                             char *param)
6718 {
6719         return &event_hist_trigger_ops;
6720 }
6721
6722 static void hist_clear(struct event_trigger_data *data)
6723 {
6724         struct hist_trigger_data *hist_data = data->private_data;
6725
6726         if (data->name)
6727                 pause_named_trigger(data);
6728
6729         tracepoint_synchronize_unregister();
6730
6731         tracing_map_clear(hist_data->map);
6732
6733         if (data->name)
6734                 unpause_named_trigger(data);
6735 }
6736
6737 static bool compatible_field(struct ftrace_event_field *field,
6738                              struct ftrace_event_field *test_field)
6739 {
6740         if (field == test_field)
6741                 return true;
6742         if (field == NULL || test_field == NULL)
6743                 return false;
6744         if (strcmp(field->name, test_field->name) != 0)
6745                 return false;
6746         if (strcmp(field->type, test_field->type) != 0)
6747                 return false;
6748         if (field->size != test_field->size)
6749                 return false;
6750         if (field->is_signed != test_field->is_signed)
6751                 return false;
6752
6753         return true;
6754 }
6755
6756 static bool hist_trigger_match(struct event_trigger_data *data,
6757                                struct event_trigger_data *data_test,
6758                                struct event_trigger_data *named_data,
6759                                bool ignore_filter)
6760 {
6761         struct tracing_map_sort_key *sort_key, *sort_key_test;
6762         struct hist_trigger_data *hist_data, *hist_data_test;
6763         struct hist_field *key_field, *key_field_test;
6764         unsigned int i;
6765
6766         if (named_data && (named_data != data_test) &&
6767             (named_data != data_test->named_data))
6768                 return false;
6769
6770         if (!named_data && is_named_trigger(data_test))
6771                 return false;
6772
6773         hist_data = data->private_data;
6774         hist_data_test = data_test->private_data;
6775
6776         if (hist_data->n_vals != hist_data_test->n_vals ||
6777             hist_data->n_fields != hist_data_test->n_fields ||
6778             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6779                 return false;
6780
6781         if (!ignore_filter) {
6782                 if ((data->filter_str && !data_test->filter_str) ||
6783                    (!data->filter_str && data_test->filter_str))
6784                         return false;
6785         }
6786
6787         for_each_hist_field(i, hist_data) {
6788                 key_field = hist_data->fields[i];
6789                 key_field_test = hist_data_test->fields[i];
6790
6791                 if (key_field->flags != key_field_test->flags)
6792                         return false;
6793                 if (!compatible_field(key_field->field, key_field_test->field))
6794                         return false;
6795                 if (key_field->offset != key_field_test->offset)
6796                         return false;
6797                 if (key_field->size != key_field_test->size)
6798                         return false;
6799                 if (key_field->is_signed != key_field_test->is_signed)
6800                         return false;
6801                 if (!!key_field->var.name != !!key_field_test->var.name)
6802                         return false;
6803                 if (key_field->var.name &&
6804                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6805                         return false;
6806         }
6807
6808         for (i = 0; i < hist_data->n_sort_keys; i++) {
6809                 sort_key = &hist_data->sort_keys[i];
6810                 sort_key_test = &hist_data_test->sort_keys[i];
6811
6812                 if (sort_key->field_idx != sort_key_test->field_idx ||
6813                     sort_key->descending != sort_key_test->descending)
6814                         return false;
6815         }
6816
6817         if (!ignore_filter && data->filter_str &&
6818             (strcmp(data->filter_str, data_test->filter_str) != 0))
6819                 return false;
6820
6821         if (!actions_match(hist_data, hist_data_test))
6822                 return false;
6823
6824         return true;
6825 }
6826
6827 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
6828                                  struct event_trigger_data *data,
6829                                  struct trace_event_file *file)
6830 {
6831         struct hist_trigger_data *hist_data = data->private_data;
6832         struct event_trigger_data *test, *named_data = NULL;
6833         struct trace_array *tr = file->tr;
6834         int ret = 0;
6835
6836         if (hist_data->attrs->name) {
6837                 named_data = find_named_trigger(hist_data->attrs->name);
6838                 if (named_data) {
6839                         if (!hist_trigger_match(data, named_data, named_data,
6840                                                 true)) {
6841                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6842                                 ret = -EINVAL;
6843                                 goto out;
6844                         }
6845                 }
6846         }
6847
6848         if (hist_data->attrs->name && !named_data)
6849                 goto new;
6850
6851         lockdep_assert_held(&event_mutex);
6852
6853         list_for_each_entry(test, &file->triggers, list) {
6854                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6855                         if (!hist_trigger_match(data, test, named_data, false))
6856                                 continue;
6857                         if (hist_data->attrs->pause)
6858                                 test->paused = true;
6859                         else if (hist_data->attrs->cont)
6860                                 test->paused = false;
6861                         else if (hist_data->attrs->clear)
6862                                 hist_clear(test);
6863                         else {
6864                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6865                                 ret = -EEXIST;
6866                         }
6867                         goto out;
6868                 }
6869         }
6870  new:
6871         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6872                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6873                 ret = -ENOENT;
6874                 goto out;
6875         }
6876
6877         if (hist_data->attrs->pause)
6878                 data->paused = true;
6879
6880         if (named_data) {
6881                 data->private_data = named_data->private_data;
6882                 set_named_trigger_data(data, named_data);
6883                 data->ops = &event_hist_trigger_named_ops;
6884         }
6885
6886         if (data->ops->init) {
6887                 ret = data->ops->init(data->ops, data);
6888                 if (ret < 0)
6889                         goto out;
6890         }
6891
6892         if (hist_data->enable_timestamps) {
6893                 char *clock = hist_data->attrs->clock;
6894
6895                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6896                 if (ret) {
6897                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6898                         goto out;
6899                 }
6900
6901                 tracing_set_time_stamp_abs(file->tr, true);
6902         }
6903
6904         if (named_data)
6905                 destroy_hist_data(hist_data);
6906
6907         ret++;
6908  out:
6909         return ret;
6910 }
6911
6912 static int hist_trigger_enable(struct event_trigger_data *data,
6913                                struct trace_event_file *file)
6914 {
6915         int ret = 0;
6916
6917         list_add_tail_rcu(&data->list, &file->triggers);
6918
6919         update_cond_flag(file);
6920
6921         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6922                 list_del_rcu(&data->list);
6923                 update_cond_flag(file);
6924                 ret--;
6925         }
6926
6927         return ret;
6928 }
6929
6930 static bool have_hist_trigger_match(struct event_trigger_data *data,
6931                                     struct trace_event_file *file)
6932 {
6933         struct hist_trigger_data *hist_data = data->private_data;
6934         struct event_trigger_data *test, *named_data = NULL;
6935         bool match = false;
6936
6937         lockdep_assert_held(&event_mutex);
6938
6939         if (hist_data->attrs->name)
6940                 named_data = find_named_trigger(hist_data->attrs->name);
6941
6942         list_for_each_entry(test, &file->triggers, list) {
6943                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6944                         if (hist_trigger_match(data, test, named_data, false)) {
6945                                 match = true;
6946                                 break;
6947                         }
6948                 }
6949         }
6950
6951         return match;
6952 }
6953
6954 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6955                                     struct trace_event_file *file)
6956 {
6957         struct hist_trigger_data *hist_data = data->private_data;
6958         struct event_trigger_data *test, *named_data = NULL;
6959
6960         lockdep_assert_held(&event_mutex);
6961
6962         if (hist_data->attrs->name)
6963                 named_data = find_named_trigger(hist_data->attrs->name);
6964
6965         list_for_each_entry(test, &file->triggers, list) {
6966                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6967                         if (!hist_trigger_match(data, test, named_data, false))
6968                                 continue;
6969                         hist_data = test->private_data;
6970                         if (check_var_refs(hist_data))
6971                                 return true;
6972                         break;
6973                 }
6974         }
6975
6976         return false;
6977 }
6978
6979 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6980                                     struct event_trigger_data *data,
6981                                     struct trace_event_file *file)
6982 {
6983         struct hist_trigger_data *hist_data = data->private_data;
6984         struct event_trigger_data *test, *named_data = NULL;
6985         bool unregistered = false;
6986
6987         lockdep_assert_held(&event_mutex);
6988
6989         if (hist_data->attrs->name)
6990                 named_data = find_named_trigger(hist_data->attrs->name);
6991
6992         list_for_each_entry(test, &file->triggers, list) {
6993                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6994                         if (!hist_trigger_match(data, test, named_data, false))
6995                                 continue;
6996                         unregistered = true;
6997                         list_del_rcu(&test->list);
6998                         trace_event_trigger_enable_disable(file, 0);
6999                         update_cond_flag(file);
7000                         break;
7001                 }
7002         }
7003
7004         if (unregistered && test->ops->free)
7005                 test->ops->free(test->ops, test);
7006
7007         if (hist_data->enable_timestamps) {
7008                 if (!hist_data->remove || unregistered)
7009                         tracing_set_time_stamp_abs(file->tr, false);
7010         }
7011 }
7012
7013 static bool hist_file_check_refs(struct trace_event_file *file)
7014 {
7015         struct hist_trigger_data *hist_data;
7016         struct event_trigger_data *test;
7017
7018         lockdep_assert_held(&event_mutex);
7019
7020         list_for_each_entry(test, &file->triggers, list) {
7021                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7022                         hist_data = test->private_data;
7023                         if (check_var_refs(hist_data))
7024                                 return true;
7025                 }
7026         }
7027
7028         return false;
7029 }
7030
7031 static void hist_unreg_all(struct trace_event_file *file)
7032 {
7033         struct event_trigger_data *test, *n;
7034         struct hist_trigger_data *hist_data;
7035         struct synth_event *se;
7036         const char *se_name;
7037
7038         lockdep_assert_held(&event_mutex);
7039
7040         if (hist_file_check_refs(file))
7041                 return;
7042
7043         list_for_each_entry_safe(test, n, &file->triggers, list) {
7044                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7045                         hist_data = test->private_data;
7046                         list_del_rcu(&test->list);
7047                         trace_event_trigger_enable_disable(file, 0);
7048
7049                         se_name = trace_event_name(file->event_call);
7050                         se = find_synth_event(se_name);
7051                         if (se)
7052                                 se->ref--;
7053
7054                         update_cond_flag(file);
7055                         if (hist_data->enable_timestamps)
7056                                 tracing_set_time_stamp_abs(file->tr, false);
7057                         if (test->ops->free)
7058                                 test->ops->free(test->ops, test);
7059                 }
7060         }
7061 }
7062
7063 static int event_hist_trigger_func(struct event_command *cmd_ops,
7064                                    struct trace_event_file *file,
7065                                    char *glob, char *cmd, char *param)
7066 {
7067         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
7068         struct event_trigger_data *trigger_data;
7069         struct hist_trigger_attrs *attrs;
7070         struct event_trigger_ops *trigger_ops;
7071         struct hist_trigger_data *hist_data;
7072         struct synth_event *se;
7073         const char *se_name;
7074         bool remove = false;
7075         char *trigger, *p;
7076         int ret = 0;
7077
7078         lockdep_assert_held(&event_mutex);
7079
7080         if (glob && strlen(glob)) {
7081                 hist_err_clear();
7082                 last_cmd_set(file, param);
7083         }
7084
7085         if (!param)
7086                 return -EINVAL;
7087
7088         if (glob[0] == '!')
7089                 remove = true;
7090
7091         /*
7092          * separate the trigger from the filter (k:v [if filter])
7093          * allowing for whitespace in the trigger
7094          */
7095         p = trigger = param;
7096         do {
7097                 p = strstr(p, "if");
7098                 if (!p)
7099                         break;
7100                 if (p == param)
7101                         return -EINVAL;
7102                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
7103                         p++;
7104                         continue;
7105                 }
7106                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
7107                         return -EINVAL;
7108                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
7109                         p++;
7110                         continue;
7111                 }
7112                 break;
7113         } while (p);
7114
7115         if (!p)
7116                 param = NULL;
7117         else {
7118                 *(p - 1) = '\0';
7119                 param = strstrip(p);
7120                 trigger = strstrip(trigger);
7121         }
7122
7123         attrs = parse_hist_trigger_attrs(file->tr, trigger);
7124         if (IS_ERR(attrs))
7125                 return PTR_ERR(attrs);
7126
7127         if (attrs->map_bits)
7128                 hist_trigger_bits = attrs->map_bits;
7129
7130         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7131         if (IS_ERR(hist_data)) {
7132                 destroy_hist_trigger_attrs(attrs);
7133                 return PTR_ERR(hist_data);
7134         }
7135
7136         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
7137
7138         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
7139         if (!trigger_data) {
7140                 ret = -ENOMEM;
7141                 goto out_free;
7142         }
7143
7144         trigger_data->count = -1;
7145         trigger_data->ops = trigger_ops;
7146         trigger_data->cmd_ops = cmd_ops;
7147
7148         INIT_LIST_HEAD(&trigger_data->list);
7149         RCU_INIT_POINTER(trigger_data->filter, NULL);
7150
7151         trigger_data->private_data = hist_data;
7152
7153         /* if param is non-empty, it's supposed to be a filter */
7154         if (param && cmd_ops->set_filter) {
7155                 ret = cmd_ops->set_filter(param, trigger_data, file);
7156                 if (ret < 0)
7157                         goto out_free;
7158         }
7159
7160         if (remove) {
7161                 if (!have_hist_trigger_match(trigger_data, file))
7162                         goto out_free;
7163
7164                 if (hist_trigger_check_refs(trigger_data, file)) {
7165                         ret = -EBUSY;
7166                         goto out_free;
7167                 }
7168
7169                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7170                 se_name = trace_event_name(file->event_call);
7171                 se = find_synth_event(se_name);
7172                 if (se)
7173                         se->ref--;
7174                 ret = 0;
7175                 goto out_free;
7176         }
7177
7178         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
7179         /*
7180          * The above returns on success the # of triggers registered,
7181          * but if it didn't register any it returns zero.  Consider no
7182          * triggers registered a failure too.
7183          */
7184         if (!ret) {
7185                 if (!(attrs->pause || attrs->cont || attrs->clear))
7186                         ret = -ENOENT;
7187                 goto out_free;
7188         } else if (ret < 0)
7189                 goto out_free;
7190
7191         if (get_named_trigger_data(trigger_data))
7192                 goto enable;
7193
7194         if (has_hist_vars(hist_data))
7195                 save_hist_vars(hist_data);
7196
7197         ret = create_actions(hist_data);
7198         if (ret)
7199                 goto out_unreg;
7200
7201         ret = tracing_map_init(hist_data->map);
7202         if (ret)
7203                 goto out_unreg;
7204 enable:
7205         ret = hist_trigger_enable(trigger_data, file);
7206         if (ret)
7207                 goto out_unreg;
7208
7209         se_name = trace_event_name(file->event_call);
7210         se = find_synth_event(se_name);
7211         if (se)
7212                 se->ref++;
7213         /* Just return zero, not the number of registered triggers */
7214         ret = 0;
7215  out:
7216         if (ret == 0)
7217                 hist_err_clear();
7218
7219         return ret;
7220  out_unreg:
7221         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7222  out_free:
7223         if (cmd_ops->set_filter)
7224                 cmd_ops->set_filter(NULL, trigger_data, NULL);
7225
7226         remove_hist_vars(hist_data);
7227
7228         kfree(trigger_data);
7229
7230         destroy_hist_data(hist_data);
7231         goto out;
7232 }
7233
7234 static struct event_command trigger_hist_cmd = {
7235         .name                   = "hist",
7236         .trigger_type           = ETT_EVENT_HIST,
7237         .flags                  = EVENT_CMD_FL_NEEDS_REC,
7238         .func                   = event_hist_trigger_func,
7239         .reg                    = hist_register_trigger,
7240         .unreg                  = hist_unregister_trigger,
7241         .unreg_all              = hist_unreg_all,
7242         .get_trigger_ops        = event_hist_get_trigger_ops,
7243         .set_filter             = set_trigger_filter,
7244 };
7245
7246 __init int register_trigger_hist_cmd(void)
7247 {
7248         int ret;
7249
7250         ret = register_event_command(&trigger_hist_cmd);
7251         WARN_ON(ret < 0);
7252
7253         return ret;
7254 }
7255
7256 static void
7257 hist_enable_trigger(struct event_trigger_data *data, void *rec,
7258                     struct ring_buffer_event *event)
7259 {
7260         struct enable_trigger_data *enable_data = data->private_data;
7261         struct event_trigger_data *test;
7262
7263         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
7264                                 lockdep_is_held(&event_mutex)) {
7265                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7266                         if (enable_data->enable)
7267                                 test->paused = false;
7268                         else
7269                                 test->paused = true;
7270                 }
7271         }
7272 }
7273
7274 static void
7275 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
7276                           struct ring_buffer_event *event)
7277 {
7278         if (!data->count)
7279                 return;
7280
7281         if (data->count != -1)
7282                 (data->count)--;
7283
7284         hist_enable_trigger(data, rec, event);
7285 }
7286
7287 static struct event_trigger_ops hist_enable_trigger_ops = {
7288         .func                   = hist_enable_trigger,
7289         .print                  = event_enable_trigger_print,
7290         .init                   = event_trigger_init,
7291         .free                   = event_enable_trigger_free,
7292 };
7293
7294 static struct event_trigger_ops hist_enable_count_trigger_ops = {
7295         .func                   = hist_enable_count_trigger,
7296         .print                  = event_enable_trigger_print,
7297         .init                   = event_trigger_init,
7298         .free                   = event_enable_trigger_free,
7299 };
7300
7301 static struct event_trigger_ops hist_disable_trigger_ops = {
7302         .func                   = hist_enable_trigger,
7303         .print                  = event_enable_trigger_print,
7304         .init                   = event_trigger_init,
7305         .free                   = event_enable_trigger_free,
7306 };
7307
7308 static struct event_trigger_ops hist_disable_count_trigger_ops = {
7309         .func                   = hist_enable_count_trigger,
7310         .print                  = event_enable_trigger_print,
7311         .init                   = event_trigger_init,
7312         .free                   = event_enable_trigger_free,
7313 };
7314
7315 static struct event_trigger_ops *
7316 hist_enable_get_trigger_ops(char *cmd, char *param)
7317 {
7318         struct event_trigger_ops *ops;
7319         bool enable;
7320
7321         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
7322
7323         if (enable)
7324                 ops = param ? &hist_enable_count_trigger_ops :
7325                         &hist_enable_trigger_ops;
7326         else
7327                 ops = param ? &hist_disable_count_trigger_ops :
7328                         &hist_disable_trigger_ops;
7329
7330         return ops;
7331 }
7332
7333 static void hist_enable_unreg_all(struct trace_event_file *file)
7334 {
7335         struct event_trigger_data *test, *n;
7336
7337         list_for_each_entry_safe(test, n, &file->triggers, list) {
7338                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
7339                         list_del_rcu(&test->list);
7340                         update_cond_flag(file);
7341                         trace_event_trigger_enable_disable(file, 0);
7342                         if (test->ops->free)
7343                                 test->ops->free(test->ops, test);
7344                 }
7345         }
7346 }
7347
7348 static struct event_command trigger_hist_enable_cmd = {
7349         .name                   = ENABLE_HIST_STR,
7350         .trigger_type           = ETT_HIST_ENABLE,
7351         .func                   = event_enable_trigger_func,
7352         .reg                    = event_enable_register_trigger,
7353         .unreg                  = event_enable_unregister_trigger,
7354         .unreg_all              = hist_enable_unreg_all,
7355         .get_trigger_ops        = hist_enable_get_trigger_ops,
7356         .set_filter             = set_trigger_filter,
7357 };
7358
7359 static struct event_command trigger_hist_disable_cmd = {
7360         .name                   = DISABLE_HIST_STR,
7361         .trigger_type           = ETT_HIST_ENABLE,
7362         .func                   = event_enable_trigger_func,
7363         .reg                    = event_enable_register_trigger,
7364         .unreg                  = event_enable_unregister_trigger,
7365         .unreg_all              = hist_enable_unreg_all,
7366         .get_trigger_ops        = hist_enable_get_trigger_ops,
7367         .set_filter             = set_trigger_filter,
7368 };
7369
7370 static __init void unregister_trigger_hist_enable_disable_cmds(void)
7371 {
7372         unregister_event_command(&trigger_hist_enable_cmd);
7373         unregister_event_command(&trigger_hist_disable_cmd);
7374 }
7375
7376 __init int register_trigger_hist_enable_disable_cmds(void)
7377 {
7378         int ret;
7379
7380         ret = register_event_command(&trigger_hist_enable_cmd);
7381         if (WARN_ON(ret < 0))
7382                 return ret;
7383         ret = register_event_command(&trigger_hist_disable_cmd);
7384         if (WARN_ON(ret < 0))
7385                 unregister_trigger_hist_enable_disable_cmds();
7386
7387         return ret;
7388 }
7389
7390 static __init int trace_events_hist_init(void)
7391 {
7392         struct dentry *entry = NULL;
7393         struct dentry *d_tracer;
7394         int err = 0;
7395
7396         err = dyn_event_register(&synth_event_ops);
7397         if (err) {
7398                 pr_warn("Could not register synth_event_ops\n");
7399                 return err;
7400         }
7401
7402         d_tracer = tracing_init_dentry();
7403         if (IS_ERR(d_tracer)) {
7404                 err = PTR_ERR(d_tracer);
7405                 goto err;
7406         }
7407
7408         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
7409                                     NULL, &synth_events_fops);
7410         if (!entry) {
7411                 err = -ENODEV;
7412                 goto err;
7413         }
7414
7415         return err;
7416  err:
7417         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
7418
7419         return err;
7420 }
7421
7422 fs_initcall(trace_events_hist_init);