kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace.h"
  23 #include "trace_dynevent.h"
  24
  25 #define SYNTH_SYSTEM            "synthetic"
  26 #define SYNTH_FIELDS_MAX        32
  27
  28 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  29
  30 #define ERRORS                                                          \
  31         C(NONE,                 "No error"),                            \
  32         C(DUPLICATE_VAR,        "Variable already defined"),            \
  33         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  34         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  35         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  36         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  37         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  38         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  39         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  40         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  41         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  42         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  43         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  44         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  45         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  46         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  47         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  48         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  49         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  50         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  51         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  52         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  53         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  54         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  55         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  56         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  57         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  58         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  59         C(INVALID_PARAM,        "Invalid action param"),                \
  60         C(ACTION_NOT_FOUND,     "No action found"),                     \
  61         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  62         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  63         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  64         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  65         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  66         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  67         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  68         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  69         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  70         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  71         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  72         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  73         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  74         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  75
  76 #undef C
  77 #define C(a, b)         HIST_ERR_##a
  78
  79 enum { ERRORS };
  80
  81 #undef C
  82 #define C(a, b)         b
  83
  84 static const char *err_text[] = { ERRORS };
  85
  86 struct hist_field;
  87
  88 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  89                                 struct tracing_map_elt *elt,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96
  97 enum field_op_id {
  98         FIELD_OP_NONE,
  99         FIELD_OP_PLUS,
 100         FIELD_OP_MINUS,
 101         FIELD_OP_UNARY_MINUS,
 102 };
 103
 104 /*
 105  * A hist_var (histogram variable) contains variable information for
 106  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 107  * flag set.  A hist_var has a variable name e.g. ts0, and is
 108  * associated with a given histogram trigger, as specified by
 109  * hist_data.  The hist_var idx is the unique index assigned to the
 110  * variable by the hist trigger's tracing_map.  The idx is what is
 111  * used to set a variable's value and, by a variable reference, to
 112  * retrieve it.
 113  */
 114 struct hist_var {
 115         char                            *name;
 116         struct hist_trigger_data        *hist_data;
 117         unsigned int                    idx;
 118 };
 119
 120 struct hist_field {
 121         struct ftrace_event_field       *field;
 122         unsigned long                   flags;
 123         hist_field_fn_t                 fn;
 124         unsigned int                    ref;
 125         unsigned int                    size;
 126         unsigned int                    offset;
 127         unsigned int                    is_signed;
 128         const char                      *type;
 129         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 130         struct hist_trigger_data        *hist_data;
 131
 132         /*
 133          * Variable fields contain variable-specific info in var.
 134          */
 135         struct hist_var                 var;
 136         enum field_op_id                operator;
 137         char                            *system;
 138         char                            *event_name;
 139
 140         /*
 141          * The name field is used for EXPR and VAR_REF fields.  VAR
 142          * fields contain the variable name in var.name.
 143          */
 144         char                            *name;
 145
 146         /*
 147          * When a histogram trigger is hit, if it has any references
 148          * to variables, the values of those variables are collected
 149          * into a var_ref_vals array by resolve_var_refs().  The
 150          * current value of each variable is read from the tracing_map
 151          * using the hist field's hist_var.idx and entered into the
 152          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 153          */
 154         unsigned int                    var_ref_idx;
 155         bool                            read_once;
 156 };
 157
 158 static u64 hist_field_none(struct hist_field *field,
 159                            struct tracing_map_elt *elt,
 160                            struct ring_buffer_event *rbe,
 161                            void *event)
 162 {
 163         return 0;
 164 }
 165
 166 static u64 hist_field_counter(struct hist_field *field,
 167                               struct tracing_map_elt *elt,
 168                               struct ring_buffer_event *rbe,
 169                               void *event)
 170 {
 171         return 1;
 172 }
 173
 174 static u64 hist_field_string(struct hist_field *hist_field,
 175                              struct tracing_map_elt *elt,
 176                              struct ring_buffer_event *rbe,
 177                              void *event)
 178 {
 179         char *addr = (char *)(event + hist_field->field->offset);
 180
 181         return (u64)(unsigned long)addr;
 182 }
 183
 184 static u64 hist_field_dynstring(struct hist_field *hist_field,
 185                                 struct tracing_map_elt *elt,
 186                                 struct ring_buffer_event *rbe,
 187                                 void *event)
 188 {
 189         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 190         int str_loc = str_item & 0xffff;
 191         char *addr = (char *)(event + str_loc);
 192
 193         return (u64)(unsigned long)addr;
 194 }
 195
 196 static u64 hist_field_pstring(struct hist_field *hist_field,
 197                               struct tracing_map_elt *elt,
 198                               struct ring_buffer_event *rbe,
 199                               void *event)
 200 {
 201         char **addr = (char **)(event + hist_field->field->offset);
 202
 203         return (u64)(unsigned long)*addr;
 204 }
 205
 206 static u64 hist_field_log2(struct hist_field *hist_field,
 207                            struct tracing_map_elt *elt,
 208                            struct ring_buffer_event *rbe,
 209                            void *event)
 210 {
 211         struct hist_field *operand = hist_field->operands[0];
 212
 213         u64 val = operand->fn(operand, elt, rbe, event);
 214
 215         return (u64) ilog2(roundup_pow_of_two(val));
 216 }
 217
 218 static u64 hist_field_plus(struct hist_field *hist_field,
 219                            struct tracing_map_elt *elt,
 220                            struct ring_buffer_event *rbe,
 221                            void *event)
 222 {
 223         struct hist_field *operand1 = hist_field->operands[0];
 224         struct hist_field *operand2 = hist_field->operands[1];
 225
 226         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 227         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 228
 229         return val1 + val2;
 230 }
 231
 232 static u64 hist_field_minus(struct hist_field *hist_field,
 233                             struct tracing_map_elt *elt,
 234                             struct ring_buffer_event *rbe,
 235                             void *event)
 236 {
 237         struct hist_field *operand1 = hist_field->operands[0];
 238         struct hist_field *operand2 = hist_field->operands[1];
 239
 240         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 241         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 242
 243         return val1 - val2;
 244 }
 245
 246 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 247                                   struct tracing_map_elt *elt,
 248                                   struct ring_buffer_event *rbe,
 249                                   void *event)
 250 {
 251         struct hist_field *operand = hist_field->operands[0];
 252
 253         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 254         u64 val = (u64)-sval;
 255
 256         return val;
 257 }
 258
 259 #define DEFINE_HIST_FIELD_FN(type)                                      \
 260         static u64 hist_field_##type(struct hist_field *hist_field,     \
 261                                      struct tracing_map_elt *elt,       \
 262                                      struct ring_buffer_event *rbe,     \
 263                                      void *event)                       \
 264 {                                                                       \
 265         type *addr = (type *)(event + hist_field->field->offset);       \
 266                                                                         \
 267         return (u64)(unsigned long)*addr;                               \
 268 }
 269
 270 DEFINE_HIST_FIELD_FN(s64);
 271 DEFINE_HIST_FIELD_FN(u64);
 272 DEFINE_HIST_FIELD_FN(s32);
 273 DEFINE_HIST_FIELD_FN(u32);
 274 DEFINE_HIST_FIELD_FN(s16);
 275 DEFINE_HIST_FIELD_FN(u16);
 276 DEFINE_HIST_FIELD_FN(s8);
 277 DEFINE_HIST_FIELD_FN(u8);
 278
 279 #define for_each_hist_field(i, hist_data)       \
 280         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 281
 282 #define for_each_hist_val_field(i, hist_data)   \
 283         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 284
 285 #define for_each_hist_key_field(i, hist_data)   \
 286         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 287
 288 #define HIST_STACKTRACE_DEPTH   16
 289 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 290 #define HIST_STACKTRACE_SKIP    5
 291
 292 #define HITCOUNT_IDX            0
 293 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 294
 295 enum hist_field_flags {
 296         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 297         HIST_FIELD_FL_KEY               = 1 << 1,
 298         HIST_FIELD_FL_STRING            = 1 << 2,
 299         HIST_FIELD_FL_HEX               = 1 << 3,
 300         HIST_FIELD_FL_SYM               = 1 << 4,
 301         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 302         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 303         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 304         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 305         HIST_FIELD_FL_LOG2              = 1 << 9,
 306         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 307         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 308         HIST_FIELD_FL_VAR               = 1 << 12,
 309         HIST_FIELD_FL_EXPR              = 1 << 13,
 310         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 311         HIST_FIELD_FL_CPU               = 1 << 15,
 312         HIST_FIELD_FL_ALIAS             = 1 << 16,
 313 };
 314
 315 struct var_defs {
 316         unsigned int    n_vars;
 317         char            *name[TRACING_MAP_VARS_MAX];
 318         char            *expr[TRACING_MAP_VARS_MAX];
 319 };
 320
 321 struct hist_trigger_attrs {
 322         char            *keys_str;
 323         char            *vals_str;
 324         char            *sort_key_str;
 325         char            *name;
 326         char            *clock;
 327         bool            pause;
 328         bool            cont;
 329         bool            clear;
 330         bool            ts_in_usecs;
 331         unsigned int    map_bits;
 332
 333         char            *assignment_str[TRACING_MAP_VARS_MAX];
 334         unsigned int    n_assignments;
 335
 336         char            *action_str[HIST_ACTIONS_MAX];
 337         unsigned int    n_actions;
 338
 339         struct var_defs var_defs;
 340 };
 341
 342 struct field_var {
 343         struct hist_field       *var;
 344         struct hist_field       *val;
 345 };
 346
 347 struct field_var_hist {
 348         struct hist_trigger_data        *hist_data;
 349         char                            *cmd;
 350 };
 351
 352 struct hist_trigger_data {
 353         struct hist_field               *fields[HIST_FIELDS_MAX];
 354         unsigned int                    n_vals;
 355         unsigned int                    n_keys;
 356         unsigned int                    n_fields;
 357         unsigned int                    n_vars;
 358         unsigned int                    key_size;
 359         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 360         unsigned int                    n_sort_keys;
 361         struct trace_event_file         *event_file;
 362         struct hist_trigger_attrs       *attrs;
 363         struct tracing_map              *map;
 364         bool                            enable_timestamps;
 365         bool                            remove;
 366         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 367         unsigned int                    n_var_refs;
 368
 369         struct action_data              *actions[HIST_ACTIONS_MAX];
 370         unsigned int                    n_actions;
 371
 372         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 373         unsigned int                    n_field_vars;
 374         unsigned int                    n_field_var_str;
 375         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 376         unsigned int                    n_field_var_hists;
 377
 378         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 379         unsigned int                    n_save_vars;
 380         unsigned int                    n_save_var_str;
 381 };
 382
 383 static int create_synth_event(int argc, const char **argv);
 384 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 385 static int synth_event_release(struct dyn_event *ev);
 386 static bool synth_event_is_busy(struct dyn_event *ev);
 387 static bool synth_event_match(const char *system, const char *event,
 388                         int argc, const char **argv, struct dyn_event *ev);
 389
 390 static struct dyn_event_operations synth_event_ops = {
 391         .create = create_synth_event,
 392         .show = synth_event_show,
 393         .is_busy = synth_event_is_busy,
 394         .free = synth_event_release,
 395         .match = synth_event_match,
 396 };
 397
 398 struct synth_field {
 399         char *type;
 400         char *name;
 401         size_t size;
 402         unsigned int offset;
 403         bool is_signed;
 404         bool is_string;
 405 };
 406
 407 struct synth_event {
 408         struct dyn_event                        devent;
 409         int                                     ref;
 410         char                                    *name;
 411         struct synth_field                      **fields;
 412         unsigned int                            n_fields;
 413         unsigned int                            n_u64;
 414         struct trace_event_class                class;
 415         struct trace_event_call                 call;
 416         struct tracepoint                       *tp;
 417         struct module                           *mod;
 418 };
 419
 420 static bool is_synth_event(struct dyn_event *ev)
 421 {
 422         return ev->ops == &synth_event_ops;
 423 }
 424
 425 static struct synth_event *to_synth_event(struct dyn_event *ev)
 426 {
 427         return container_of(ev, struct synth_event, devent);
 428 }
 429
 430 static bool synth_event_is_busy(struct dyn_event *ev)
 431 {
 432         struct synth_event *event = to_synth_event(ev);
 433
 434         return event->ref != 0;
 435 }
 436
 437 static bool synth_event_match(const char *system, const char *event,
 438                         int argc, const char **argv, struct dyn_event *ev)
 439 {
 440         struct synth_event *sev = to_synth_event(ev);
 441
 442         return strcmp(sev->name, event) == 0 &&
 443                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 444 }
 445
 446 struct action_data;
 447
 448 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 449                              struct tracing_map_elt *elt, void *rec,
 450                              struct ring_buffer_event *rbe, void *key,
 451                              struct action_data *data, u64 *var_ref_vals);
 452
 453 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 454
 455 enum handler_id {
 456         HANDLER_ONMATCH = 1,
 457         HANDLER_ONMAX,
 458         HANDLER_ONCHANGE,
 459 };
 460
 461 enum action_id {
 462         ACTION_SAVE = 1,
 463         ACTION_TRACE,
 464         ACTION_SNAPSHOT,
 465 };
 466
 467 struct action_data {
 468         enum handler_id         handler;
 469         enum action_id          action;
 470         char                    *action_name;
 471         action_fn_t             fn;
 472
 473         unsigned int            n_params;
 474         char                    *params[SYNTH_FIELDS_MAX];
 475
 476         /*
 477          * When a histogram trigger is hit, the values of any
 478          * references to variables, including variables being passed
 479          * as parameters to synthetic events, are collected into a
 480          * var_ref_vals array.  This var_ref_idx array is an array of
 481          * indices into the var_ref_vals array, one for each synthetic
 482          * event param, and is passed to the synthetic event
 483          * invocation.
 484          */
 485         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 486         struct synth_event      *synth_event;
 487         bool                    use_trace_keyword;
 488         char                    *synth_event_name;
 489
 490         union {
 491                 struct {
 492                         char                    *event;
 493                         char                    *event_system;
 494                 } match_data;
 495
 496                 struct {
 497                         /*
 498                          * var_str contains the $-unstripped variable
 499                          * name referenced by var_ref, and used when
 500                          * printing the action.  Because var_ref
 501                          * creation is deferred to create_actions(),
 502                          * we need a per-action way to save it until
 503                          * then, thus var_str.
 504                          */
 505                         char                    *var_str;
 506
 507                         /*
 508                          * var_ref refers to the variable being
 509                          * tracked e.g onmax($var).
 510                          */
 511                         struct hist_field       *var_ref;
 512
 513                         /*
 514                          * track_var contains the 'invisible' tracking
 515                          * variable created to keep the current
 516                          * e.g. max value.
 517                          */
 518                         struct hist_field       *track_var;
 519
 520                         check_track_val_fn_t    check_val;
 521                         action_fn_t             save_data;
 522                 } track_data;
 523         };
 524 };
 525
 526 struct track_data {
 527         u64                             track_val;
 528         bool                            updated;
 529
 530         unsigned int                    key_len;
 531         void                            *key;
 532         struct tracing_map_elt          elt;
 533
 534         struct action_data              *action_data;
 535         struct hist_trigger_data        *hist_data;
 536 };
 537
 538 struct hist_elt_data {
 539         char *comm;
 540         u64 *var_ref_vals;
 541         char *field_var_str[SYNTH_FIELDS_MAX];
 542 };
 543
 544 struct snapshot_context {
 545         struct tracing_map_elt  *elt;
 546         void                    *key;
 547 };
 548
 549 static void track_data_free(struct track_data *track_data)
 550 {
 551         struct hist_elt_data *elt_data;
 552
 553         if (!track_data)
 554                 return;
 555
 556         kfree(track_data->key);
 557
 558         elt_data = track_data->elt.private_data;
 559         if (elt_data) {
 560                 kfree(elt_data->comm);
 561                 kfree(elt_data);
 562         }
 563
 564         kfree(track_data);
 565 }
 566
 567 static struct track_data *track_data_alloc(unsigned int key_len,
 568                                            struct action_data *action_data,
 569                                            struct hist_trigger_data *hist_data)
 570 {
 571         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 572         struct hist_elt_data *elt_data;
 573
 574         if (!data)
 575                 return ERR_PTR(-ENOMEM);
 576
 577         data->key = kzalloc(key_len, GFP_KERNEL);
 578         if (!data->key) {
 579                 track_data_free(data);
 580                 return ERR_PTR(-ENOMEM);
 581         }
 582
 583         data->key_len = key_len;
 584         data->action_data = action_data;
 585         data->hist_data = hist_data;
 586
 587         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 588         if (!elt_data) {
 589                 track_data_free(data);
 590                 return ERR_PTR(-ENOMEM);
 591         }
 592         data->elt.private_data = elt_data;
 593
 594         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 595         if (!elt_data->comm) {
 596                 track_data_free(data);
 597                 return ERR_PTR(-ENOMEM);
 598         }
 599
 600         return data;
 601 }
 602
 603 static char last_cmd[MAX_FILTER_STR_VAL];
 604 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 605
 606 static int errpos(char *str)
 607 {
 608         return err_pos(last_cmd, str);
 609 }
 610
 611 static void last_cmd_set(struct trace_event_file *file, char *str)
 612 {
 613         const char *system = NULL, *name = NULL;
 614         struct trace_event_call *call;
 615
 616         if (!str)
 617                 return;
 618
 619         strcpy(last_cmd, "hist:");
 620         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 621
 622         if (file) {
 623                 call = file->event_call;
 624
 625                 system = call->class->system;
 626                 if (system) {
 627                         name = trace_event_name(call);
 628                         if (!name)
 629                                 system = NULL;
 630                 }
 631         }
 632
 633         if (system)
 634                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 635 }
 636
 637 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 638 {
 639         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 640                         err_type, err_pos);
 641 }
 642
 643 static void hist_err_clear(void)
 644 {
 645         last_cmd[0] = '\0';
 646         last_cmd_loc[0] = '\0';
 647 }
 648
 649 struct synth_trace_event {
 650         struct trace_entry      ent;
 651         u64                     fields[];
 652 };
 653
 654 static int synth_event_define_fields(struct trace_event_call *call)
 655 {
 656         struct synth_trace_event trace;
 657         int offset = offsetof(typeof(trace), fields);
 658         struct synth_event *event = call->data;
 659         unsigned int i, size, n_u64;
 660         char *name, *type;
 661         bool is_signed;
 662         int ret = 0;
 663
 664         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 665                 size = event->fields[i]->size;
 666                 is_signed = event->fields[i]->is_signed;
 667                 type = event->fields[i]->type;
 668                 name = event->fields[i]->name;
 669                 ret = trace_define_field(call, type, name, offset, size,
 670                                          is_signed, FILTER_OTHER);
 671                 if (ret)
 672                         break;
 673
 674                 event->fields[i]->offset = n_u64;
 675
 676                 if (event->fields[i]->is_string) {
 677                         offset += STR_VAR_LEN_MAX;
 678                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 679                 } else {
 680                         offset += sizeof(u64);
 681                         n_u64++;
 682                 }
 683         }
 684
 685         event->n_u64 = n_u64;
 686
 687         return ret;
 688 }
 689
 690 static bool synth_field_signed(char *type)
 691 {
 692         if (str_has_prefix(type, "u"))
 693                 return false;
 694         if (strcmp(type, "gfp_t") == 0)
 695                 return false;
 696
 697         return true;
 698 }
 699
 700 static int synth_field_is_string(char *type)
 701 {
 702         if (strstr(type, "char[") != NULL)
 703                 return true;
 704
 705         return false;
 706 }
 707
 708 static int synth_field_string_size(char *type)
 709 {
 710         char buf[4], *end, *start;
 711         unsigned int len;
 712         int size, err;
 713
 714         start = strstr(type, "char[");
 715         if (start == NULL)
 716                 return -EINVAL;
 717         start += sizeof("char[") - 1;
 718
 719         end = strchr(type, ']');
 720         if (!end || end < start)
 721                 return -EINVAL;
 722
 723         len = end - start;
 724         if (len > 3)
 725                 return -EINVAL;
 726
 727         strncpy(buf, start, len);
 728         buf[len] = '\0';
 729
 730         err = kstrtouint(buf, 0, &size);
 731         if (err)
 732                 return err;
 733
 734         if (size > STR_VAR_LEN_MAX)
 735                 return -EINVAL;
 736
 737         return size;
 738 }
 739
 740 static int synth_field_size(char *type)
 741 {
 742         int size = 0;
 743
 744         if (strcmp(type, "s64") == 0)
 745                 size = sizeof(s64);
 746         else if (strcmp(type, "u64") == 0)
 747                 size = sizeof(u64);
 748         else if (strcmp(type, "s32") == 0)
 749                 size = sizeof(s32);
 750         else if (strcmp(type, "u32") == 0)
 751                 size = sizeof(u32);
 752         else if (strcmp(type, "s16") == 0)
 753                 size = sizeof(s16);
 754         else if (strcmp(type, "u16") == 0)
 755                 size = sizeof(u16);
 756         else if (strcmp(type, "s8") == 0)
 757                 size = sizeof(s8);
 758         else if (strcmp(type, "u8") == 0)
 759                 size = sizeof(u8);
 760         else if (strcmp(type, "char") == 0)
 761                 size = sizeof(char);
 762         else if (strcmp(type, "unsigned char") == 0)
 763                 size = sizeof(unsigned char);
 764         else if (strcmp(type, "int") == 0)
 765                 size = sizeof(int);
 766         else if (strcmp(type, "unsigned int") == 0)
 767                 size = sizeof(unsigned int);
 768         else if (strcmp(type, "long") == 0)
 769                 size = sizeof(long);
 770         else if (strcmp(type, "unsigned long") == 0)
 771                 size = sizeof(unsigned long);
 772         else if (strcmp(type, "pid_t") == 0)
 773                 size = sizeof(pid_t);
 774         else if (strcmp(type, "gfp_t") == 0)
 775                 size = sizeof(gfp_t);
 776         else if (synth_field_is_string(type))
 777                 size = synth_field_string_size(type);
 778
 779         return size;
 780 }
 781
 782 static const char *synth_field_fmt(char *type)
 783 {
 784         const char *fmt = "%llu";
 785
 786         if (strcmp(type, "s64") == 0)
 787                 fmt = "%lld";
 788         else if (strcmp(type, "u64") == 0)
 789                 fmt = "%llu";
 790         else if (strcmp(type, "s32") == 0)
 791                 fmt = "%d";
 792         else if (strcmp(type, "u32") == 0)
 793                 fmt = "%u";
 794         else if (strcmp(type, "s16") == 0)
 795                 fmt = "%d";
 796         else if (strcmp(type, "u16") == 0)
 797                 fmt = "%u";
 798         else if (strcmp(type, "s8") == 0)
 799                 fmt = "%d";
 800         else if (strcmp(type, "u8") == 0)
 801                 fmt = "%u";
 802         else if (strcmp(type, "char") == 0)
 803                 fmt = "%d";
 804         else if (strcmp(type, "unsigned char") == 0)
 805                 fmt = "%u";
 806         else if (strcmp(type, "int") == 0)
 807                 fmt = "%d";
 808         else if (strcmp(type, "unsigned int") == 0)
 809                 fmt = "%u";
 810         else if (strcmp(type, "long") == 0)
 811                 fmt = "%ld";
 812         else if (strcmp(type, "unsigned long") == 0)
 813                 fmt = "%lu";
 814         else if (strcmp(type, "pid_t") == 0)
 815                 fmt = "%d";
 816         else if (strcmp(type, "gfp_t") == 0)
 817                 fmt = "%x";
 818         else if (synth_field_is_string(type))
 819                 fmt = "%s";
 820
 821         return fmt;
 822 }
 823
 824 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 825                                            int flags,
 826                                            struct trace_event *event)
 827 {
 828         struct trace_array *tr = iter->tr;
 829         struct trace_seq *s = &iter->seq;
 830         struct synth_trace_event *entry;
 831         struct synth_event *se;
 832         unsigned int i, n_u64;
 833         char print_fmt[32];
 834         const char *fmt;
 835
 836         entry = (struct synth_trace_event *)iter->ent;
 837         se = container_of(event, struct synth_event, call.event);
 838
 839         trace_seq_printf(s, "%s: ", se->name);
 840
 841         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 842                 if (trace_seq_has_overflowed(s))
 843                         goto end;
 844
 845                 fmt = synth_field_fmt(se->fields[i]->type);
 846
 847                 /* parameter types */
 848                 if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
 849                         trace_seq_printf(s, "%s ", fmt);
 850
 851                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 852
 853                 /* parameter values */
 854                 if (se->fields[i]->is_string) {
 855                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 856                                          (char *)&entry->fields[n_u64],
 857                                          i == se->n_fields - 1 ? "" : " ");
 858                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 859                 } else {
 860                         struct trace_print_flags __flags[] = {
 861                             __def_gfpflag_names, {-1, NULL} };
 862
 863                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 864                                          entry->fields[n_u64],
 865                                          i == se->n_fields - 1 ? "" : " ");
 866
 867                         if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 868                                 trace_seq_puts(s, " (");
 869                                 trace_print_flags_seq(s, "|",
 870                                                       entry->fields[n_u64],
 871                                                       __flags);
 872                                 trace_seq_putc(s, ')');
 873                         }
 874                         n_u64++;
 875                 }
 876         }
 877 end:
 878         trace_seq_putc(s, '\n');
 879
 880         return trace_handle_return(s);
 881 }
 882
 883 static struct trace_event_functions synth_event_funcs = {
 884         .trace          = print_synth_event
 885 };
 886
 887 static notrace void trace_event_raw_event_synth(void *__data,
 888                                                 u64 *var_ref_vals,
 889                                                 unsigned int *var_ref_idx)
 890 {
 891         struct trace_event_file *trace_file = __data;
 892         struct synth_trace_event *entry;
 893         struct trace_event_buffer fbuffer;
 894         struct trace_buffer *buffer;
 895         struct synth_event *event;
 896         unsigned int i, n_u64, val_idx;
 897         int fields_size = 0;
 898
 899         event = trace_file->event_call->data;
 900
 901         if (trace_trigger_soft_disabled(trace_file))
 902                 return;
 903
 904         fields_size = event->n_u64 * sizeof(u64);
 905
 906         /*
 907          * Avoid ring buffer recursion detection, as this event
 908          * is being performed within another event.
 909          */
 910         buffer = trace_file->tr->array_buffer.buffer;
 911         ring_buffer_nest_start(buffer);
 912
 913         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 914                                            sizeof(*entry) + fields_size);
 915         if (!entry)
 916                 goto out;
 917
 918         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 919                 val_idx = var_ref_idx[i];
 920                 if (event->fields[i]->is_string) {
 921                         char *str_val = (char *)(long)var_ref_vals[val_idx];
 922                         char *str_field = (char *)&entry->fields[n_u64];
 923
 924                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 925                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 926                 } else {
 927                         struct synth_field *field = event->fields[i];
 928                         u64 val = var_ref_vals[val_idx];
 929
 930                         switch (field->size) {
 931                         case 1:
 932                                 *(u8 *)&entry->fields[n_u64] = (u8)val;
 933                                 break;
 934
 935                         case 2:
 936                                 *(u16 *)&entry->fields[n_u64] = (u16)val;
 937                                 break;
 938
 939                         case 4:
 940                                 *(u32 *)&entry->fields[n_u64] = (u32)val;
 941                                 break;
 942
 943                         default:
 944                                 entry->fields[n_u64] = val;
 945                                 break;
 946                         }
 947                         n_u64++;
 948                 }
 949         }
 950
 951         trace_event_buffer_commit(&fbuffer);
 952 out:
 953         ring_buffer_nest_end(buffer);
 954 }
 955
 956 static void free_synth_event_print_fmt(struct trace_event_call *call)
 957 {
 958         if (call) {
 959                 kfree(call->print_fmt);
 960                 call->print_fmt = NULL;
 961         }
 962 }
 963
 964 static int __set_synth_event_print_fmt(struct synth_event *event,
 965                                        char *buf, int len)
 966 {
 967         const char *fmt;
 968         int pos = 0;
 969         int i;
 970
 971         /* When len=0, we just calculate the needed length */
 972 #define LEN_OR_ZERO (len ? len - pos : 0)
 973
 974         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 975         for (i = 0; i < event->n_fields; i++) {
 976                 fmt = synth_field_fmt(event->fields[i]->type);
 977                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 978                                 event->fields[i]->name, fmt,
 979                                 i == event->n_fields - 1 ? "" : ", ");
 980         }
 981         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 982
 983         for (i = 0; i < event->n_fields; i++) {
 984                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 985                                 ", REC->%s", event->fields[i]->name);
 986         }
 987
 988 #undef LEN_OR_ZERO
 989
 990         /* return the length of print_fmt */
 991         return pos;
 992 }
 993
 994 static int set_synth_event_print_fmt(struct trace_event_call *call)
 995 {
 996         struct synth_event *event = call->data;
 997         char *print_fmt;
 998         int len;
 999
1000         /* First: called with 0 length to calculate the needed length */
1001         len = __set_synth_event_print_fmt(event, NULL, 0);
1002
1003         print_fmt = kmalloc(len + 1, GFP_KERNEL);
1004         if (!print_fmt)
1005                 return -ENOMEM;
1006
1007         /* Second: actually write the @print_fmt */
1008         __set_synth_event_print_fmt(event, print_fmt, len + 1);
1009         call->print_fmt = print_fmt;
1010
1011         return 0;
1012 }
1013
1014 static void free_synth_field(struct synth_field *field)
1015 {
1016         kfree(field->type);
1017         kfree(field->name);
1018         kfree(field);
1019 }
1020
1021 static struct synth_field *parse_synth_field(int argc, const char **argv,
1022                                              int *consumed)
1023 {
1024         struct synth_field *field;
1025         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
1026         int len, ret = 0;
1027
1028         if (field_type[0] == ';')
1029                 field_type++;
1030
1031         if (!strcmp(field_type, "unsigned")) {
1032                 if (argc < 3)
1033                         return ERR_PTR(-EINVAL);
1034                 prefix = "unsigned ";
1035                 field_type = argv[1];
1036                 field_name = argv[2];
1037                 *consumed = 3;
1038         } else {
1039                 field_name = argv[1];
1040                 *consumed = 2;
1041         }
1042
1043         field = kzalloc(sizeof(*field), GFP_KERNEL);
1044         if (!field)
1045                 return ERR_PTR(-ENOMEM);
1046
1047         len = strlen(field_name);
1048         array = strchr(field_name, '[');
1049         if (array)
1050                 len -= strlen(array);
1051         else if (field_name[len - 1] == ';')
1052                 len--;
1053
1054         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1055         if (!field->name) {
1056                 ret = -ENOMEM;
1057                 goto free;
1058         }
1059
1060         if (field_type[0] == ';')
1061                 field_type++;
1062         len = strlen(field_type) + 1;
1063         if (array)
1064                 len += strlen(array);
1065         if (prefix)
1066                 len += strlen(prefix);
1067
1068         field->type = kzalloc(len, GFP_KERNEL);
1069         if (!field->type) {
1070                 ret = -ENOMEM;
1071                 goto free;
1072         }
1073         if (prefix)
1074                 strcat(field->type, prefix);
1075         strcat(field->type, field_type);
1076         if (array) {
1077                 strcat(field->type, array);
1078                 if (field->type[len - 1] == ';')
1079                         field->type[len - 1] = '\0';
1080         }
1081
1082         field->size = synth_field_size(field->type);
1083         if (!field->size) {
1084                 ret = -EINVAL;
1085                 goto free;
1086         }
1087
1088         if (synth_field_is_string(field->type))
1089                 field->is_string = true;
1090
1091         field->is_signed = synth_field_signed(field->type);
1092
1093  out:
1094         return field;
1095  free:
1096         free_synth_field(field);
1097         field = ERR_PTR(ret);
1098         goto out;
1099 }
1100
1101 static void free_synth_tracepoint(struct tracepoint *tp)
1102 {
1103         if (!tp)
1104                 return;
1105
1106         kfree(tp->name);
1107         kfree(tp);
1108 }
1109
1110 static struct tracepoint *alloc_synth_tracepoint(char *name)
1111 {
1112         struct tracepoint *tp;
1113
1114         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1115         if (!tp)
1116                 return ERR_PTR(-ENOMEM);
1117
1118         tp->name = kstrdup(name, GFP_KERNEL);
1119         if (!tp->name) {
1120                 kfree(tp);
1121                 return ERR_PTR(-ENOMEM);
1122         }
1123
1124         return tp;
1125 }
1126
1127 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1128                                     unsigned int *var_ref_idx);
1129
1130 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1131                                unsigned int *var_ref_idx)
1132 {
1133         struct tracepoint *tp = event->tp;
1134
1135         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1136                 struct tracepoint_func *probe_func_ptr;
1137                 synth_probe_func_t probe_func;
1138                 void *__data;
1139
1140                 if (!(cpu_online(raw_smp_processor_id())))
1141                         return;
1142
1143                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1144                 if (probe_func_ptr) {
1145                         do {
1146                                 probe_func = probe_func_ptr->func;
1147                                 __data = probe_func_ptr->data;
1148                                 probe_func(__data, var_ref_vals, var_ref_idx);
1149                         } while ((++probe_func_ptr)->func);
1150                 }
1151         }
1152 }
1153
1154 static struct synth_event *find_synth_event(const char *name)
1155 {
1156         struct dyn_event *pos;
1157         struct synth_event *event;
1158
1159         for_each_dyn_event(pos) {
1160                 if (!is_synth_event(pos))
1161                         continue;
1162                 event = to_synth_event(pos);
1163                 if (strcmp(event->name, name) == 0)
1164                         return event;
1165         }
1166
1167         return NULL;
1168 }
1169
1170 static struct trace_event_fields synth_event_fields_array[] = {
1171         { .type = TRACE_FUNCTION_TYPE,
1172           .define_fields = synth_event_define_fields },
1173         {}
1174 };
1175
1176 static int register_synth_event(struct synth_event *event)
1177 {
1178         struct trace_event_call *call = &event->call;
1179         int ret = 0;
1180
1181         event->call.class = &event->class;
1182         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1183         if (!event->class.system) {
1184                 ret = -ENOMEM;
1185                 goto out;
1186         }
1187
1188         event->tp = alloc_synth_tracepoint(event->name);
1189         if (IS_ERR(event->tp)) {
1190                 ret = PTR_ERR(event->tp);
1191                 event->tp = NULL;
1192                 goto out;
1193         }
1194
1195         INIT_LIST_HEAD(&call->class->fields);
1196         call->event.funcs = &synth_event_funcs;
1197         call->class->fields_array = synth_event_fields_array;
1198
1199         ret = register_trace_event(&call->event);
1200         if (!ret) {
1201                 ret = -ENODEV;
1202                 goto out;
1203         }
1204         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1205         call->class->reg = trace_event_reg;
1206         call->class->probe = trace_event_raw_event_synth;
1207         call->data = event;
1208         call->tp = event->tp;
1209
1210         ret = trace_add_event_call(call);
1211         if (ret) {
1212                 pr_warn("Failed to register synthetic event: %s\n",
1213                         trace_event_name(call));
1214                 goto err;
1215         }
1216
1217         ret = set_synth_event_print_fmt(call);
1218         if (ret < 0) {
1219                 trace_remove_event_call(call);
1220                 goto err;
1221         }
1222  out:
1223         return ret;
1224  err:
1225         unregister_trace_event(&call->event);
1226         goto out;
1227 }
1228
1229 static int unregister_synth_event(struct synth_event *event)
1230 {
1231         struct trace_event_call *call = &event->call;
1232         int ret;
1233
1234         ret = trace_remove_event_call(call);
1235
1236         return ret;
1237 }
1238
1239 static void free_synth_event(struct synth_event *event)
1240 {
1241         unsigned int i;
1242
1243         if (!event)
1244                 return;
1245
1246         for (i = 0; i < event->n_fields; i++)
1247                 free_synth_field(event->fields[i]);
1248
1249         kfree(event->fields);
1250         kfree(event->name);
1251         kfree(event->class.system);
1252         free_synth_tracepoint(event->tp);
1253         free_synth_event_print_fmt(&event->call);
1254         kfree(event);
1255 }
1256
1257 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1258                                              struct synth_field **fields)
1259 {
1260         struct synth_event *event;
1261         unsigned int i;
1262
1263         event = kzalloc(sizeof(*event), GFP_KERNEL);
1264         if (!event) {
1265                 event = ERR_PTR(-ENOMEM);
1266                 goto out;
1267         }
1268
1269         event->name = kstrdup(name, GFP_KERNEL);
1270         if (!event->name) {
1271                 kfree(event);
1272                 event = ERR_PTR(-ENOMEM);
1273                 goto out;
1274         }
1275
1276         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1277         if (!event->fields) {
1278                 free_synth_event(event);
1279                 event = ERR_PTR(-ENOMEM);
1280                 goto out;
1281         }
1282
1283         dyn_event_init(&event->devent, &synth_event_ops);
1284
1285         for (i = 0; i < n_fields; i++)
1286                 event->fields[i] = fields[i];
1287
1288         event->n_fields = n_fields;
1289  out:
1290         return event;
1291 }
1292
1293 static void action_trace(struct hist_trigger_data *hist_data,
1294                          struct tracing_map_elt *elt, void *rec,
1295                          struct ring_buffer_event *rbe, void *key,
1296                          struct action_data *data, u64 *var_ref_vals)
1297 {
1298         struct synth_event *event = data->synth_event;
1299
1300         trace_synth(event, var_ref_vals, data->var_ref_idx);
1301 }
1302
1303 struct hist_var_data {
1304         struct list_head list;
1305         struct hist_trigger_data *hist_data;
1306 };
1307
1308 static int synth_event_check_arg_fn(void *data)
1309 {
1310         struct dynevent_arg_pair *arg_pair = data;
1311         int size;
1312
1313         size = synth_field_size((char *)arg_pair->lhs);
1314
1315         return size ? 0 : -EINVAL;
1316 }
1317
1318 /**
1319  * synth_event_add_field - Add a new field to a synthetic event cmd
1320  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1321  * @type: The type of the new field to add
1322  * @name: The name of the new field to add
1323  *
1324  * Add a new field to a synthetic event cmd object.  Field ordering is in
1325  * the same order the fields are added.
1326  *
1327  * See synth_field_size() for available types. If field_name contains
1328  * [n] the field is considered to be an array.
1329  *
1330  * Return: 0 if successful, error otherwise.
1331  */
1332 int synth_event_add_field(struct dynevent_cmd *cmd, const char *type,
1333                           const char *name)
1334 {
1335         struct dynevent_arg_pair arg_pair;
1336         int ret;
1337
1338         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1339                 return -EINVAL;
1340
1341         if (!type || !name)
1342                 return -EINVAL;
1343
1344         dynevent_arg_pair_init(&arg_pair, 0, ';');
1345
1346         arg_pair.lhs = type;
1347         arg_pair.rhs = name;
1348
1349         ret = dynevent_arg_pair_add(cmd, &arg_pair, synth_event_check_arg_fn);
1350         if (ret)
1351                 return ret;
1352
1353         if (++cmd->n_fields > SYNTH_FIELDS_MAX)
1354                 ret = -EINVAL;
1355
1356         return ret;
1357 }
1358 EXPORT_SYMBOL_GPL(synth_event_add_field);
1359
1360 /**
1361  * synth_event_add_field_str - Add a new field to a synthetic event cmd
1362  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1363  * @type_name: The type and name of the new field to add, as a single string
1364  *
1365  * Add a new field to a synthetic event cmd object, as a single
1366  * string.  The @type_name string is expected to be of the form 'type
1367  * name', which will be appended by ';'.  No sanity checking is done -
1368  * what's passed in is assumed to already be well-formed.  Field
1369  * ordering is in the same order the fields are added.
1370  *
1371  * See synth_field_size() for available types. If field_name contains
1372  * [n] the field is considered to be an array.
1373  *
1374  * Return: 0 if successful, error otherwise.
1375  */
1376 int synth_event_add_field_str(struct dynevent_cmd *cmd, const char *type_name)
1377 {
1378         struct dynevent_arg arg;
1379         int ret;
1380
1381         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1382                 return -EINVAL;
1383
1384         if (!type_name)
1385                 return -EINVAL;
1386
1387         dynevent_arg_init(&arg, ';');
1388
1389         arg.str = type_name;
1390
1391         ret = dynevent_arg_add(cmd, &arg, NULL);
1392         if (ret)
1393                 return ret;
1394
1395         if (++cmd->n_fields > SYNTH_FIELDS_MAX)
1396                 ret = -EINVAL;
1397
1398         return ret;
1399 }
1400 EXPORT_SYMBOL_GPL(synth_event_add_field_str);
1401
1402 /**
1403  * synth_event_add_fields - Add multiple fields to a synthetic event cmd
1404  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1405  * @fields: An array of type/name field descriptions
1406  * @n_fields: The number of field descriptions contained in the fields array
1407  *
1408  * Add a new set of fields to a synthetic event cmd object.  The event
1409  * fields that will be defined for the event should be passed in as an
1410  * array of struct synth_field_desc, and the number of elements in the
1411  * array passed in as n_fields.  Field ordering will retain the
1412  * ordering given in the fields array.
1413  *
1414  * See synth_field_size() for available types. If field_name contains
1415  * [n] the field is considered to be an array.
1416  *
1417  * Return: 0 if successful, error otherwise.
1418  */
1419 int synth_event_add_fields(struct dynevent_cmd *cmd,
1420                            struct synth_field_desc *fields,
1421                            unsigned int n_fields)
1422 {
1423         unsigned int i;
1424         int ret = 0;
1425
1426         for (i = 0; i < n_fields; i++) {
1427                 if (fields[i].type == NULL || fields[i].name == NULL) {
1428                         ret = -EINVAL;
1429                         break;
1430                 }
1431
1432                 ret = synth_event_add_field(cmd, fields[i].type, fields[i].name);
1433                 if (ret)
1434                         break;
1435         }
1436
1437         return ret;
1438 }
1439 EXPORT_SYMBOL_GPL(synth_event_add_fields);
1440
1441 /**
1442  * __synth_event_gen_cmd_start - Start a synthetic event command from arg list
1443  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1444  * @name: The name of the synthetic event
1445  * @mod: The module creating the event, NULL if not created from a module
1446  * @args: Variable number of arg (pairs), one pair for each field
1447  *
1448  * NOTE: Users normally won't want to call this function directly, but
1449  * rather use the synth_event_gen_cmd_start() wrapper, which
1450  * automatically adds a NULL to the end of the arg list.  If this
1451  * function is used directly, make sure the last arg in the variable
1452  * arg list is NULL.
1453  *
1454  * Generate a synthetic event command to be executed by
1455  * synth_event_gen_cmd_end().  This function can be used to generate
1456  * the complete command or only the first part of it; in the latter
1457  * case, synth_event_add_field(), synth_event_add_field_str(), or
1458  * synth_event_add_fields() can be used to add more fields following
1459  * this.
1460  *
1461  * There should be an even number variable args, each pair consisting
1462  * of a type followed by a field name.
1463  *
1464  * See synth_field_size() for available types. If field_name contains
1465  * [n] the field is considered to be an array.
1466  *
1467  * Return: 0 if successful, error otherwise.
1468  */
1469 int __synth_event_gen_cmd_start(struct dynevent_cmd *cmd, const char *name,
1470                                 struct module *mod, ...)
1471 {
1472         struct dynevent_arg arg;
1473         va_list args;
1474         int ret;
1475
1476         cmd->event_name = name;
1477         cmd->private_data = mod;
1478
1479         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1480                 return -EINVAL;
1481
1482         dynevent_arg_init(&arg, 0);
1483         arg.str = name;
1484         ret = dynevent_arg_add(cmd, &arg, NULL);
1485         if (ret)
1486                 return ret;
1487
1488         va_start(args, mod);
1489         for (;;) {
1490                 const char *type, *name;
1491
1492                 type = va_arg(args, const char *);
1493                 if (!type)
1494                         break;
1495                 name = va_arg(args, const char *);
1496                 if (!name)
1497                         break;
1498
1499                 if (++cmd->n_fields > SYNTH_FIELDS_MAX) {
1500                         ret = -EINVAL;
1501                         break;
1502                 }
1503
1504                 ret = synth_event_add_field(cmd, type, name);
1505                 if (ret)
1506                         break;
1507         }
1508         va_end(args);
1509
1510         return ret;
1511 }
1512 EXPORT_SYMBOL_GPL(__synth_event_gen_cmd_start);
1513
1514 /**
1515  * synth_event_gen_cmd_array_start - Start synthetic event command from an array
1516  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1517  * @name: The name of the synthetic event
1518  * @fields: An array of type/name field descriptions
1519  * @n_fields: The number of field descriptions contained in the fields array
1520  *
1521  * Generate a synthetic event command to be executed by
1522  * synth_event_gen_cmd_end().  This function can be used to generate
1523  * the complete command or only the first part of it; in the latter
1524  * case, synth_event_add_field(), synth_event_add_field_str(), or
1525  * synth_event_add_fields() can be used to add more fields following
1526  * this.
1527  *
1528  * The event fields that will be defined for the event should be
1529  * passed in as an array of struct synth_field_desc, and the number of
1530  * elements in the array passed in as n_fields.  Field ordering will
1531  * retain the ordering given in the fields array.
1532  *
1533  * See synth_field_size() for available types. If field_name contains
1534  * [n] the field is considered to be an array.
1535  *
1536  * Return: 0 if successful, error otherwise.
1537  */
1538 int synth_event_gen_cmd_array_start(struct dynevent_cmd *cmd, const char *name,
1539                                     struct module *mod,
1540                                     struct synth_field_desc *fields,
1541                                     unsigned int n_fields)
1542 {
1543         struct dynevent_arg arg;
1544         unsigned int i;
1545         int ret = 0;
1546
1547         cmd->event_name = name;
1548         cmd->private_data = mod;
1549
1550         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1551                 return -EINVAL;
1552
1553         if (n_fields > SYNTH_FIELDS_MAX)
1554                 return -EINVAL;
1555
1556         dynevent_arg_init(&arg, 0);
1557         arg.str = name;
1558         ret = dynevent_arg_add(cmd, &arg, NULL);
1559         if (ret)
1560                 return ret;
1561
1562         for (i = 0; i < n_fields; i++) {
1563                 if (fields[i].type == NULL || fields[i].name == NULL)
1564                         return -EINVAL;
1565
1566                 ret = synth_event_add_field(cmd, fields[i].type, fields[i].name);
1567                 if (ret)
1568                         break;
1569         }
1570
1571         return ret;
1572 }
1573 EXPORT_SYMBOL_GPL(synth_event_gen_cmd_array_start);
1574
1575 static int __create_synth_event(int argc, const char *name, const char **argv)
1576 {
1577         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1578         struct synth_event *event = NULL;
1579         int i, consumed = 0, n_fields = 0, ret = 0;
1580
1581         /*
1582          * Argument syntax:
1583          *  - Add synthetic event: <event_name> field[;field] ...
1584          *  - Remove synthetic event: !<event_name> field[;field] ...
1585          *      where 'field' = type field_name
1586          */
1587
1588         if (name[0] == '\0' || argc < 1)
1589                 return -EINVAL;
1590
1591         mutex_lock(&event_mutex);
1592
1593         event = find_synth_event(name);
1594         if (event) {
1595                 ret = -EEXIST;
1596                 goto out;
1597         }
1598
1599         for (i = 0; i < argc - 1; i++) {
1600                 if (strcmp(argv[i], ";") == 0)
1601                         continue;
1602                 if (n_fields == SYNTH_FIELDS_MAX) {
1603                         ret = -EINVAL;
1604                         goto err;
1605                 }
1606
1607                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1608                 if (IS_ERR(field)) {
1609                         ret = PTR_ERR(field);
1610                         goto err;
1611                 }
1612                 fields[n_fields++] = field;
1613                 i += consumed - 1;
1614         }
1615
1616         if (i < argc && strcmp(argv[i], ";") != 0) {
1617                 ret = -EINVAL;
1618                 goto err;
1619         }
1620
1621         event = alloc_synth_event(name, n_fields, fields);
1622         if (IS_ERR(event)) {
1623                 ret = PTR_ERR(event);
1624                 event = NULL;
1625                 goto err;
1626         }
1627         ret = register_synth_event(event);
1628         if (!ret)
1629                 dyn_event_add(&event->devent);
1630         else
1631                 free_synth_event(event);
1632  out:
1633         mutex_unlock(&event_mutex);
1634
1635         return ret;
1636  err:
1637         for (i = 0; i < n_fields; i++)
1638                 free_synth_field(fields[i]);
1639
1640         goto out;
1641 }
1642
1643 /**
1644  * synth_event_create - Create a new synthetic event
1645  * @name: The name of the new sythetic event
1646  * @fields: An array of type/name field descriptions
1647  * @n_fields: The number of field descriptions contained in the fields array
1648  * @mod: The module creating the event, NULL if not created from a module
1649  *
1650  * Create a new synthetic event with the given name under the
1651  * trace/events/synthetic/ directory.  The event fields that will be
1652  * defined for the event should be passed in as an array of struct
1653  * synth_field_desc, and the number elements in the array passed in as
1654  * n_fields. Field ordering will retain the ordering given in the
1655  * fields array.
1656  *
1657  * If the new synthetic event is being created from a module, the mod
1658  * param must be non-NULL.  This will ensure that the trace buffer
1659  * won't contain unreadable events.
1660  *
1661  * The new synth event should be deleted using synth_event_delete()
1662  * function.  The new synthetic event can be generated from modules or
1663  * other kernel code using trace_synth_event() and related functions.
1664  *
1665  * Return: 0 if successful, error otherwise.
1666  */
1667 int synth_event_create(const char *name, struct synth_field_desc *fields,
1668                        unsigned int n_fields, struct module *mod)
1669 {
1670         struct dynevent_cmd cmd;
1671         char *buf;
1672         int ret;
1673
1674         buf = kzalloc(MAX_DYNEVENT_CMD_LEN, GFP_KERNEL);
1675         if (!buf)
1676                 return -ENOMEM;
1677
1678         synth_event_cmd_init(&cmd, buf, MAX_DYNEVENT_CMD_LEN);
1679
1680         ret = synth_event_gen_cmd_array_start(&cmd, name, mod,
1681                                               fields, n_fields);
1682         if (ret)
1683                 goto out;
1684
1685         ret = synth_event_gen_cmd_end(&cmd);
1686  out:
1687         kfree(buf);
1688
1689         return ret;
1690 }
1691 EXPORT_SYMBOL_GPL(synth_event_create);
1692
1693 static int destroy_synth_event(struct synth_event *se)
1694 {
1695         int ret;
1696
1697         if (se->ref)
1698                 ret = -EBUSY;
1699         else {
1700                 ret = unregister_synth_event(se);
1701                 if (!ret) {
1702                         dyn_event_remove(&se->devent);
1703                         free_synth_event(se);
1704                 }
1705         }
1706
1707         return ret;
1708 }
1709
1710 /**
1711  * synth_event_delete - Delete a synthetic event
1712  * @event_name: The name of the new sythetic event
1713  *
1714  * Delete a synthetic event that was created with synth_event_create().
1715  *
1716  * Return: 0 if successful, error otherwise.
1717  */
1718 int synth_event_delete(const char *event_name)
1719 {
1720         struct synth_event *se = NULL;
1721         struct module *mod = NULL;
1722         int ret = -ENOENT;
1723
1724         mutex_lock(&event_mutex);
1725         se = find_synth_event(event_name);
1726         if (se) {
1727                 mod = se->mod;
1728                 ret = destroy_synth_event(se);
1729         }
1730         mutex_unlock(&event_mutex);
1731
1732         if (mod) {
1733                 mutex_lock(&trace_types_lock);
1734                 /*
1735                  * It is safest to reset the ring buffer if the module
1736                  * being unloaded registered any events that were
1737                  * used. The only worry is if a new module gets
1738                  * loaded, and takes on the same id as the events of
1739                  * this module. When printing out the buffer, traced
1740                  * events left over from this module may be passed to
1741                  * the new module events and unexpected results may
1742                  * occur.
1743                  */
1744                 tracing_reset_all_online_cpus();
1745                 mutex_unlock(&trace_types_lock);
1746         }
1747
1748         return ret;
1749 }
1750 EXPORT_SYMBOL_GPL(synth_event_delete);
1751
1752 static int create_or_delete_synth_event(int argc, char **argv)
1753 {
1754         const char *name = argv[0];
1755         int ret;
1756
1757         /* trace_run_command() ensures argc != 0 */
1758         if (name[0] == '!') {
1759                 ret = synth_event_delete(name + 1);
1760                 return ret;
1761         }
1762
1763         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1764         return ret == -ECANCELED ? -EINVAL : ret;
1765 }
1766
1767 static int synth_event_run_command(struct dynevent_cmd *cmd)
1768 {
1769         struct synth_event *se;
1770         int ret;
1771
1772         ret = trace_run_command(cmd->seq.buffer, create_or_delete_synth_event);
1773         if (ret)
1774                 return ret;
1775
1776         se = find_synth_event(cmd->event_name);
1777         if (WARN_ON(!se))
1778                 return -ENOENT;
1779
1780         se->mod = cmd->private_data;
1781
1782         return ret;
1783 }
1784
1785 /**
1786  * synth_event_cmd_init - Initialize a synthetic event command object
1787  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1788  * @buf: A pointer to the buffer used to build the command
1789  * @maxlen: The length of the buffer passed in @buf
1790  *
1791  * Initialize a synthetic event command object.  Use this before
1792  * calling any of the other dyenvent_cmd functions.
1793  */
1794 void synth_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen)
1795 {
1796         dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_SYNTH,
1797                           synth_event_run_command);
1798 }
1799 EXPORT_SYMBOL_GPL(synth_event_cmd_init);
1800
1801 /**
1802  * synth_event_trace - Trace a synthetic event
1803  * @file: The trace_event_file representing the synthetic event
1804  * @n_vals: The number of values in vals
1805  * @args: Variable number of args containing the event values
1806  *
1807  * Trace a synthetic event using the values passed in the variable
1808  * argument list.
1809  *
1810  * The argument list should be a list 'n_vals' u64 values.  The number
1811  * of vals must match the number of field in the synthetic event, and
1812  * must be in the same order as the synthetic event fields.
1813  *
1814  * All vals should be cast to u64, and string vals are just pointers
1815  * to strings, cast to u64.  Strings will be copied into space
1816  * reserved in the event for the string, using these pointers.
1817  *
1818  * Return: 0 on success, err otherwise.
1819  */
1820 int synth_event_trace(struct trace_event_file *file, unsigned int n_vals, ...)
1821 {
1822         struct trace_event_buffer fbuffer;
1823         struct synth_trace_event *entry;
1824         struct trace_buffer *buffer;
1825         struct synth_event *event;
1826         unsigned int i, n_u64;
1827         int fields_size = 0;
1828         va_list args;
1829         int ret = 0;
1830
1831         /*
1832          * Normal event generation doesn't get called at all unless
1833          * the ENABLED bit is set (which attaches the probe thus
1834          * allowing this code to be called, etc).  Because this is
1835          * called directly by the user, we don't have that but we
1836          * still need to honor not logging when disabled.
1837          */
1838         if (!(file->flags & EVENT_FILE_FL_ENABLED))
1839                 return 0;
1840
1841         event = file->event_call->data;
1842
1843         if (n_vals != event->n_fields)
1844                 return -EINVAL;
1845
1846         if (trace_trigger_soft_disabled(file))
1847                 return -EINVAL;
1848
1849         fields_size = event->n_u64 * sizeof(u64);
1850
1851         /*
1852          * Avoid ring buffer recursion detection, as this event
1853          * is being performed within another event.
1854          */
1855         buffer = file->tr->array_buffer.buffer;
1856         ring_buffer_nest_start(buffer);
1857
1858         entry = trace_event_buffer_reserve(&fbuffer, file,
1859                                            sizeof(*entry) + fields_size);
1860         if (!entry) {
1861                 ret = -EINVAL;
1862                 goto out;
1863         }
1864
1865         va_start(args, n_vals);
1866         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
1867                 u64 val;
1868
1869                 val = va_arg(args, u64);
1870
1871                 if (event->fields[i]->is_string) {
1872                         char *str_val = (char *)(long)val;
1873                         char *str_field = (char *)&entry->fields[n_u64];
1874
1875                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
1876                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
1877                 } else {
1878                         entry->fields[n_u64] = val;
1879                         n_u64++;
1880                 }
1881         }
1882         va_end(args);
1883
1884         trace_event_buffer_commit(&fbuffer);
1885 out:
1886         ring_buffer_nest_end(buffer);
1887
1888         return ret;
1889 }
1890 EXPORT_SYMBOL_GPL(synth_event_trace);
1891
1892 /**
1893  * synth_event_trace_array - Trace a synthetic event from an array
1894  * @file: The trace_event_file representing the synthetic event
1895  * @vals: Array of values
1896  * @n_vals: The number of values in vals
1897  *
1898  * Trace a synthetic event using the values passed in as 'vals'.
1899  *
1900  * The 'vals' array is just an array of 'n_vals' u64.  The number of
1901  * vals must match the number of field in the synthetic event, and
1902  * must be in the same order as the synthetic event fields.
1903  *
1904  * All vals should be cast to u64, and string vals are just pointers
1905  * to strings, cast to u64.  Strings will be copied into space
1906  * reserved in the event for the string, using these pointers.
1907  *
1908  * Return: 0 on success, err otherwise.
1909  */
1910 int synth_event_trace_array(struct trace_event_file *file, u64 *vals,
1911                             unsigned int n_vals)
1912 {
1913         struct trace_event_buffer fbuffer;
1914         struct synth_trace_event *entry;
1915         struct trace_buffer *buffer;
1916         struct synth_event *event;
1917         unsigned int i, n_u64;
1918         int fields_size = 0;
1919         int ret = 0;
1920
1921         /*
1922          * Normal event generation doesn't get called at all unless
1923          * the ENABLED bit is set (which attaches the probe thus
1924          * allowing this code to be called, etc).  Because this is
1925          * called directly by the user, we don't have that but we
1926          * still need to honor not logging when disabled.
1927          */
1928         if (!(file->flags & EVENT_FILE_FL_ENABLED))
1929                 return 0;
1930
1931         event = file->event_call->data;
1932
1933         if (n_vals != event->n_fields)
1934                 return -EINVAL;
1935
1936         if (trace_trigger_soft_disabled(file))
1937                 return -EINVAL;
1938
1939         fields_size = event->n_u64 * sizeof(u64);
1940
1941         /*
1942          * Avoid ring buffer recursion detection, as this event
1943          * is being performed within another event.
1944          */
1945         buffer = file->tr->array_buffer.buffer;
1946         ring_buffer_nest_start(buffer);
1947
1948         entry = trace_event_buffer_reserve(&fbuffer, file,
1949                                            sizeof(*entry) + fields_size);
1950         if (!entry) {
1951                 ret = -EINVAL;
1952                 goto out;
1953         }
1954
1955         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
1956                 if (event->fields[i]->is_string) {
1957                         char *str_val = (char *)(long)vals[i];
1958                         char *str_field = (char *)&entry->fields[n_u64];
1959
1960                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
1961                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
1962                 } else {
1963                         entry->fields[n_u64] = vals[i];
1964                         n_u64++;
1965                 }
1966         }
1967
1968         trace_event_buffer_commit(&fbuffer);
1969 out:
1970         ring_buffer_nest_end(buffer);
1971
1972         return ret;
1973 }
1974 EXPORT_SYMBOL_GPL(synth_event_trace_array);
1975
1976 /**
1977  * synth_event_trace_start - Start piecewise synthetic event trace
1978  * @file: The trace_event_file representing the synthetic event
1979  * @trace_state: A pointer to object tracking the piecewise trace state
1980  *
1981  * Start the trace of a synthetic event field-by-field rather than all
1982  * at once.
1983  *
1984  * This function 'opens' an event trace, which means space is reserved
1985  * for the event in the trace buffer, after which the event's
1986  * individual field values can be set through either
1987  * synth_event_add_next_val() or synth_event_add_val().
1988  *
1989  * A pointer to a trace_state object is passed in, which will keep
1990  * track of the current event trace state until the event trace is
1991  * closed (and the event finally traced) using
1992  * synth_event_trace_end().
1993  *
1994  * Note that synth_event_trace_end() must be called after all values
1995  * have been added for each event trace, regardless of whether adding
1996  * all field values succeeded or not.
1997  *
1998  * Note also that for a given event trace, all fields must be added
1999  * using either synth_event_add_next_val() or synth_event_add_val()
2000  * but not both together or interleaved.
2001  *
2002  * Return: 0 on success, err otherwise.
2003  */
2004 int synth_event_trace_start(struct trace_event_file *file,
2005                             struct synth_event_trace_state *trace_state)
2006 {
2007         struct synth_trace_event *entry;
2008         int fields_size = 0;
2009         int ret = 0;
2010
2011         if (!trace_state) {
2012                 ret = -EINVAL;
2013                 goto out;
2014         }
2015
2016         memset(trace_state, '\0', sizeof(*trace_state));
2017
2018         /*
2019          * Normal event tracing doesn't get called at all unless the
2020          * ENABLED bit is set (which attaches the probe thus allowing
2021          * this code to be called, etc).  Because this is called
2022          * directly by the user, we don't have that but we still need
2023          * to honor not logging when disabled.  For the the iterated
2024          * trace case, we save the enabed state upon start and just
2025          * ignore the following data calls.
2026          */
2027         if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
2028                 trace_state->enabled = false;
2029                 goto out;
2030         }
2031
2032         trace_state->enabled = true;
2033
2034         trace_state->event = file->event_call->data;
2035
2036         if (trace_trigger_soft_disabled(file)) {
2037                 ret = -EINVAL;
2038                 goto out;
2039         }
2040
2041         fields_size = trace_state->event->n_u64 * sizeof(u64);
2042
2043         /*
2044          * Avoid ring buffer recursion detection, as this event
2045          * is being performed within another event.
2046          */
2047         trace_state->buffer = file->tr->array_buffer.buffer;
2048         ring_buffer_nest_start(trace_state->buffer);
2049
2050         entry = trace_event_buffer_reserve(&trace_state->fbuffer, file,
2051                                            sizeof(*entry) + fields_size);
2052         if (!entry) {
2053                 ret = -EINVAL;
2054                 goto out;
2055         }
2056
2057         trace_state->entry = entry;
2058 out:
2059         return ret;
2060 }
2061 EXPORT_SYMBOL_GPL(synth_event_trace_start);
2062
2063 static int __synth_event_add_val(const char *field_name, u64 val,
2064                                  struct synth_event_trace_state *trace_state)
2065 {
2066         struct synth_field *field = NULL;
2067         struct synth_trace_event *entry;
2068         struct synth_event *event;
2069         int i, ret = 0;
2070
2071         if (!trace_state) {
2072                 ret = -EINVAL;
2073                 goto out;
2074         }
2075
2076         /* can't mix add_next_synth_val() with add_synth_val() */
2077         if (field_name) {
2078                 if (trace_state->add_next) {
2079                         ret = -EINVAL;
2080                         goto out;
2081                 }
2082                 trace_state->add_name = true;
2083         } else {
2084                 if (trace_state->add_name) {
2085                         ret = -EINVAL;
2086                         goto out;
2087                 }
2088                 trace_state->add_next = true;
2089         }
2090
2091         if (!trace_state->enabled)
2092                 goto out;
2093
2094         event = trace_state->event;
2095         if (trace_state->add_name) {
2096                 for (i = 0; i < event->n_fields; i++) {
2097                         field = event->fields[i];
2098                         if (strcmp(field->name, field_name) == 0)
2099                                 break;
2100                 }
2101                 if (!field) {
2102                         ret = -EINVAL;
2103                         goto out;
2104                 }
2105         } else {
2106                 if (trace_state->cur_field >= event->n_fields) {
2107                         ret = -EINVAL;
2108                         goto out;
2109                 }
2110                 field = event->fields[trace_state->cur_field++];
2111         }
2112
2113         entry = trace_state->entry;
2114         if (field->is_string) {
2115                 char *str_val = (char *)(long)val;
2116                 char *str_field;
2117
2118                 if (!str_val) {
2119                         ret = -EINVAL;
2120                         goto out;
2121                 }
2122
2123                 str_field = (char *)&entry->fields[field->offset];
2124                 strscpy(str_field, str_val, STR_VAR_LEN_MAX);
2125         } else
2126                 entry->fields[field->offset] = val;
2127  out:
2128         return ret;
2129 }
2130
2131 /**
2132  * synth_event_add_next_val - Add the next field's value to an open synth trace
2133  * @val: The value to set the next field to
2134  * @trace_state: A pointer to object tracking the piecewise trace state
2135  *
2136  * Set the value of the next field in an event that's been opened by
2137  * synth_event_trace_start().
2138  *
2139  * The val param should be the value cast to u64.  If the value points
2140  * to a string, the val param should be a char * cast to u64.
2141  *
2142  * This function assumes all the fields in an event are to be set one
2143  * after another - successive calls to this function are made, one for
2144  * each field, in the order of the fields in the event, until all
2145  * fields have been set.  If you'd rather set each field individually
2146  * without regard to ordering, synth_event_add_val() can be used
2147  * instead.
2148  *
2149  * Note however that synth_event_add_next_val() and
2150  * synth_event_add_val() can't be intermixed for a given event trace -
2151  * one or the other but not both can be used at the same time.
2152  *
2153  * Note also that synth_event_trace_end() must be called after all
2154  * values have been added for each event trace, regardless of whether
2155  * adding all field values succeeded or not.
2156  *
2157  * Return: 0 on success, err otherwise.
2158  */
2159 int synth_event_add_next_val(u64 val,
2160                              struct synth_event_trace_state *trace_state)
2161 {
2162         return __synth_event_add_val(NULL, val, trace_state);
2163 }
2164 EXPORT_SYMBOL_GPL(synth_event_add_next_val);
2165
2166 /**
2167  * synth_event_add_val - Add a named field's value to an open synth trace
2168  * @field_name: The name of the synthetic event field value to set
2169  * @val: The value to set the next field to
2170  * @trace_state: A pointer to object tracking the piecewise trace state
2171  *
2172  * Set the value of the named field in an event that's been opened by
2173  * synth_event_trace_start().
2174  *
2175  * The val param should be the value cast to u64.  If the value points
2176  * to a string, the val param should be a char * cast to u64.
2177  *
2178  * This function looks up the field name, and if found, sets the field
2179  * to the specified value.  This lookup makes this function more
2180  * expensive than synth_event_add_next_val(), so use that or the
2181  * none-piecewise synth_event_trace() instead if efficiency is more
2182  * important.
2183  *
2184  * Note however that synth_event_add_next_val() and
2185  * synth_event_add_val() can't be intermixed for a given event trace -
2186  * one or the other but not both can be used at the same time.
2187  *
2188  * Note also that synth_event_trace_end() must be called after all
2189  * values have been added for each event trace, regardless of whether
2190  * adding all field values succeeded or not.
2191  *
2192  * Return: 0 on success, err otherwise.
2193  */
2194 int synth_event_add_val(const char *field_name, u64 val,
2195                         struct synth_event_trace_state *trace_state)
2196 {
2197         return __synth_event_add_val(field_name, val, trace_state);
2198 }
2199 EXPORT_SYMBOL_GPL(synth_event_add_val);
2200
2201 /**
2202  * synth_event_trace_end - End piecewise synthetic event trace
2203  * @trace_state: A pointer to object tracking the piecewise trace state
2204  *
2205  * End the trace of a synthetic event opened by
2206  * synth_event_trace__start().
2207  *
2208  * This function 'closes' an event trace, which basically means that
2209  * it commits the reserved event and cleans up other loose ends.
2210  *
2211  * A pointer to a trace_state object is passed in, which will keep
2212  * track of the current event trace state opened with
2213  * synth_event_trace_start().
2214  *
2215  * Note that this function must be called after all values have been
2216  * added for each event trace, regardless of whether adding all field
2217  * values succeeded or not.
2218  *
2219  * Return: 0 on success, err otherwise.
2220  */
2221 int synth_event_trace_end(struct synth_event_trace_state *trace_state)
2222 {
2223         if (!trace_state)
2224                 return -EINVAL;
2225
2226         trace_event_buffer_commit(&trace_state->fbuffer);
2227
2228         ring_buffer_nest_end(trace_state->buffer);
2229
2230         return 0;
2231 }
2232 EXPORT_SYMBOL_GPL(synth_event_trace_end);
2233
2234 static int create_synth_event(int argc, const char **argv)
2235 {
2236         const char *name = argv[0];
2237         int len;
2238
2239         if (name[0] != 's' || name[1] != ':')
2240                 return -ECANCELED;
2241         name += 2;
2242
2243         /* This interface accepts group name prefix */
2244         if (strchr(name, '/')) {
2245                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
2246                 if (len == 0)
2247                         return -EINVAL;
2248                 name += len;
2249         }
2250         return __create_synth_event(argc - 1, name, argv + 1);
2251 }
2252
2253 static int synth_event_release(struct dyn_event *ev)
2254 {
2255         struct synth_event *event = to_synth_event(ev);
2256         int ret;
2257
2258         if (event->ref)
2259                 return -EBUSY;
2260
2261         ret = unregister_synth_event(event);
2262         if (ret)
2263                 return ret;
2264
2265         dyn_event_remove(ev);
2266         free_synth_event(event);
2267         return 0;
2268 }
2269
2270 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
2271 {
2272         struct synth_field *field;
2273         unsigned int i;
2274
2275         seq_printf(m, "%s\t", event->name);
2276
2277         for (i = 0; i < event->n_fields; i++) {
2278                 field = event->fields[i];
2279
2280                 /* parameter values */
2281                 seq_printf(m, "%s %s%s", field->type, field->name,
2282                            i == event->n_fields - 1 ? "" : "; ");
2283         }
2284
2285         seq_putc(m, '\n');
2286
2287         return 0;
2288 }
2289
2290 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
2291 {
2292         struct synth_event *event = to_synth_event(ev);
2293
2294         seq_printf(m, "s:%s/", event->class.system);
2295
2296         return __synth_event_show(m, event);
2297 }
2298
2299 static int synth_events_seq_show(struct seq_file *m, void *v)
2300 {
2301         struct dyn_event *ev = v;
2302
2303         if (!is_synth_event(ev))
2304                 return 0;
2305
2306         return __synth_event_show(m, to_synth_event(ev));
2307 }
2308
2309 static const struct seq_operations synth_events_seq_op = {
2310         .start  = dyn_event_seq_start,
2311         .next   = dyn_event_seq_next,
2312         .stop   = dyn_event_seq_stop,
2313         .show   = synth_events_seq_show,
2314 };
2315
2316 static int synth_events_open(struct inode *inode, struct file *file)
2317 {
2318         int ret;
2319
2320         ret = security_locked_down(LOCKDOWN_TRACEFS);
2321         if (ret)
2322                 return ret;
2323
2324         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
2325                 ret = dyn_events_release_all(&synth_event_ops);
2326                 if (ret < 0)
2327                         return ret;
2328         }
2329
2330         return seq_open(file, &synth_events_seq_op);
2331 }
2332
2333 static ssize_t synth_events_write(struct file *file,
2334                                   const char __user *buffer,
2335                                   size_t count, loff_t *ppos)
2336 {
2337         return trace_parse_run_command(file, buffer, count, ppos,
2338                                        create_or_delete_synth_event);
2339 }
2340
2341 static const struct file_operations synth_events_fops = {
2342         .open           = synth_events_open,
2343         .write          = synth_events_write,
2344         .read           = seq_read,
2345         .llseek         = seq_lseek,
2346         .release        = seq_release,
2347 };
2348
2349 static u64 hist_field_timestamp(struct hist_field *hist_field,
2350                                 struct tracing_map_elt *elt,
2351                                 struct ring_buffer_event *rbe,
2352                                 void *event)
2353 {
2354         struct hist_trigger_data *hist_data = hist_field->hist_data;
2355         struct trace_array *tr = hist_data->event_file->tr;
2356
2357         u64 ts = ring_buffer_event_time_stamp(rbe);
2358
2359         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
2360                 ts = ns2usecs(ts);
2361
2362         return ts;
2363 }
2364
2365 static u64 hist_field_cpu(struct hist_field *hist_field,
2366                           struct tracing_map_elt *elt,
2367                           struct ring_buffer_event *rbe,
2368                           void *event)
2369 {
2370         int cpu = smp_processor_id();
2371
2372         return cpu;
2373 }
2374
2375 /**
2376  * check_field_for_var_ref - Check if a VAR_REF field references a variable
2377  * @hist_field: The VAR_REF field to check
2378  * @var_data: The hist trigger that owns the variable
2379  * @var_idx: The trigger variable identifier
2380  *
2381  * Check the given VAR_REF field to see whether or not it references
2382  * the given variable associated with the given trigger.
2383  *
2384  * Return: The VAR_REF field if it does reference the variable, NULL if not
2385  */
2386 static struct hist_field *
2387 check_field_for_var_ref(struct hist_field *hist_field,
2388                         struct hist_trigger_data *var_data,
2389                         unsigned int var_idx)
2390 {
2391         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
2392
2393         if (hist_field && hist_field->var.idx == var_idx &&
2394             hist_field->var.hist_data == var_data)
2395                 return hist_field;
2396
2397         return NULL;
2398 }
2399
2400 /**
2401  * find_var_ref - Check if a trigger has a reference to a trigger variable
2402  * @hist_data: The hist trigger that might have a reference to the variable
2403  * @var_data: The hist trigger that owns the variable
2404  * @var_idx: The trigger variable identifier
2405  *
2406  * Check the list of var_refs[] on the first hist trigger to see
2407  * whether any of them are references to the variable on the second
2408  * trigger.
2409  *
2410  * Return: The VAR_REF field referencing the variable if so, NULL if not
2411  */
2412 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
2413                                        struct hist_trigger_data *var_data,
2414                                        unsigned int var_idx)
2415 {
2416         struct hist_field *hist_field;
2417         unsigned int i;
2418
2419         for (i = 0; i < hist_data->n_var_refs; i++) {
2420                 hist_field = hist_data->var_refs[i];
2421                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
2422                         return hist_field;
2423         }
2424
2425         return NULL;
2426 }
2427
2428 /**
2429  * find_any_var_ref - Check if there is a reference to a given trigger variable
2430  * @hist_data: The hist trigger
2431  * @var_idx: The trigger variable identifier
2432  *
2433  * Check to see whether the given variable is currently referenced by
2434  * any other trigger.
2435  *
2436  * The trigger the variable is defined on is explicitly excluded - the
2437  * assumption being that a self-reference doesn't prevent a trigger
2438  * from being removed.
2439  *
2440  * Return: The VAR_REF field referencing the variable if so, NULL if not
2441  */
2442 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
2443                                            unsigned int var_idx)
2444 {
2445         struct trace_array *tr = hist_data->event_file->tr;
2446         struct hist_field *found = NULL;
2447         struct hist_var_data *var_data;
2448
2449         list_for_each_entry(var_data, &tr->hist_vars, list) {
2450                 if (var_data->hist_data == hist_data)
2451                         continue;
2452                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
2453                 if (found)
2454                         break;
2455         }
2456
2457         return found;
2458 }
2459
2460 /**
2461  * check_var_refs - Check if there is a reference to any of trigger's variables
2462  * @hist_data: The hist trigger
2463  *
2464  * A trigger can define one or more variables.  If any one of them is
2465  * currently referenced by any other trigger, this function will
2466  * determine that.
2467
2468  * Typically used to determine whether or not a trigger can be removed
2469  * - if there are any references to a trigger's variables, it cannot.
2470  *
2471  * Return: True if there is a reference to any of trigger's variables
2472  */
2473 static bool check_var_refs(struct hist_trigger_data *hist_data)
2474 {
2475         struct hist_field *field;
2476         bool found = false;
2477         int i;
2478
2479         for_each_hist_field(i, hist_data) {
2480                 field = hist_data->fields[i];
2481                 if (field && field->flags & HIST_FIELD_FL_VAR) {
2482                         if (find_any_var_ref(hist_data, field->var.idx)) {
2483                                 found = true;
2484                                 break;
2485                         }
2486                 }
2487         }
2488
2489         return found;
2490 }
2491
2492 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
2493 {
2494         struct trace_array *tr = hist_data->event_file->tr;
2495         struct hist_var_data *var_data, *found = NULL;
2496
2497         list_for_each_entry(var_data, &tr->hist_vars, list) {
2498                 if (var_data->hist_data == hist_data) {
2499                         found = var_data;
2500                         break;
2501                 }
2502         }
2503
2504         return found;
2505 }
2506
2507 static bool field_has_hist_vars(struct hist_field *hist_field,
2508                                 unsigned int level)
2509 {
2510         int i;
2511
2512         if (level > 3)
2513                 return false;
2514
2515         if (!hist_field)
2516                 return false;
2517
2518         if (hist_field->flags & HIST_FIELD_FL_VAR ||
2519             hist_field->flags & HIST_FIELD_FL_VAR_REF)
2520                 return true;
2521
2522         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
2523                 struct hist_field *operand;
2524
2525                 operand = hist_field->operands[i];
2526                 if (field_has_hist_vars(operand, level + 1))
2527                         return true;
2528         }
2529
2530         return false;
2531 }
2532
2533 static bool has_hist_vars(struct hist_trigger_data *hist_data)
2534 {
2535         struct hist_field *hist_field;
2536         int i;
2537
2538         for_each_hist_field(i, hist_data) {
2539                 hist_field = hist_data->fields[i];
2540                 if (field_has_hist_vars(hist_field, 0))
2541                         return true;
2542         }
2543
2544         return false;
2545 }
2546
2547 static int save_hist_vars(struct hist_trigger_data *hist_data)
2548 {
2549         struct trace_array *tr = hist_data->event_file->tr;
2550         struct hist_var_data *var_data;
2551
2552         var_data = find_hist_vars(hist_data);
2553         if (var_data)
2554                 return 0;
2555
2556         if (tracing_check_open_get_tr(tr))
2557                 return -ENODEV;
2558
2559         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
2560         if (!var_data) {
2561                 trace_array_put(tr);
2562                 return -ENOMEM;
2563         }
2564
2565         var_data->hist_data = hist_data;
2566         list_add(&var_data->list, &tr->hist_vars);
2567
2568         return 0;
2569 }
2570
2571 static void remove_hist_vars(struct hist_trigger_data *hist_data)
2572 {
2573         struct trace_array *tr = hist_data->event_file->tr;
2574         struct hist_var_data *var_data;
2575
2576         var_data = find_hist_vars(hist_data);
2577         if (!var_data)
2578                 return;
2579
2580         if (WARN_ON(check_var_refs(hist_data)))
2581                 return;
2582
2583         list_del(&var_data->list);
2584
2585         kfree(var_data);
2586
2587         trace_array_put(tr);
2588 }
2589
2590 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
2591                                          const char *var_name)
2592 {
2593         struct hist_field *hist_field, *found = NULL;
2594         int i;
2595
2596         for_each_hist_field(i, hist_data) {
2597                 hist_field = hist_data->fields[i];
2598                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
2599                     strcmp(hist_field->var.name, var_name) == 0) {
2600                         found = hist_field;
2601                         break;
2602                 }
2603         }
2604
2605         return found;
2606 }
2607
2608 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
2609                                    struct trace_event_file *file,
2610                                    const char *var_name)
2611 {
2612         struct hist_trigger_data *test_data;
2613         struct event_trigger_data *test;
2614         struct hist_field *hist_field;
2615
2616         lockdep_assert_held(&event_mutex);
2617
2618         hist_field = find_var_field(hist_data, var_name);
2619         if (hist_field)
2620                 return hist_field;
2621
2622         list_for_each_entry(test, &file->triggers, list) {
2623                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2624                         test_data = test->private_data;
2625                         hist_field = find_var_field(test_data, var_name);
2626                         if (hist_field)
2627                                 return hist_field;
2628                 }
2629         }
2630
2631         return NULL;
2632 }
2633
2634 static struct trace_event_file *find_var_file(struct trace_array *tr,
2635                                               char *system,
2636                                               char *event_name,
2637                                               char *var_name)
2638 {
2639         struct hist_trigger_data *var_hist_data;
2640         struct hist_var_data *var_data;
2641         struct trace_event_file *file, *found = NULL;
2642
2643         if (system)
2644                 return find_event_file(tr, system, event_name);
2645
2646         list_for_each_entry(var_data, &tr->hist_vars, list) {
2647                 var_hist_data = var_data->hist_data;
2648                 file = var_hist_data->event_file;
2649                 if (file == found)
2650                         continue;
2651
2652                 if (find_var_field(var_hist_data, var_name)) {
2653                         if (found) {
2654                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
2655                                 return NULL;
2656                         }
2657
2658                         found = file;
2659                 }
2660         }
2661
2662         return found;
2663 }
2664
2665 static struct hist_field *find_file_var(struct trace_event_file *file,
2666                                         const char *var_name)
2667 {
2668         struct hist_trigger_data *test_data;
2669         struct event_trigger_data *test;
2670         struct hist_field *hist_field;
2671
2672         lockdep_assert_held(&event_mutex);
2673
2674         list_for_each_entry(test, &file->triggers, list) {
2675                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2676                         test_data = test->private_data;
2677                         hist_field = find_var_field(test_data, var_name);
2678                         if (hist_field)
2679                                 return hist_field;
2680                 }
2681         }
2682
2683         return NULL;
2684 }
2685
2686 static struct hist_field *
2687 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
2688 {
2689         struct trace_array *tr = hist_data->event_file->tr;
2690         struct hist_field *hist_field, *found = NULL;
2691         struct trace_event_file *file;
2692         unsigned int i;
2693
2694         for (i = 0; i < hist_data->n_actions; i++) {
2695                 struct action_data *data = hist_data->actions[i];
2696
2697                 if (data->handler == HANDLER_ONMATCH) {
2698                         char *system = data->match_data.event_system;
2699                         char *event_name = data->match_data.event;
2700
2701                         file = find_var_file(tr, system, event_name, var_name);
2702                         if (!file)
2703                                 continue;
2704                         hist_field = find_file_var(file, var_name);
2705                         if (hist_field) {
2706                                 if (found) {
2707                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
2708                                                  errpos(var_name));
2709                                         return ERR_PTR(-EINVAL);
2710                                 }
2711
2712                                 found = hist_field;
2713                         }
2714                 }
2715         }
2716         return found;
2717 }
2718
2719 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
2720                                          char *system,
2721                                          char *event_name,
2722                                          char *var_name)
2723 {
2724         struct trace_array *tr = hist_data->event_file->tr;
2725         struct hist_field *hist_field = NULL;
2726         struct trace_event_file *file;
2727
2728         if (!system || !event_name) {
2729                 hist_field = find_match_var(hist_data, var_name);
2730                 if (IS_ERR(hist_field))
2731                         return NULL;
2732                 if (hist_field)
2733                         return hist_field;
2734         }
2735
2736         file = find_var_file(tr, system, event_name, var_name);
2737         if (!file)
2738                 return NULL;
2739
2740         hist_field = find_file_var(file, var_name);
2741
2742         return hist_field;
2743 }
2744
2745 static u64 hist_field_var_ref(struct hist_field *hist_field,
2746                               struct tracing_map_elt *elt,
2747                               struct ring_buffer_event *rbe,
2748                               void *event)
2749 {
2750         struct hist_elt_data *elt_data;
2751         u64 var_val = 0;
2752
2753         if (WARN_ON_ONCE(!elt))
2754                 return var_val;
2755
2756         elt_data = elt->private_data;
2757         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
2758
2759         return var_val;
2760 }
2761
2762 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
2763                              u64 *var_ref_vals, bool self)
2764 {
2765         struct hist_trigger_data *var_data;
2766         struct tracing_map_elt *var_elt;
2767         struct hist_field *hist_field;
2768         unsigned int i, var_idx;
2769         bool resolved = true;
2770         u64 var_val = 0;
2771
2772         for (i = 0; i < hist_data->n_var_refs; i++) {
2773                 hist_field = hist_data->var_refs[i];
2774                 var_idx = hist_field->var.idx;
2775                 var_data = hist_field->var.hist_data;
2776
2777                 if (var_data == NULL) {
2778                         resolved = false;
2779                         break;
2780                 }
2781
2782                 if ((self && var_data != hist_data) ||
2783                     (!self && var_data == hist_data))
2784                         continue;
2785
2786                 var_elt = tracing_map_lookup(var_data->map, key);
2787                 if (!var_elt) {
2788                         resolved = false;
2789                         break;
2790                 }
2791
2792                 if (!tracing_map_var_set(var_elt, var_idx)) {
2793                         resolved = false;
2794                         break;
2795                 }
2796
2797                 if (self || !hist_field->read_once)
2798                         var_val = tracing_map_read_var(var_elt, var_idx);
2799                 else
2800                         var_val = tracing_map_read_var_once(var_elt, var_idx);
2801
2802                 var_ref_vals[i] = var_val;
2803         }
2804
2805         return resolved;
2806 }
2807
2808 static const char *hist_field_name(struct hist_field *field,
2809                                    unsigned int level)
2810 {
2811         const char *field_name = "";
2812
2813         if (level > 1)
2814                 return field_name;
2815
2816         if (field->field)
2817                 field_name = field->field->name;
2818         else if (field->flags & HIST_FIELD_FL_LOG2 ||
2819                  field->flags & HIST_FIELD_FL_ALIAS)
2820                 field_name = hist_field_name(field->operands[0], ++level);
2821         else if (field->flags & HIST_FIELD_FL_CPU)
2822                 field_name = "cpu";
2823         else if (field->flags & HIST_FIELD_FL_EXPR ||
2824                  field->flags & HIST_FIELD_FL_VAR_REF) {
2825                 if (field->system) {
2826                         static char full_name[MAX_FILTER_STR_VAL];
2827
2828                         strcat(full_name, field->system);
2829                         strcat(full_name, ".");
2830                         strcat(full_name, field->event_name);
2831                         strcat(full_name, ".");
2832                         strcat(full_name, field->name);
2833                         field_name = full_name;
2834                 } else
2835                         field_name = field->name;
2836         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
2837                 field_name = "common_timestamp";
2838
2839         if (field_name == NULL)
2840                 field_name = "";
2841
2842         return field_name;
2843 }
2844
2845 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
2846 {
2847         hist_field_fn_t fn = NULL;
2848
2849         switch (field_size) {
2850         case 8:
2851                 if (field_is_signed)
2852                         fn = hist_field_s64;
2853                 else
2854                         fn = hist_field_u64;
2855                 break;
2856         case 4:
2857                 if (field_is_signed)
2858                         fn = hist_field_s32;
2859                 else
2860                         fn = hist_field_u32;
2861                 break;
2862         case 2:
2863                 if (field_is_signed)
2864                         fn = hist_field_s16;
2865                 else
2866                         fn = hist_field_u16;
2867                 break;
2868         case 1:
2869                 if (field_is_signed)
2870                         fn = hist_field_s8;
2871                 else
2872                         fn = hist_field_u8;
2873                 break;
2874         }
2875
2876         return fn;
2877 }
2878
2879 static int parse_map_size(char *str)
2880 {
2881         unsigned long size, map_bits;
2882         int ret;
2883
2884         ret = kstrtoul(str, 0, &size);
2885         if (ret)
2886                 goto out;
2887
2888         map_bits = ilog2(roundup_pow_of_two(size));
2889         if (map_bits < TRACING_MAP_BITS_MIN ||
2890             map_bits > TRACING_MAP_BITS_MAX)
2891                 ret = -EINVAL;
2892         else
2893                 ret = map_bits;
2894  out:
2895         return ret;
2896 }
2897
2898 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2899 {
2900         unsigned int i;
2901
2902         if (!attrs)
2903                 return;
2904
2905         for (i = 0; i < attrs->n_assignments; i++)
2906                 kfree(attrs->assignment_str[i]);
2907
2908         for (i = 0; i < attrs->n_actions; i++)
2909                 kfree(attrs->action_str[i]);
2910
2911         kfree(attrs->name);
2912         kfree(attrs->sort_key_str);
2913         kfree(attrs->keys_str);
2914         kfree(attrs->vals_str);
2915         kfree(attrs->clock);
2916         kfree(attrs);
2917 }
2918
2919 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2920 {
2921         int ret = -EINVAL;
2922
2923         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2924                 return ret;
2925
2926         if ((str_has_prefix(str, "onmatch(")) ||
2927             (str_has_prefix(str, "onmax(")) ||
2928             (str_has_prefix(str, "onchange("))) {
2929                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2930                 if (!attrs->action_str[attrs->n_actions]) {
2931                         ret = -ENOMEM;
2932                         return ret;
2933                 }
2934                 attrs->n_actions++;
2935                 ret = 0;
2936         }
2937         return ret;
2938 }
2939
2940 static int parse_assignment(struct trace_array *tr,
2941                             char *str, struct hist_trigger_attrs *attrs)
2942 {
2943         int len, ret = 0;
2944
2945         if ((len = str_has_prefix(str, "key=")) ||
2946             (len = str_has_prefix(str, "keys="))) {
2947                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
2948                 if (!attrs->keys_str) {
2949                         ret = -ENOMEM;
2950                         goto out;
2951                 }
2952         } else if ((len = str_has_prefix(str, "val=")) ||
2953                    (len = str_has_prefix(str, "vals=")) ||
2954                    (len = str_has_prefix(str, "values="))) {
2955                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
2956                 if (!attrs->vals_str) {
2957                         ret = -ENOMEM;
2958                         goto out;
2959                 }
2960         } else if ((len = str_has_prefix(str, "sort="))) {
2961                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
2962                 if (!attrs->sort_key_str) {
2963                         ret = -ENOMEM;
2964                         goto out;
2965                 }
2966         } else if (str_has_prefix(str, "name=")) {
2967                 attrs->name = kstrdup(str, GFP_KERNEL);
2968                 if (!attrs->name) {
2969                         ret = -ENOMEM;
2970                         goto out;
2971                 }
2972         } else if ((len = str_has_prefix(str, "clock="))) {
2973                 str += len;
2974
2975                 str = strstrip(str);
2976                 attrs->clock = kstrdup(str, GFP_KERNEL);
2977                 if (!attrs->clock) {
2978                         ret = -ENOMEM;
2979                         goto out;
2980                 }
2981         } else if ((len = str_has_prefix(str, "size="))) {
2982                 int map_bits = parse_map_size(str + len);
2983
2984                 if (map_bits < 0) {
2985                         ret = map_bits;
2986                         goto out;
2987                 }
2988                 attrs->map_bits = map_bits;
2989         } else {
2990                 char *assignment;
2991
2992                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2993                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2994                         ret = -EINVAL;
2995                         goto out;
2996                 }
2997
2998                 assignment = kstrdup(str, GFP_KERNEL);
2999                 if (!assignment) {
3000                         ret = -ENOMEM;
3001                         goto out;
3002                 }
3003
3004                 attrs->assignment_str[attrs->n_assignments++] = assignment;
3005         }
3006  out:
3007         return ret;
3008 }
3009
3010 static struct hist_trigger_attrs *
3011 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
3012 {
3013         struct hist_trigger_attrs *attrs;
3014         int ret = 0;
3015
3016         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3017         if (!attrs)
3018                 return ERR_PTR(-ENOMEM);
3019
3020         while (trigger_str) {
3021                 char *str = strsep(&trigger_str, ":");
3022                 char *rhs;
3023
3024                 rhs = strchr(str, '=');
3025                 if (rhs) {
3026                         if (!strlen(++rhs)) {
3027                                 ret = -EINVAL;
3028                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
3029                                 goto free;
3030                         }
3031                         ret = parse_assignment(tr, str, attrs);
3032                         if (ret)
3033                                 goto free;
3034                 } else if (strcmp(str, "pause") == 0)
3035                         attrs->pause = true;
3036                 else if ((strcmp(str, "cont") == 0) ||
3037                          (strcmp(str, "continue") == 0))
3038                         attrs->cont = true;
3039                 else if (strcmp(str, "clear") == 0)
3040                         attrs->clear = true;
3041                 else {
3042                         ret = parse_action(str, attrs);
3043                         if (ret)
3044                                 goto free;
3045                 }
3046         }
3047
3048         if (!attrs->keys_str) {
3049                 ret = -EINVAL;
3050                 goto free;
3051         }
3052
3053         if (!attrs->clock) {
3054                 attrs->clock = kstrdup("global", GFP_KERNEL);
3055                 if (!attrs->clock) {
3056                         ret = -ENOMEM;
3057                         goto free;
3058                 }
3059         }
3060
3061         return attrs;
3062  free:
3063         destroy_hist_trigger_attrs(attrs);
3064
3065         return ERR_PTR(ret);
3066 }
3067
3068 static inline void save_comm(char *comm, struct task_struct *task)
3069 {
3070         if (!task->pid) {
3071                 strcpy(comm, "<idle>");
3072                 return;
3073         }
3074
3075         if (WARN_ON_ONCE(task->pid < 0)) {
3076                 strcpy(comm, "<XXX>");
3077                 return;
3078         }
3079
3080         strncpy(comm, task->comm, TASK_COMM_LEN);
3081 }
3082
3083 static void hist_elt_data_free(struct hist_elt_data *elt_data)
3084 {
3085         unsigned int i;
3086
3087         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
3088                 kfree(elt_data->field_var_str[i]);
3089
3090         kfree(elt_data->comm);
3091         kfree(elt_data);
3092 }
3093
3094 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
3095 {
3096         struct hist_elt_data *elt_data = elt->private_data;
3097
3098         hist_elt_data_free(elt_data);
3099 }
3100
3101 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
3102 {
3103         struct hist_trigger_data *hist_data = elt->map->private_data;
3104         unsigned int size = TASK_COMM_LEN;
3105         struct hist_elt_data *elt_data;
3106         struct hist_field *key_field;
3107         unsigned int i, n_str;
3108
3109         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
3110         if (!elt_data)
3111                 return -ENOMEM;
3112
3113         for_each_hist_key_field(i, hist_data) {
3114                 key_field = hist_data->fields[i];
3115
3116                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
3117                         elt_data->comm = kzalloc(size, GFP_KERNEL);
3118                         if (!elt_data->comm) {
3119                                 kfree(elt_data);
3120                                 return -ENOMEM;
3121                         }
3122                         break;
3123                 }
3124         }
3125
3126         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
3127
3128         size = STR_VAR_LEN_MAX;
3129
3130         for (i = 0; i < n_str; i++) {
3131                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
3132                 if (!elt_data->field_var_str[i]) {
3133                         hist_elt_data_free(elt_data);
3134                         return -ENOMEM;
3135                 }
3136         }
3137
3138         elt->private_data = elt_data;
3139
3140         return 0;
3141 }
3142
3143 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
3144 {
3145         struct hist_elt_data *elt_data = elt->private_data;
3146
3147         if (elt_data->comm)
3148                 save_comm(elt_data->comm, current);
3149 }
3150
3151 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
3152         .elt_alloc      = hist_trigger_elt_data_alloc,
3153         .elt_free       = hist_trigger_elt_data_free,
3154         .elt_init       = hist_trigger_elt_data_init,
3155 };
3156
3157 static const char *get_hist_field_flags(struct hist_field *hist_field)
3158 {
3159         const char *flags_str = NULL;
3160
3161         if (hist_field->flags & HIST_FIELD_FL_HEX)
3162                 flags_str = "hex";
3163         else if (hist_field->flags & HIST_FIELD_FL_SYM)
3164                 flags_str = "sym";
3165         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
3166                 flags_str = "sym-offset";
3167         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
3168                 flags_str = "execname";
3169         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
3170                 flags_str = "syscall";
3171         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
3172                 flags_str = "log2";
3173         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
3174                 flags_str = "usecs";
3175
3176         return flags_str;
3177 }
3178
3179 static void expr_field_str(struct hist_field *field, char *expr)
3180 {
3181         if (field->flags & HIST_FIELD_FL_VAR_REF)
3182                 strcat(expr, "$");
3183
3184         strcat(expr, hist_field_name(field, 0));
3185
3186         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
3187                 const char *flags_str = get_hist_field_flags(field);
3188
3189                 if (flags_str) {
3190                         strcat(expr, ".");
3191                         strcat(expr, flags_str);
3192                 }
3193         }
3194 }
3195
3196 static char *expr_str(struct hist_field *field, unsigned int level)
3197 {
3198         char *expr;
3199
3200         if (level > 1)
3201                 return NULL;
3202
3203         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3204         if (!expr)
3205                 return NULL;
3206
3207         if (!field->operands[0]) {
3208                 expr_field_str(field, expr);
3209                 return expr;
3210         }
3211
3212         if (field->operator == FIELD_OP_UNARY_MINUS) {
3213                 char *subexpr;
3214
3215                 strcat(expr, "-(");
3216                 subexpr = expr_str(field->operands[0], ++level);
3217                 if (!subexpr) {
3218                         kfree(expr);
3219                         return NULL;
3220                 }
3221                 strcat(expr, subexpr);
3222                 strcat(expr, ")");
3223
3224                 kfree(subexpr);
3225
3226                 return expr;
3227         }
3228
3229         expr_field_str(field->operands[0], expr);
3230
3231         switch (field->operator) {
3232         case FIELD_OP_MINUS:
3233                 strcat(expr, "-");
3234                 break;
3235         case FIELD_OP_PLUS:
3236                 strcat(expr, "+");
3237                 break;
3238         default:
3239                 kfree(expr);
3240                 return NULL;
3241         }
3242
3243         expr_field_str(field->operands[1], expr);
3244
3245         return expr;
3246 }
3247
3248 static int contains_operator(char *str)
3249 {
3250         enum field_op_id field_op = FIELD_OP_NONE;
3251         char *op;
3252
3253         op = strpbrk(str, "+-");
3254         if (!op)
3255                 return FIELD_OP_NONE;
3256
3257         switch (*op) {
3258         case '-':
3259                 if (*str == '-')
3260                         field_op = FIELD_OP_UNARY_MINUS;
3261                 else
3262                         field_op = FIELD_OP_MINUS;
3263                 break;
3264         case '+':
3265                 field_op = FIELD_OP_PLUS;
3266                 break;
3267         default:
3268                 break;
3269         }
3270
3271         return field_op;
3272 }
3273
3274 static void get_hist_field(struct hist_field *hist_field)
3275 {
3276         hist_field->ref++;
3277 }
3278
3279 static void __destroy_hist_field(struct hist_field *hist_field)
3280 {
3281         if (--hist_field->ref > 1)
3282                 return;
3283
3284         kfree(hist_field->var.name);
3285         kfree(hist_field->name);
3286         kfree(hist_field->type);
3287
3288         kfree(hist_field);
3289 }
3290
3291 static void destroy_hist_field(struct hist_field *hist_field,
3292                                unsigned int level)
3293 {
3294         unsigned int i;
3295
3296         if (level > 3)
3297                 return;
3298
3299         if (!hist_field)
3300                 return;
3301
3302         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
3303                 return; /* var refs will be destroyed separately */
3304
3305         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
3306                 destroy_hist_field(hist_field->operands[i], level + 1);
3307
3308         __destroy_hist_field(hist_field);
3309 }
3310
3311 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
3312                                             struct ftrace_event_field *field,
3313                                             unsigned long flags,
3314                                             char *var_name)
3315 {
3316         struct hist_field *hist_field;
3317
3318         if (field && is_function_field(field))
3319                 return NULL;
3320
3321         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3322         if (!hist_field)
3323                 return NULL;
3324
3325         hist_field->ref = 1;
3326
3327         hist_field->hist_data = hist_data;
3328
3329         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
3330                 goto out; /* caller will populate */
3331
3332         if (flags & HIST_FIELD_FL_VAR_REF) {
3333                 hist_field->fn = hist_field_var_ref;
3334                 goto out;
3335         }
3336
3337         if (flags & HIST_FIELD_FL_HITCOUNT) {
3338                 hist_field->fn = hist_field_counter;
3339                 hist_field->size = sizeof(u64);
3340                 hist_field->type = kstrdup("u64", GFP_KERNEL);
3341                 if (!hist_field->type)
3342                         goto free;
3343                 goto out;
3344         }
3345
3346         if (flags & HIST_FIELD_FL_STACKTRACE) {
3347                 hist_field->fn = hist_field_none;
3348                 goto out;
3349         }
3350
3351         if (flags & HIST_FIELD_FL_LOG2) {
3352                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
3353                 hist_field->fn = hist_field_log2;
3354                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
3355                 hist_field->size = hist_field->operands[0]->size;
3356                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
3357                 if (!hist_field->type)
3358                         goto free;
3359                 goto out;
3360         }
3361
3362         if (flags & HIST_FIELD_FL_TIMESTAMP) {
3363                 hist_field->fn = hist_field_timestamp;
3364                 hist_field->size = sizeof(u64);
3365                 hist_field->type = kstrdup("u64", GFP_KERNEL);
3366                 if (!hist_field->type)
3367                         goto free;
3368                 goto out;
3369         }
3370
3371         if (flags & HIST_FIELD_FL_CPU) {
3372                 hist_field->fn = hist_field_cpu;
3373                 hist_field->size = sizeof(int);
3374                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
3375                 if (!hist_field->type)
3376                         goto free;
3377                 goto out;
3378         }
3379
3380         if (WARN_ON_ONCE(!field))
3381                 goto out;
3382
3383         if (is_string_field(field)) {
3384                 flags |= HIST_FIELD_FL_STRING;
3385
3386                 hist_field->size = MAX_FILTER_STR_VAL;
3387                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
3388                 if (!hist_field->type)
3389                         goto free;
3390
3391                 if (field->filter_type == FILTER_STATIC_STRING)
3392                         hist_field->fn = hist_field_string;
3393                 else if (field->filter_type == FILTER_DYN_STRING)
3394                         hist_field->fn = hist_field_dynstring;
3395                 else
3396                         hist_field->fn = hist_field_pstring;
3397         } else {
3398                 hist_field->size = field->size;
3399                 hist_field->is_signed = field->is_signed;
3400                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
3401                 if (!hist_field->type)
3402                         goto free;
3403
3404                 hist_field->fn = select_value_fn(field->size,
3405                                                  field->is_signed);
3406                 if (!hist_field->fn) {
3407                         destroy_hist_field(hist_field, 0);
3408                         return NULL;
3409                 }
3410         }
3411  out:
3412         hist_field->field = field;
3413         hist_field->flags = flags;
3414
3415         if (var_name) {
3416                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
3417                 if (!hist_field->var.name)
3418                         goto free;
3419         }
3420
3421         return hist_field;
3422  free:
3423         destroy_hist_field(hist_field, 0);
3424         return NULL;
3425 }
3426
3427 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
3428 {
3429         unsigned int i;
3430
3431         for (i = 0; i < HIST_FIELDS_MAX; i++) {
3432                 if (hist_data->fields[i]) {
3433                         destroy_hist_field(hist_data->fields[i], 0);
3434                         hist_data->fields[i] = NULL;
3435                 }
3436         }
3437
3438         for (i = 0; i < hist_data->n_var_refs; i++) {
3439                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
3440                 __destroy_hist_field(hist_data->var_refs[i]);
3441                 hist_data->var_refs[i] = NULL;
3442         }
3443 }
3444
3445 static int init_var_ref(struct hist_field *ref_field,
3446                         struct hist_field *var_field,
3447                         char *system, char *event_name)
3448 {
3449         int err = 0;
3450
3451         ref_field->var.idx = var_field->var.idx;
3452         ref_field->var.hist_data = var_field->hist_data;
3453         ref_field->size = var_field->size;
3454         ref_field->is_signed = var_field->is_signed;
3455         ref_field->flags |= var_field->flags &
3456                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3457
3458         if (system) {
3459                 ref_field->system = kstrdup(system, GFP_KERNEL);
3460                 if (!ref_field->system)
3461                         return -ENOMEM;
3462         }
3463
3464         if (event_name) {
3465                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
3466                 if (!ref_field->event_name) {
3467                         err = -ENOMEM;
3468                         goto free;
3469                 }
3470         }
3471
3472         if (var_field->var.name) {
3473                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
3474                 if (!ref_field->name) {
3475                         err = -ENOMEM;
3476                         goto free;
3477                 }
3478         } else if (var_field->name) {
3479                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
3480                 if (!ref_field->name) {
3481                         err = -ENOMEM;
3482                         goto free;
3483                 }
3484         }
3485
3486         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
3487         if (!ref_field->type) {
3488                 err = -ENOMEM;
3489                 goto free;
3490         }
3491  out:
3492         return err;
3493  free:
3494         kfree(ref_field->system);
3495         kfree(ref_field->event_name);
3496         kfree(ref_field->name);
3497
3498         goto out;
3499 }
3500
3501 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
3502                             struct hist_field *var_field)
3503 {
3504         struct hist_field *ref_field;
3505         int i;
3506
3507         for (i = 0; i < hist_data->n_var_refs; i++) {
3508                 ref_field = hist_data->var_refs[i];
3509                 if (ref_field->var.idx == var_field->var.idx &&
3510                     ref_field->var.hist_data == var_field->hist_data)
3511                         return i;
3512         }
3513
3514         return -ENOENT;
3515 }
3516
3517 /**
3518  * create_var_ref - Create a variable reference and attach it to trigger
3519  * @hist_data: The trigger that will be referencing the variable
3520  * @var_field: The VAR field to create a reference to
3521  * @system: The optional system string
3522  * @event_name: The optional event_name string
3523  *
3524  * Given a variable hist_field, create a VAR_REF hist_field that
3525  * represents a reference to it.
3526  *
3527  * This function also adds the reference to the trigger that
3528  * now references the variable.
3529  *
3530  * Return: The VAR_REF field if successful, NULL if not
3531  */
3532 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
3533                                          struct hist_field *var_field,
3534                                          char *system, char *event_name)
3535 {
3536         unsigned long flags = HIST_FIELD_FL_VAR_REF;
3537         struct hist_field *ref_field;
3538         int i;
3539
3540         /* Check if the variable already exists */
3541         for (i = 0; i < hist_data->n_var_refs; i++) {
3542                 ref_field = hist_data->var_refs[i];
3543                 if (ref_field->var.idx == var_field->var.idx &&
3544                     ref_field->var.hist_data == var_field->hist_data) {
3545                         get_hist_field(ref_field);
3546                         return ref_field;
3547                 }
3548         }
3549
3550         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
3551         if (ref_field) {
3552                 if (init_var_ref(ref_field, var_field, system, event_name)) {
3553                         destroy_hist_field(ref_field, 0);
3554                         return NULL;
3555                 }
3556
3557                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3558                 ref_field->var_ref_idx = hist_data->n_var_refs++;
3559         }
3560
3561         return ref_field;
3562 }
3563
3564 static bool is_var_ref(char *var_name)
3565 {
3566         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
3567                 return false;
3568
3569         return true;
3570 }
3571
3572 static char *field_name_from_var(struct hist_trigger_data *hist_data,
3573                                  char *var_name)
3574 {
3575         char *name, *field;
3576         unsigned int i;
3577
3578         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3579                 name = hist_data->attrs->var_defs.name[i];
3580
3581                 if (strcmp(var_name, name) == 0) {
3582                         field = hist_data->attrs->var_defs.expr[i];
3583                         if (contains_operator(field) || is_var_ref(field))
3584                                 continue;
3585                         return field;
3586                 }
3587         }
3588
3589         return NULL;
3590 }
3591
3592 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
3593                                  char *system, char *event_name,
3594                                  char *var_name)
3595 {
3596         struct trace_event_call *call;
3597
3598         if (system && event_name) {
3599                 call = hist_data->event_file->event_call;
3600
3601                 if (strcmp(system, call->class->system) != 0)
3602                         return NULL;
3603
3604                 if (strcmp(event_name, trace_event_name(call)) != 0)
3605                         return NULL;
3606         }
3607
3608         if (!!system != !!event_name)
3609                 return NULL;
3610
3611         if (!is_var_ref(var_name))
3612                 return NULL;
3613
3614         var_name++;
3615
3616         return field_name_from_var(hist_data, var_name);
3617 }
3618
3619 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
3620                                         char *system, char *event_name,
3621                                         char *var_name)
3622 {
3623         struct hist_field *var_field = NULL, *ref_field = NULL;
3624         struct trace_array *tr = hist_data->event_file->tr;
3625
3626         if (!is_var_ref(var_name))
3627                 return NULL;
3628
3629         var_name++;
3630
3631         var_field = find_event_var(hist_data, system, event_name, var_name);
3632         if (var_field)
3633                 ref_field = create_var_ref(hist_data, var_field,
3634                                            system, event_name);
3635
3636         if (!ref_field)
3637                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
3638
3639         return ref_field;
3640 }
3641
3642 static struct ftrace_event_field *
3643 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
3644             char *field_str, unsigned long *flags)
3645 {
3646         struct ftrace_event_field *field = NULL;
3647         char *field_name, *modifier, *str;
3648         struct trace_array *tr = file->tr;
3649
3650         modifier = str = kstrdup(field_str, GFP_KERNEL);
3651         if (!modifier)
3652                 return ERR_PTR(-ENOMEM);
3653
3654         field_name = strsep(&modifier, ".");
3655         if (modifier) {
3656                 if (strcmp(modifier, "hex") == 0)
3657                         *flags |= HIST_FIELD_FL_HEX;
3658                 else if (strcmp(modifier, "sym") == 0)
3659                         *flags |= HIST_FIELD_FL_SYM;
3660                 else if (strcmp(modifier, "sym-offset") == 0)
3661                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
3662                 else if ((strcmp(modifier, "execname") == 0) &&
3663                          (strcmp(field_name, "common_pid") == 0))
3664                         *flags |= HIST_FIELD_FL_EXECNAME;
3665                 else if (strcmp(modifier, "syscall") == 0)
3666                         *flags |= HIST_FIELD_FL_SYSCALL;
3667                 else if (strcmp(modifier, "log2") == 0)
3668                         *flags |= HIST_FIELD_FL_LOG2;
3669                 else if (strcmp(modifier, "usecs") == 0)
3670                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
3671                 else {
3672                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
3673                         field = ERR_PTR(-EINVAL);
3674                         goto out;
3675                 }
3676         }
3677
3678         if (strcmp(field_name, "common_timestamp") == 0) {
3679                 *flags |= HIST_FIELD_FL_TIMESTAMP;
3680                 hist_data->enable_timestamps = true;
3681                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
3682                         hist_data->attrs->ts_in_usecs = true;
3683         } else if (strcmp(field_name, "cpu") == 0)
3684                 *flags |= HIST_FIELD_FL_CPU;
3685         else {
3686                 field = trace_find_event_field(file->event_call, field_name);
3687                 if (!field || !field->size) {
3688                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
3689                         field = ERR_PTR(-EINVAL);
3690                         goto out;
3691                 }
3692         }
3693  out:
3694         kfree(str);
3695
3696         return field;
3697 }
3698
3699 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
3700                                        struct hist_field *var_ref,
3701                                        char *var_name)
3702 {
3703         struct hist_field *alias = NULL;
3704         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
3705
3706         alias = create_hist_field(hist_data, NULL, flags, var_name);
3707         if (!alias)
3708                 return NULL;
3709
3710         alias->fn = var_ref->fn;
3711         alias->operands[0] = var_ref;
3712
3713         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
3714                 destroy_hist_field(alias, 0);
3715                 return NULL;
3716         }
3717
3718         alias->var_ref_idx = var_ref->var_ref_idx;
3719
3720         return alias;
3721 }
3722
3723 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
3724                                      struct trace_event_file *file, char *str,
3725                                      unsigned long *flags, char *var_name)
3726 {
3727         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
3728         struct ftrace_event_field *field = NULL;
3729         struct hist_field *hist_field = NULL;
3730         int ret = 0;
3731
3732         s = strchr(str, '.');
3733         if (s) {
3734                 s = strchr(++s, '.');
3735                 if (s) {
3736                         ref_system = strsep(&str, ".");
3737                         if (!str) {
3738                                 ret = -EINVAL;
3739                                 goto out;
3740                         }
3741                         ref_event = strsep(&str, ".");
3742                         if (!str) {
3743                                 ret = -EINVAL;
3744                                 goto out;
3745                         }
3746                         ref_var = str;
3747                 }
3748         }
3749
3750         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
3751         if (!s) {
3752                 hist_field = parse_var_ref(hist_data, ref_system,
3753                                            ref_event, ref_var);
3754                 if (hist_field) {
3755                         if (var_name) {
3756                                 hist_field = create_alias(hist_data, hist_field, var_name);
3757                                 if (!hist_field) {
3758                                         ret = -ENOMEM;
3759                                         goto out;
3760                                 }
3761                         }
3762                         return hist_field;
3763                 }
3764         } else
3765                 str = s;
3766
3767         field = parse_field(hist_data, file, str, flags);
3768         if (IS_ERR(field)) {
3769                 ret = PTR_ERR(field);
3770                 goto out;
3771         }
3772
3773         hist_field = create_hist_field(hist_data, field, *flags, var_name);
3774         if (!hist_field) {
3775                 ret = -ENOMEM;
3776                 goto out;
3777         }
3778
3779         return hist_field;
3780  out:
3781         return ERR_PTR(ret);
3782 }
3783
3784 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3785                                      struct trace_event_file *file,
3786                                      char *str, unsigned long flags,
3787                                      char *var_name, unsigned int level);
3788
3789 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
3790                                       struct trace_event_file *file,
3791                                       char *str, unsigned long flags,
3792                                       char *var_name, unsigned int level)
3793 {
3794         struct hist_field *operand1, *expr = NULL;
3795         unsigned long operand_flags;
3796         int ret = 0;
3797         char *s;
3798
3799         /* we support only -(xxx) i.e. explicit parens required */
3800
3801         if (level > 3) {
3802                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3803                 ret = -EINVAL;
3804                 goto free;
3805         }
3806
3807         str++; /* skip leading '-' */
3808
3809         s = strchr(str, '(');
3810         if (s)
3811                 str++;
3812         else {
3813                 ret = -EINVAL;
3814                 goto free;
3815         }
3816
3817         s = strrchr(str, ')');
3818         if (s)
3819                 *s = '\0';
3820         else {
3821                 ret = -EINVAL; /* no closing ')' */
3822                 goto free;
3823         }
3824
3825         flags |= HIST_FIELD_FL_EXPR;
3826         expr = create_hist_field(hist_data, NULL, flags, var_name);
3827         if (!expr) {
3828                 ret = -ENOMEM;
3829                 goto free;
3830         }
3831
3832         operand_flags = 0;
3833         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3834         if (IS_ERR(operand1)) {
3835                 ret = PTR_ERR(operand1);
3836                 goto free;
3837         }
3838
3839         expr->flags |= operand1->flags &
3840                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3841         expr->fn = hist_field_unary_minus;
3842         expr->operands[0] = operand1;
3843         expr->operator = FIELD_OP_UNARY_MINUS;
3844         expr->name = expr_str(expr, 0);
3845         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3846         if (!expr->type) {
3847                 ret = -ENOMEM;
3848                 goto free;
3849         }
3850
3851         return expr;
3852  free:
3853         destroy_hist_field(expr, 0);
3854         return ERR_PTR(ret);
3855 }
3856
3857 static int check_expr_operands(struct trace_array *tr,
3858                                struct hist_field *operand1,
3859                                struct hist_field *operand2)
3860 {
3861         unsigned long operand1_flags = operand1->flags;
3862         unsigned long operand2_flags = operand2->flags;
3863
3864         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
3865             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
3866                 struct hist_field *var;
3867
3868                 var = find_var_field(operand1->var.hist_data, operand1->name);
3869                 if (!var)
3870                         return -EINVAL;
3871                 operand1_flags = var->flags;
3872         }
3873
3874         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
3875             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
3876                 struct hist_field *var;
3877
3878                 var = find_var_field(operand2->var.hist_data, operand2->name);
3879                 if (!var)
3880                         return -EINVAL;
3881                 operand2_flags = var->flags;
3882         }
3883
3884         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
3885             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
3886                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
3887                 return -EINVAL;
3888         }
3889
3890         return 0;
3891 }
3892
3893 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3894                                      struct trace_event_file *file,
3895                                      char *str, unsigned long flags,
3896                                      char *var_name, unsigned int level)
3897 {
3898         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
3899         unsigned long operand_flags;
3900         int field_op, ret = -EINVAL;
3901         char *sep, *operand1_str;
3902
3903         if (level > 3) {
3904                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3905                 return ERR_PTR(-EINVAL);
3906         }
3907
3908         field_op = contains_operator(str);
3909
3910         if (field_op == FIELD_OP_NONE)
3911                 return parse_atom(hist_data, file, str, &flags, var_name);
3912
3913         if (field_op == FIELD_OP_UNARY_MINUS)
3914                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
3915
3916         switch (field_op) {
3917         case FIELD_OP_MINUS:
3918                 sep = "-";
3919                 break;
3920         case FIELD_OP_PLUS:
3921                 sep = "+";
3922                 break;
3923         default:
3924                 goto free;
3925         }
3926
3927         operand1_str = strsep(&str, sep);
3928         if (!operand1_str || !str)
3929                 goto free;
3930
3931         operand_flags = 0;
3932         operand1 = parse_atom(hist_data, file, operand1_str,
3933                               &operand_flags, NULL);
3934         if (IS_ERR(operand1)) {
3935                 ret = PTR_ERR(operand1);
3936                 operand1 = NULL;
3937                 goto free;
3938         }
3939
3940         /* rest of string could be another expression e.g. b+c in a+b+c */
3941         operand_flags = 0;
3942         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3943         if (IS_ERR(operand2)) {
3944                 ret = PTR_ERR(operand2);
3945                 operand2 = NULL;
3946                 goto free;
3947         }
3948
3949         ret = check_expr_operands(file->tr, operand1, operand2);
3950         if (ret)
3951                 goto free;
3952
3953         flags |= HIST_FIELD_FL_EXPR;
3954
3955         flags |= operand1->flags &
3956                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3957
3958         expr = create_hist_field(hist_data, NULL, flags, var_name);
3959         if (!expr) {
3960                 ret = -ENOMEM;
3961                 goto free;
3962         }
3963
3964         operand1->read_once = true;
3965         operand2->read_once = true;
3966
3967         expr->operands[0] = operand1;
3968         expr->operands[1] = operand2;
3969         expr->operator = field_op;
3970         expr->name = expr_str(expr, 0);
3971         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3972         if (!expr->type) {
3973                 ret = -ENOMEM;
3974                 goto free;
3975         }
3976
3977         switch (field_op) {
3978         case FIELD_OP_MINUS:
3979                 expr->fn = hist_field_minus;
3980                 break;
3981         case FIELD_OP_PLUS:
3982                 expr->fn = hist_field_plus;
3983                 break;
3984         default:
3985                 ret = -EINVAL;
3986                 goto free;
3987         }
3988
3989         return expr;
3990  free:
3991         destroy_hist_field(operand1, 0);
3992         destroy_hist_field(operand2, 0);
3993         destroy_hist_field(expr, 0);
3994
3995         return ERR_PTR(ret);
3996 }
3997
3998 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3999                                  struct trace_event_file *file)
4000 {
4001         struct event_trigger_data *test;
4002
4003         lockdep_assert_held(&event_mutex);
4004
4005         list_for_each_entry(test, &file->triggers, list) {
4006                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
4007                         if (test->private_data == hist_data)
4008                                 return test->filter_str;
4009                 }
4010         }
4011
4012         return NULL;
4013 }
4014
4015 static struct event_command trigger_hist_cmd;
4016 static int event_hist_trigger_func(struct event_command *cmd_ops,
4017                                    struct trace_event_file *file,
4018                                    char *glob, char *cmd, char *param);
4019
4020 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
4021                             struct hist_trigger_data *hist_data,
4022                             unsigned int n_keys)
4023 {
4024         struct hist_field *target_hist_field, *hist_field;
4025         unsigned int n, i, j;
4026
4027         if (hist_data->n_fields - hist_data->n_vals != n_keys)
4028                 return false;
4029
4030         i = hist_data->n_vals;
4031         j = target_hist_data->n_vals;
4032
4033         for (n = 0; n < n_keys; n++) {
4034                 hist_field = hist_data->fields[i + n];
4035                 target_hist_field = target_hist_data->fields[j + n];
4036
4037                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
4038                         return false;
4039                 if (hist_field->size != target_hist_field->size)
4040                         return false;
4041                 if (hist_field->is_signed != target_hist_field->is_signed)
4042                         return false;
4043         }
4044
4045         return true;
4046 }
4047
4048 static struct hist_trigger_data *
4049 find_compatible_hist(struct hist_trigger_data *target_hist_data,
4050                      struct trace_event_file *file)
4051 {
4052         struct hist_trigger_data *hist_data;
4053         struct event_trigger_data *test;
4054         unsigned int n_keys;
4055
4056         lockdep_assert_held(&event_mutex);
4057
4058         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
4059
4060         list_for_each_entry(test, &file->triggers, list) {
4061                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
4062                         hist_data = test->private_data;
4063
4064                         if (compatible_keys(target_hist_data, hist_data, n_keys))
4065                                 return hist_data;
4066                 }
4067         }
4068
4069         return NULL;
4070 }
4071
4072 static struct trace_event_file *event_file(struct trace_array *tr,
4073                                            char *system, char *event_name)
4074 {
4075         struct trace_event_file *file;
4076
4077         file = __find_event_file(tr, system, event_name);
4078         if (!file)
4079                 return ERR_PTR(-EINVAL);
4080
4081         return file;
4082 }
4083
4084 static struct hist_field *
4085 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
4086                          char *system, char *event_name, char *field_name)
4087 {
4088         struct hist_field *event_var;
4089         char *synthetic_name;
4090
4091         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
4092         if (!synthetic_name)
4093                 return ERR_PTR(-ENOMEM);
4094
4095         strcpy(synthetic_name, "synthetic_");
4096         strcat(synthetic_name, field_name);
4097
4098         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
4099
4100         kfree(synthetic_name);
4101
4102         return event_var;
4103 }
4104
4105 /**
4106  * create_field_var_hist - Automatically create a histogram and var for a field
4107  * @target_hist_data: The target hist trigger
4108  * @subsys_name: Optional subsystem name
4109  * @event_name: Optional event name
4110  * @field_name: The name of the field (and the resulting variable)
4111  *
4112  * Hist trigger actions fetch data from variables, not directly from
4113  * events.  However, for convenience, users are allowed to directly
4114  * specify an event field in an action, which will be automatically
4115  * converted into a variable on their behalf.
4116
4117  * If a user specifies a field on an event that isn't the event the
4118  * histogram currently being defined (the target event histogram), the
4119  * only way that can be accomplished is if a new hist trigger is
4120  * created and the field variable defined on that.
4121  *
4122  * This function creates a new histogram compatible with the target
4123  * event (meaning a histogram with the same key as the target
4124  * histogram), and creates a variable for the specified field, but
4125  * with 'synthetic_' prepended to the variable name in order to avoid
4126  * collision with normal field variables.
4127  *
4128  * Return: The variable created for the field.
4129  */
4130 static struct hist_field *
4131 create_field_var_hist(struct hist_trigger_data *target_hist_data,
4132                       char *subsys_name, char *event_name, char *field_name)
4133 {
4134         struct trace_array *tr = target_hist_data->event_file->tr;
4135         struct hist_field *event_var = ERR_PTR(-EINVAL);
4136         struct hist_trigger_data *hist_data;
4137         unsigned int i, n, first = true;
4138         struct field_var_hist *var_hist;
4139         struct trace_event_file *file;
4140         struct hist_field *key_field;
4141         char *saved_filter;
4142         char *cmd;
4143         int ret;
4144
4145         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
4146                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
4147                 return ERR_PTR(-EINVAL);
4148         }
4149
4150         file = event_file(tr, subsys_name, event_name);
4151
4152         if (IS_ERR(file)) {
4153                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
4154                 ret = PTR_ERR(file);
4155                 return ERR_PTR(ret);
4156         }
4157
4158         /*
4159          * Look for a histogram compatible with target.  We'll use the
4160          * found histogram specification to create a new matching
4161          * histogram with our variable on it.  target_hist_data is not
4162          * yet a registered histogram so we can't use that.
4163          */
4164         hist_data = find_compatible_hist(target_hist_data, file);
4165         if (!hist_data) {
4166                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
4167                 return ERR_PTR(-EINVAL);
4168         }
4169
4170         /* See if a synthetic field variable has already been created */
4171         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
4172                                              event_name, field_name);
4173         if (!IS_ERR_OR_NULL(event_var))
4174                 return event_var;
4175
4176         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
4177         if (!var_hist)
4178                 return ERR_PTR(-ENOMEM);
4179
4180         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
4181         if (!cmd) {
4182                 kfree(var_hist);
4183                 return ERR_PTR(-ENOMEM);
4184         }
4185
4186         /* Use the same keys as the compatible histogram */
4187         strcat(cmd, "keys=");
4188
4189         for_each_hist_key_field(i, hist_data) {
4190                 key_field = hist_data->fields[i];
4191                 if (!first)
4192                         strcat(cmd, ",");
4193                 strcat(cmd, key_field->field->name);
4194                 first = false;
4195         }
4196
4197         /* Create the synthetic field variable specification */
4198         strcat(cmd, ":synthetic_");
4199         strcat(cmd, field_name);
4200         strcat(cmd, "=");
4201         strcat(cmd, field_name);
4202
4203         /* Use the same filter as the compatible histogram */
4204         saved_filter = find_trigger_filter(hist_data, file);
4205         if (saved_filter) {
4206                 strcat(cmd, " if ");
4207                 strcat(cmd, saved_filter);
4208         }
4209
4210         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
4211         if (!var_hist->cmd) {
4212                 kfree(cmd);
4213                 kfree(var_hist);
4214                 return ERR_PTR(-ENOMEM);
4215         }
4216
4217         /* Save the compatible histogram information */
4218         var_hist->hist_data = hist_data;
4219
4220         /* Create the new histogram with our variable */
4221         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
4222                                       "", "hist", cmd);
4223         if (ret) {
4224                 kfree(cmd);
4225                 kfree(var_hist->cmd);
4226                 kfree(var_hist);
4227                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
4228                 return ERR_PTR(ret);
4229         }
4230
4231         kfree(cmd);
4232
4233         /* If we can't find the variable, something went wrong */
4234         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
4235                                              event_name, field_name);
4236         if (IS_ERR_OR_NULL(event_var)) {
4237                 kfree(var_hist->cmd);
4238                 kfree(var_hist);
4239                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
4240                 return ERR_PTR(-EINVAL);
4241         }
4242
4243         n = target_hist_data->n_field_var_hists;
4244         target_hist_data->field_var_hists[n] = var_hist;
4245         target_hist_data->n_field_var_hists++;
4246
4247         return event_var;
4248 }
4249
4250 static struct hist_field *
4251 find_target_event_var(struct hist_trigger_data *hist_data,
4252                       char *subsys_name, char *event_name, char *var_name)
4253 {
4254         struct trace_event_file *file = hist_data->event_file;
4255         struct hist_field *hist_field = NULL;
4256
4257         if (subsys_name) {
4258                 struct trace_event_call *call;
4259
4260                 if (!event_name)
4261                         return NULL;
4262
4263                 call = file->event_call;
4264
4265                 if (strcmp(subsys_name, call->class->system) != 0)
4266                         return NULL;
4267
4268                 if (strcmp(event_name, trace_event_name(call)) != 0)
4269                         return NULL;
4270         }
4271
4272         hist_field = find_var_field(hist_data, var_name);
4273
4274         return hist_field;
4275 }
4276
4277 static inline void __update_field_vars(struct tracing_map_elt *elt,
4278                                        struct ring_buffer_event *rbe,
4279                                        void *rec,
4280                                        struct field_var **field_vars,
4281                                        unsigned int n_field_vars,
4282                                        unsigned int field_var_str_start)
4283 {
4284         struct hist_elt_data *elt_data = elt->private_data;
4285         unsigned int i, j, var_idx;
4286         u64 var_val;
4287
4288         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
4289                 struct field_var *field_var = field_vars[i];
4290                 struct hist_field *var = field_var->var;
4291                 struct hist_field *val = field_var->val;
4292
4293                 var_val = val->fn(val, elt, rbe, rec);
4294                 var_idx = var->var.idx;
4295
4296                 if (val->flags & HIST_FIELD_FL_STRING) {
4297                         char *str = elt_data->field_var_str[j++];
4298                         char *val_str = (char *)(uintptr_t)var_val;
4299
4300                         strscpy(str, val_str, STR_VAR_LEN_MAX);
4301                         var_val = (u64)(uintptr_t)str;
4302                 }
4303                 tracing_map_set_var(elt, var_idx, var_val);
4304         }
4305 }
4306
4307 static void update_field_vars(struct hist_trigger_data *hist_data,
4308                               struct tracing_map_elt *elt,
4309                               struct ring_buffer_event *rbe,
4310                               void *rec)
4311 {
4312         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
4313                             hist_data->n_field_vars, 0);
4314 }
4315
4316 static void save_track_data_vars(struct hist_trigger_data *hist_data,
4317                                  struct tracing_map_elt *elt, void *rec,
4318                                  struct ring_buffer_event *rbe, void *key,
4319                                  struct action_data *data, u64 *var_ref_vals)
4320 {
4321         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
4322                             hist_data->n_save_vars, hist_data->n_field_var_str);
4323 }
4324
4325 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
4326                                      struct trace_event_file *file,
4327                                      char *name, int size, const char *type)
4328 {
4329         struct hist_field *var;
4330         int idx;
4331
4332         if (find_var(hist_data, file, name) && !hist_data->remove) {
4333                 var = ERR_PTR(-EINVAL);
4334                 goto out;
4335         }
4336
4337         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
4338         if (!var) {
4339                 var = ERR_PTR(-ENOMEM);
4340                 goto out;
4341         }
4342
4343         idx = tracing_map_add_var(hist_data->map);
4344         if (idx < 0) {
4345                 kfree(var);
4346                 var = ERR_PTR(-EINVAL);
4347                 goto out;
4348         }
4349
4350         var->flags = HIST_FIELD_FL_VAR;
4351         var->var.idx = idx;
4352         var->var.hist_data = var->hist_data = hist_data;
4353         var->size = size;
4354         var->var.name = kstrdup(name, GFP_KERNEL);
4355         var->type = kstrdup(type, GFP_KERNEL);
4356         if (!var->var.name || !var->type) {
4357                 kfree(var->var.name);
4358                 kfree(var->type);
4359                 kfree(var);
4360                 var = ERR_PTR(-ENOMEM);
4361         }
4362  out:
4363         return var;
4364 }
4365
4366 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
4367                                           struct trace_event_file *file,
4368                                           char *field_name)
4369 {
4370         struct hist_field *val = NULL, *var = NULL;
4371         unsigned long flags = HIST_FIELD_FL_VAR;
4372         struct trace_array *tr = file->tr;
4373         struct field_var *field_var;
4374         int ret = 0;
4375
4376         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
4377                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
4378                 ret = -EINVAL;
4379                 goto err;
4380         }
4381
4382         val = parse_atom(hist_data, file, field_name, &flags, NULL);
4383         if (IS_ERR(val)) {
4384                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
4385                 ret = PTR_ERR(val);
4386                 goto err;
4387         }
4388
4389         var = create_var(hist_data, file, field_name, val->size, val->type);
4390         if (IS_ERR(var)) {
4391                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
4392                 kfree(val);
4393                 ret = PTR_ERR(var);
4394                 goto err;
4395         }
4396
4397         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
4398         if (!field_var) {
4399                 kfree(val);
4400                 kfree(var);
4401                 ret =  -ENOMEM;
4402                 goto err;
4403         }
4404
4405         field_var->var = var;
4406         field_var->val = val;
4407  out:
4408         return field_var;
4409  err:
4410         field_var = ERR_PTR(ret);
4411         goto out;
4412 }
4413
4414 /**
4415  * create_target_field_var - Automatically create a variable for a field
4416  * @target_hist_data: The target hist trigger
4417  * @subsys_name: Optional subsystem name
4418  * @event_name: Optional event name
4419  * @var_name: The name of the field (and the resulting variable)
4420  *
4421  * Hist trigger actions fetch data from variables, not directly from
4422  * events.  However, for convenience, users are allowed to directly
4423  * specify an event field in an action, which will be automatically
4424  * converted into a variable on their behalf.
4425
4426  * This function creates a field variable with the name var_name on
4427  * the hist trigger currently being defined on the target event.  If
4428  * subsys_name and event_name are specified, this function simply
4429  * verifies that they do in fact match the target event subsystem and
4430  * event name.
4431  *
4432  * Return: The variable created for the field.
4433  */
4434 static struct field_var *
4435 create_target_field_var(struct hist_trigger_data *target_hist_data,
4436                         char *subsys_name, char *event_name, char *var_name)
4437 {
4438         struct trace_event_file *file = target_hist_data->event_file;
4439
4440         if (subsys_name) {
4441                 struct trace_event_call *call;
4442
4443                 if (!event_name)
4444                         return NULL;
4445
4446                 call = file->event_call;
4447
4448                 if (strcmp(subsys_name, call->class->system) != 0)
4449                         return NULL;
4450
4451                 if (strcmp(event_name, trace_event_name(call)) != 0)
4452                         return NULL;
4453         }
4454
4455         return create_field_var(target_hist_data, file, var_name);
4456 }
4457
4458 static bool check_track_val_max(u64 track_val, u64 var_val)
4459 {
4460         if (var_val <= track_val)
4461                 return false;
4462
4463         return true;
4464 }
4465
4466 static bool check_track_val_changed(u64 track_val, u64 var_val)
4467 {
4468         if (var_val == track_val)
4469                 return false;
4470
4471         return true;
4472 }
4473
4474 static u64 get_track_val(struct hist_trigger_data *hist_data,
4475                          struct tracing_map_elt *elt,
4476                          struct action_data *data)
4477 {
4478         unsigned int track_var_idx = data->track_data.track_var->var.idx;
4479         u64 track_val;
4480
4481         track_val = tracing_map_read_var(elt, track_var_idx);
4482
4483         return track_val;
4484 }
4485
4486 static void save_track_val(struct hist_trigger_data *hist_data,
4487                            struct tracing_map_elt *elt,
4488                            struct action_data *data, u64 var_val)
4489 {
4490         unsigned int track_var_idx = data->track_data.track_var->var.idx;
4491
4492         tracing_map_set_var(elt, track_var_idx, var_val);
4493 }
4494
4495 static void save_track_data(struct hist_trigger_data *hist_data,
4496                             struct tracing_map_elt *elt, void *rec,
4497                             struct ring_buffer_event *rbe, void *key,
4498                             struct action_data *data, u64 *var_ref_vals)
4499 {
4500         if (data->track_data.save_data)
4501                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4502 }
4503
4504 static bool check_track_val(struct tracing_map_elt *elt,
4505                             struct action_data *data,
4506                             u64 var_val)
4507 {
4508         struct hist_trigger_data *hist_data;
4509         u64 track_val;
4510
4511         hist_data = data->track_data.track_var->hist_data;
4512         track_val = get_track_val(hist_data, elt, data);
4513
4514         return data->track_data.check_val(track_val, var_val);
4515 }
4516
4517 #ifdef CONFIG_TRACER_SNAPSHOT
4518 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
4519 {
4520         /* called with tr->max_lock held */
4521         struct track_data *track_data = tr->cond_snapshot->cond_data;
4522         struct hist_elt_data *elt_data, *track_elt_data;
4523         struct snapshot_context *context = cond_data;
4524         struct action_data *action;
4525         u64 track_val;
4526
4527         if (!track_data)
4528                 return false;
4529
4530         action = track_data->action_data;
4531
4532         track_val = get_track_val(track_data->hist_data, context->elt,
4533                                   track_data->action_data);
4534
4535         if (!action->track_data.check_val(track_data->track_val, track_val))
4536                 return false;
4537
4538         track_data->track_val = track_val;
4539         memcpy(track_data->key, context->key, track_data->key_len);
4540
4541         elt_data = context->elt->private_data;
4542         track_elt_data = track_data->elt.private_data;
4543         if (elt_data->comm)
4544                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
4545
4546         track_data->updated = true;
4547
4548         return true;
4549 }
4550
4551 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
4552                                      struct tracing_map_elt *elt, void *rec,
4553                                      struct ring_buffer_event *rbe, void *key,
4554                                      struct action_data *data,
4555                                      u64 *var_ref_vals)
4556 {
4557         struct trace_event_file *file = hist_data->event_file;
4558         struct snapshot_context context;
4559
4560         context.elt = elt;
4561         context.key = key;
4562
4563         tracing_snapshot_cond(file->tr, &context);
4564 }
4565
4566 static void hist_trigger_print_key(struct seq_file *m,
4567                                    struct hist_trigger_data *hist_data,
4568                                    void *key,
4569                                    struct tracing_map_elt *elt);
4570
4571 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
4572 {
4573         unsigned int i;
4574
4575         if (!hist_data->n_actions)
4576                 return NULL;
4577
4578         for (i = 0; i < hist_data->n_actions; i++) {
4579                 struct action_data *data = hist_data->actions[i];
4580
4581                 if (data->action == ACTION_SNAPSHOT)
4582                         return data;
4583         }
4584
4585         return NULL;
4586 }
4587
4588 static void track_data_snapshot_print(struct seq_file *m,
4589                                       struct hist_trigger_data *hist_data)
4590 {
4591         struct trace_event_file *file = hist_data->event_file;
4592         struct track_data *track_data;
4593         struct action_data *action;
4594
4595         track_data = tracing_cond_snapshot_data(file->tr);
4596         if (!track_data)
4597                 return;
4598
4599         if (!track_data->updated)
4600                 return;
4601
4602         action = snapshot_action(hist_data);
4603         if (!action)
4604                 return;
4605
4606         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
4607         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
4608                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
4609                    action->track_data.var_str, track_data->track_val);
4610
4611         seq_puts(m, "\ttriggered by event with key: ");
4612         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
4613         seq_putc(m, '\n');
4614 }
4615 #else
4616 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
4617 {
4618         return false;
4619 }
4620 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
4621                                      struct tracing_map_elt *elt, void *rec,
4622                                      struct ring_buffer_event *rbe, void *key,
4623                                      struct action_data *data,
4624                                      u64 *var_ref_vals) {}
4625 static void track_data_snapshot_print(struct seq_file *m,
4626                                       struct hist_trigger_data *hist_data) {}
4627 #endif /* CONFIG_TRACER_SNAPSHOT */
4628
4629 static void track_data_print(struct seq_file *m,
4630                              struct hist_trigger_data *hist_data,
4631                              struct tracing_map_elt *elt,
4632                              struct action_data *data)
4633 {
4634         u64 track_val = get_track_val(hist_data, elt, data);
4635         unsigned int i, save_var_idx;
4636
4637         if (data->handler == HANDLER_ONMAX)
4638                 seq_printf(m, "\n\tmax: %10llu", track_val);
4639         else if (data->handler == HANDLER_ONCHANGE)
4640                 seq_printf(m, "\n\tchanged: %10llu", track_val);
4641
4642         if (data->action == ACTION_SNAPSHOT)
4643                 return;
4644
4645         for (i = 0; i < hist_data->n_save_vars; i++) {
4646                 struct hist_field *save_val = hist_data->save_vars[i]->val;
4647                 struct hist_field *save_var = hist_data->save_vars[i]->var;
4648                 u64 val;
4649
4650                 save_var_idx = save_var->var.idx;
4651
4652                 val = tracing_map_read_var(elt, save_var_idx);
4653
4654                 if (save_val->flags & HIST_FIELD_FL_STRING) {
4655                         seq_printf(m, "  %s: %-32s", save_var->var.name,
4656                                    (char *)(uintptr_t)(val));
4657                 } else
4658                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
4659         }
4660 }
4661
4662 static void ontrack_action(struct hist_trigger_data *hist_data,
4663                            struct tracing_map_elt *elt, void *rec,
4664                            struct ring_buffer_event *rbe, void *key,
4665                            struct action_data *data, u64 *var_ref_vals)
4666 {
4667         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
4668
4669         if (check_track_val(elt, data, var_val)) {
4670                 save_track_val(hist_data, elt, data, var_val);
4671                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4672         }
4673 }
4674
4675 static void action_data_destroy(struct action_data *data)
4676 {
4677         unsigned int i;
4678
4679         lockdep_assert_held(&event_mutex);
4680
4681         kfree(data->action_name);
4682
4683         for (i = 0; i < data->n_params; i++)
4684                 kfree(data->params[i]);
4685
4686         if (data->synth_event)
4687                 data->synth_event->ref--;
4688
4689         kfree(data->synth_event_name);
4690
4691         kfree(data);
4692 }
4693
4694 static void track_data_destroy(struct hist_trigger_data *hist_data,
4695                                struct action_data *data)
4696 {
4697         struct trace_event_file *file = hist_data->event_file;
4698
4699         destroy_hist_field(data->track_data.track_var, 0);
4700
4701         if (data->action == ACTION_SNAPSHOT) {
4702                 struct track_data *track_data;
4703
4704                 track_data = tracing_cond_snapshot_data(file->tr);
4705                 if (track_data && track_data->hist_data == hist_data) {
4706                         tracing_snapshot_cond_disable(file->tr);
4707                         track_data_free(track_data);
4708                 }
4709         }
4710
4711         kfree(data->track_data.var_str);
4712
4713         action_data_destroy(data);
4714 }
4715
4716 static int action_create(struct hist_trigger_data *hist_data,
4717                          struct action_data *data);
4718
4719 static int track_data_create(struct hist_trigger_data *hist_data,
4720                              struct action_data *data)
4721 {
4722         struct hist_field *var_field, *ref_field, *track_var = NULL;
4723         struct trace_event_file *file = hist_data->event_file;
4724         struct trace_array *tr = file->tr;
4725         char *track_data_var_str;
4726         int ret = 0;
4727
4728         track_data_var_str = data->track_data.var_str;
4729         if (track_data_var_str[0] != '$') {
4730                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
4731                 return -EINVAL;
4732         }
4733         track_data_var_str++;
4734
4735         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
4736         if (!var_field) {
4737                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
4738                 return -EINVAL;
4739         }
4740
4741         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
4742         if (!ref_field)
4743                 return -ENOMEM;
4744
4745         data->track_data.var_ref = ref_field;
4746
4747         if (data->handler == HANDLER_ONMAX)
4748                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
4749         if (IS_ERR(track_var)) {
4750                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
4751                 ret = PTR_ERR(track_var);
4752                 goto out;
4753         }
4754
4755         if (data->handler == HANDLER_ONCHANGE)
4756                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
4757         if (IS_ERR(track_var)) {
4758                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
4759                 ret = PTR_ERR(track_var);
4760                 goto out;
4761         }
4762         data->track_data.track_var = track_var;
4763
4764         ret = action_create(hist_data, data);
4765  out:
4766         return ret;
4767 }
4768
4769 static int parse_action_params(struct trace_array *tr, char *params,
4770                                struct action_data *data)
4771 {
4772         char *param, *saved_param;
4773         bool first_param = true;
4774         int ret = 0;
4775
4776         while (params) {
4777                 if (data->n_params >= SYNTH_FIELDS_MAX) {
4778                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
4779                         goto out;
4780                 }
4781
4782                 param = strsep(&params, ",");
4783                 if (!param) {
4784                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
4785                         ret = -EINVAL;
4786                         goto out;
4787                 }
4788
4789                 param = strstrip(param);
4790                 if (strlen(param) < 2) {
4791                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
4792                         ret = -EINVAL;
4793                         goto out;
4794                 }
4795
4796                 saved_param = kstrdup(param, GFP_KERNEL);
4797                 if (!saved_param) {
4798                         ret = -ENOMEM;
4799                         goto out;
4800                 }
4801
4802                 if (first_param && data->use_trace_keyword) {
4803                         data->synth_event_name = saved_param;
4804                         first_param = false;
4805                         continue;
4806                 }
4807                 first_param = false;
4808
4809                 data->params[data->n_params++] = saved_param;
4810         }
4811  out:
4812         return ret;
4813 }
4814
4815 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
4816                         enum handler_id handler)
4817 {
4818         char *action_name;
4819         int ret = 0;
4820
4821         strsep(&str, ".");
4822         if (!str) {
4823                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
4824                 ret = -EINVAL;
4825                 goto out;
4826         }
4827
4828         action_name = strsep(&str, "(");
4829         if (!action_name || !str) {
4830                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
4831                 ret = -EINVAL;
4832                 goto out;
4833         }
4834
4835         if (str_has_prefix(action_name, "save")) {
4836                 char *params = strsep(&str, ")");
4837
4838                 if (!params) {
4839                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
4840                         ret = -EINVAL;
4841                         goto out;
4842                 }
4843
4844                 ret = parse_action_params(tr, params, data);
4845                 if (ret)
4846                         goto out;
4847
4848                 if (handler == HANDLER_ONMAX)
4849                         data->track_data.check_val = check_track_val_max;
4850                 else if (handler == HANDLER_ONCHANGE)
4851                         data->track_data.check_val = check_track_val_changed;
4852                 else {
4853                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
4854                         ret = -EINVAL;
4855                         goto out;
4856                 }
4857
4858                 data->track_data.save_data = save_track_data_vars;
4859                 data->fn = ontrack_action;
4860                 data->action = ACTION_SAVE;
4861         } else if (str_has_prefix(action_name, "snapshot")) {
4862                 char *params = strsep(&str, ")");
4863
4864                 if (!str) {
4865                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
4866                         ret = -EINVAL;
4867                         goto out;
4868                 }
4869
4870                 if (handler == HANDLER_ONMAX)
4871                         data->track_data.check_val = check_track_val_max;
4872                 else if (handler == HANDLER_ONCHANGE)
4873                         data->track_data.check_val = check_track_val_changed;
4874                 else {
4875                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
4876                         ret = -EINVAL;
4877                         goto out;
4878                 }
4879
4880                 data->track_data.save_data = save_track_data_snapshot;
4881                 data->fn = ontrack_action;
4882                 data->action = ACTION_SNAPSHOT;
4883         } else {
4884                 char *params = strsep(&str, ")");
4885
4886                 if (str_has_prefix(action_name, "trace"))
4887                         data->use_trace_keyword = true;
4888
4889                 if (params) {
4890                         ret = parse_action_params(tr, params, data);
4891                         if (ret)
4892                                 goto out;
4893                 }
4894
4895                 if (handler == HANDLER_ONMAX)
4896                         data->track_data.check_val = check_track_val_max;
4897                 else if (handler == HANDLER_ONCHANGE)
4898                         data->track_data.check_val = check_track_val_changed;
4899
4900                 if (handler != HANDLER_ONMATCH) {
4901                         data->track_data.save_data = action_trace;
4902                         data->fn = ontrack_action;
4903                 } else
4904                         data->fn = action_trace;
4905
4906                 data->action = ACTION_TRACE;
4907         }
4908
4909         data->action_name = kstrdup(action_name, GFP_KERNEL);
4910         if (!data->action_name) {
4911                 ret = -ENOMEM;
4912                 goto out;
4913         }
4914
4915         data->handler = handler;
4916  out:
4917         return ret;
4918 }
4919
4920 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4921                                             char *str, enum handler_id handler)
4922 {
4923         struct action_data *data;
4924         int ret = -EINVAL;
4925         char *var_str;
4926
4927         data = kzalloc(sizeof(*data), GFP_KERNEL);
4928         if (!data)
4929                 return ERR_PTR(-ENOMEM);
4930
4931         var_str = strsep(&str, ")");
4932         if (!var_str || !str) {
4933                 ret = -EINVAL;
4934                 goto free;
4935         }
4936
4937         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4938         if (!data->track_data.var_str) {
4939                 ret = -ENOMEM;
4940                 goto free;
4941         }
4942
4943         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4944         if (ret)
4945                 goto free;
4946  out:
4947         return data;
4948  free:
4949         track_data_destroy(hist_data, data);
4950         data = ERR_PTR(ret);
4951         goto out;
4952 }
4953
4954 static void onmatch_destroy(struct action_data *data)
4955 {
4956         kfree(data->match_data.event);
4957         kfree(data->match_data.event_system);
4958
4959         action_data_destroy(data);
4960 }
4961
4962 static void destroy_field_var(struct field_var *field_var)
4963 {
4964         if (!field_var)
4965                 return;
4966
4967         destroy_hist_field(field_var->var, 0);
4968         destroy_hist_field(field_var->val, 0);
4969
4970         kfree(field_var);
4971 }
4972
4973 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4974 {
4975         unsigned int i;
4976
4977         for (i = 0; i < hist_data->n_field_vars; i++)
4978                 destroy_field_var(hist_data->field_vars[i]);
4979 }
4980
4981 static void save_field_var(struct hist_trigger_data *hist_data,
4982                            struct field_var *field_var)
4983 {
4984         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4985
4986         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4987                 hist_data->n_field_var_str++;
4988 }
4989
4990
4991 static int check_synth_field(struct synth_event *event,
4992                              struct hist_field *hist_field,
4993                              unsigned int field_pos)
4994 {
4995         struct synth_field *field;
4996
4997         if (field_pos >= event->n_fields)
4998                 return -EINVAL;
4999
5000         field = event->fields[field_pos];
5001
5002         if (strcmp(field->type, hist_field->type) != 0) {
5003                 if (field->size != hist_field->size ||
5004                     field->is_signed != hist_field->is_signed)
5005                         return -EINVAL;
5006         }
5007
5008         return 0;
5009 }
5010
5011 static struct hist_field *
5012 trace_action_find_var(struct hist_trigger_data *hist_data,
5013                       struct action_data *data,
5014                       char *system, char *event, char *var)
5015 {
5016         struct trace_array *tr = hist_data->event_file->tr;
5017         struct hist_field *hist_field;
5018
5019         var++; /* skip '$' */
5020
5021         hist_field = find_target_event_var(hist_data, system, event, var);
5022         if (!hist_field) {
5023                 if (!system && data->handler == HANDLER_ONMATCH) {
5024                         system = data->match_data.event_system;
5025                         event = data->match_data.event;
5026                 }
5027
5028                 hist_field = find_event_var(hist_data, system, event, var);
5029         }
5030
5031         if (!hist_field)
5032                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
5033
5034         return hist_field;
5035 }
5036
5037 static struct hist_field *
5038 trace_action_create_field_var(struct hist_trigger_data *hist_data,
5039                               struct action_data *data, char *system,
5040                               char *event, char *var)
5041 {
5042         struct hist_field *hist_field = NULL;
5043         struct field_var *field_var;
5044
5045         /*
5046          * First try to create a field var on the target event (the
5047          * currently being defined).  This will create a variable for
5048          * unqualified fields on the target event, or if qualified,
5049          * target fields that have qualified names matching the target.
5050          */
5051         field_var = create_target_field_var(hist_data, system, event, var);
5052
5053         if (field_var && !IS_ERR(field_var)) {
5054                 save_field_var(hist_data, field_var);
5055                 hist_field = field_var->var;
5056         } else {
5057                 field_var = NULL;
5058                 /*
5059                  * If no explicit system.event is specfied, default to
5060                  * looking for fields on the onmatch(system.event.xxx)
5061                  * event.
5062                  */
5063                 if (!system && data->handler == HANDLER_ONMATCH) {
5064                         system = data->match_data.event_system;
5065                         event = data->match_data.event;
5066                 }
5067
5068                 /*
5069                  * At this point, we're looking at a field on another
5070                  * event.  Because we can't modify a hist trigger on
5071                  * another event to add a variable for a field, we need
5072                  * to create a new trigger on that event and create the
5073                  * variable at the same time.
5074                  */
5075                 hist_field = create_field_var_hist(hist_data, system, event, var);
5076                 if (IS_ERR(hist_field))
5077                         goto free;
5078         }
5079  out:
5080         return hist_field;
5081  free:
5082         destroy_field_var(field_var);
5083         hist_field = NULL;
5084         goto out;
5085 }
5086
5087 static int trace_action_create(struct hist_trigger_data *hist_data,
5088                                struct action_data *data)
5089 {
5090         struct trace_array *tr = hist_data->event_file->tr;
5091         char *event_name, *param, *system = NULL;
5092         struct hist_field *hist_field, *var_ref;
5093         unsigned int i;
5094         unsigned int field_pos = 0;
5095         struct synth_event *event;
5096         char *synth_event_name;
5097         int var_ref_idx, ret = 0;
5098
5099         lockdep_assert_held(&event_mutex);
5100
5101         if (data->use_trace_keyword)
5102                 synth_event_name = data->synth_event_name;
5103         else
5104                 synth_event_name = data->action_name;
5105
5106         event = find_synth_event(synth_event_name);
5107         if (!event) {
5108                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
5109                 return -EINVAL;
5110         }
5111
5112         event->ref++;
5113
5114         for (i = 0; i < data->n_params; i++) {
5115                 char *p;
5116
5117                 p = param = kstrdup(data->params[i], GFP_KERNEL);
5118                 if (!param) {
5119                         ret = -ENOMEM;
5120                         goto err;
5121                 }
5122
5123                 system = strsep(&param, ".");
5124                 if (!param) {
5125                         param = (char *)system;
5126                         system = event_name = NULL;
5127                 } else {
5128                         event_name = strsep(&param, ".");
5129                         if (!param) {
5130                                 kfree(p);
5131                                 ret = -EINVAL;
5132                                 goto err;
5133                         }
5134                 }
5135
5136                 if (param[0] == '$')
5137                         hist_field = trace_action_find_var(hist_data, data,
5138                                                            system, event_name,
5139                                                            param);
5140                 else
5141                         hist_field = trace_action_create_field_var(hist_data,
5142                                                                    data,
5143                                                                    system,
5144                                                                    event_name,
5145                                                                    param);
5146
5147                 if (!hist_field) {
5148                         kfree(p);
5149                         ret = -EINVAL;
5150                         goto err;
5151                 }
5152
5153                 if (check_synth_field(event, hist_field, field_pos) == 0) {
5154                         var_ref = create_var_ref(hist_data, hist_field,
5155                                                  system, event_name);
5156                         if (!var_ref) {
5157                                 kfree(p);
5158                                 ret = -ENOMEM;
5159                                 goto err;
5160                         }
5161
5162                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
5163                         if (WARN_ON(var_ref_idx < 0)) {
5164                                 ret = var_ref_idx;
5165                                 goto err;
5166                         }
5167
5168                         data->var_ref_idx[i] = var_ref_idx;
5169
5170                         field_pos++;
5171                         kfree(p);
5172                         continue;
5173                 }
5174
5175                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
5176                 kfree(p);
5177                 ret = -EINVAL;
5178                 goto err;
5179         }
5180
5181         if (field_pos != event->n_fields) {
5182                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
5183                 ret = -EINVAL;
5184                 goto err;
5185         }
5186
5187         data->synth_event = event;
5188  out:
5189         return ret;
5190  err:
5191         event->ref--;
5192
5193         goto out;
5194 }
5195
5196 static int action_create(struct hist_trigger_data *hist_data,
5197                          struct action_data *data)
5198 {
5199         struct trace_event_file *file = hist_data->event_file;
5200         struct trace_array *tr = file->tr;
5201         struct track_data *track_data;
5202         struct field_var *field_var;
5203         unsigned int i;
5204         char *param;
5205         int ret = 0;
5206
5207         if (data->action == ACTION_TRACE)
5208                 return trace_action_create(hist_data, data);
5209
5210         if (data->action == ACTION_SNAPSHOT) {
5211                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
5212                 if (IS_ERR(track_data)) {
5213                         ret = PTR_ERR(track_data);
5214                         goto out;
5215                 }
5216
5217                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
5218                                                    cond_snapshot_update);
5219                 if (ret)
5220                         track_data_free(track_data);
5221
5222                 goto out;
5223         }
5224
5225         if (data->action == ACTION_SAVE) {
5226                 if (hist_data->n_save_vars) {
5227                         ret = -EEXIST;
5228                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
5229                         goto out;
5230                 }
5231
5232                 for (i = 0; i < data->n_params; i++) {
5233                         param = kstrdup(data->params[i], GFP_KERNEL);
5234                         if (!param) {
5235                                 ret = -ENOMEM;
5236                                 goto out;
5237                         }
5238
5239                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
5240                         if (IS_ERR(field_var)) {
5241                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
5242                                          errpos(param));
5243                                 ret = PTR_ERR(field_var);
5244                                 kfree(param);
5245                                 goto out;
5246                         }
5247
5248                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
5249                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
5250                                 hist_data->n_save_var_str++;
5251                         kfree(param);
5252                 }
5253         }
5254  out:
5255         return ret;
5256 }
5257
5258 static int onmatch_create(struct hist_trigger_data *hist_data,
5259                           struct action_data *data)
5260 {
5261         return action_create(hist_data, data);
5262 }
5263
5264 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
5265 {
5266         char *match_event, *match_event_system;
5267         struct action_data *data;
5268         int ret = -EINVAL;
5269
5270         data = kzalloc(sizeof(*data), GFP_KERNEL);
5271         if (!data)
5272                 return ERR_PTR(-ENOMEM);
5273
5274         match_event = strsep(&str, ")");
5275         if (!match_event || !str) {
5276                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
5277                 goto free;
5278         }
5279
5280         match_event_system = strsep(&match_event, ".");
5281         if (!match_event) {
5282                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
5283                 goto free;
5284         }
5285
5286         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
5287                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
5288                 goto free;
5289         }
5290
5291         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
5292         if (!data->match_data.event) {
5293                 ret = -ENOMEM;
5294                 goto free;
5295         }
5296
5297         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
5298         if (!data->match_data.event_system) {
5299                 ret = -ENOMEM;
5300                 goto free;
5301         }
5302
5303         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
5304         if (ret)
5305                 goto free;
5306  out:
5307         return data;
5308  free:
5309         onmatch_destroy(data);
5310         data = ERR_PTR(ret);
5311         goto out;
5312 }
5313
5314 static int create_hitcount_val(struct hist_trigger_data *hist_data)
5315 {
5316         hist_data->fields[HITCOUNT_IDX] =
5317                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
5318         if (!hist_data->fields[HITCOUNT_IDX])
5319                 return -ENOMEM;
5320
5321         hist_data->n_vals++;
5322         hist_data->n_fields++;
5323
5324         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
5325                 return -EINVAL;
5326
5327         return 0;
5328 }
5329
5330 static int __create_val_field(struct hist_trigger_data *hist_data,
5331                               unsigned int val_idx,
5332                               struct trace_event_file *file,
5333                               char *var_name, char *field_str,
5334                               unsigned long flags)
5335 {
5336         struct hist_field *hist_field;
5337         int ret = 0;
5338
5339         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
5340         if (IS_ERR(hist_field)) {
5341                 ret = PTR_ERR(hist_field);
5342                 goto out;
5343         }
5344
5345         hist_data->fields[val_idx] = hist_field;
5346
5347         ++hist_data->n_vals;
5348         ++hist_data->n_fields;
5349
5350         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
5351                 ret = -EINVAL;
5352  out:
5353         return ret;
5354 }
5355
5356 static int create_val_field(struct hist_trigger_data *hist_data,
5357                             unsigned int val_idx,
5358                             struct trace_event_file *file,
5359                             char *field_str)
5360 {
5361         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
5362                 return -EINVAL;
5363
5364         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
5365 }
5366
5367 static int create_var_field(struct hist_trigger_data *hist_data,
5368                             unsigned int val_idx,
5369                             struct trace_event_file *file,
5370                             char *var_name, char *expr_str)
5371 {
5372         struct trace_array *tr = hist_data->event_file->tr;
5373         unsigned long flags = 0;
5374
5375         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
5376                 return -EINVAL;
5377
5378         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
5379                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
5380                 return -EINVAL;
5381         }
5382
5383         flags |= HIST_FIELD_FL_VAR;
5384         hist_data->n_vars++;
5385         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
5386                 return -EINVAL;
5387
5388         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
5389 }
5390
5391 static int create_val_fields(struct hist_trigger_data *hist_data,
5392                              struct trace_event_file *file)
5393 {
5394         char *fields_str, *field_str;
5395         unsigned int i, j = 1;
5396         int ret;
5397
5398         ret = create_hitcount_val(hist_data);
5399         if (ret)
5400                 goto out;
5401
5402         fields_str = hist_data->attrs->vals_str;
5403         if (!fields_str)
5404                 goto out;
5405
5406         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
5407                      j < TRACING_MAP_VALS_MAX; i++) {
5408                 field_str = strsep(&fields_str, ",");
5409                 if (!field_str)
5410                         break;
5411
5412                 if (strcmp(field_str, "hitcount") == 0)
5413                         continue;
5414
5415                 ret = create_val_field(hist_data, j++, file, field_str);
5416                 if (ret)
5417                         goto out;
5418         }
5419
5420         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
5421                 ret = -EINVAL;
5422  out:
5423         return ret;
5424 }
5425
5426 static int create_key_field(struct hist_trigger_data *hist_data,
5427                             unsigned int key_idx,
5428                             unsigned int key_offset,
5429                             struct trace_event_file *file,
5430                             char *field_str)
5431 {
5432         struct trace_array *tr = hist_data->event_file->tr;
5433         struct hist_field *hist_field = NULL;
5434         unsigned long flags = 0;
5435         unsigned int key_size;
5436         int ret = 0;
5437
5438         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
5439                 return -EINVAL;
5440
5441         flags |= HIST_FIELD_FL_KEY;
5442
5443         if (strcmp(field_str, "stacktrace") == 0) {
5444                 flags |= HIST_FIELD_FL_STACKTRACE;
5445                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
5446                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
5447         } else {
5448                 hist_field = parse_expr(hist_data, file, field_str, flags,
5449                                         NULL, 0);
5450                 if (IS_ERR(hist_field)) {
5451                         ret = PTR_ERR(hist_field);
5452                         goto out;
5453                 }
5454
5455                 if (field_has_hist_vars(hist_field, 0)) {
5456                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
5457                         destroy_hist_field(hist_field, 0);
5458                         ret = -EINVAL;
5459                         goto out;
5460                 }
5461
5462                 key_size = hist_field->size;
5463         }
5464
5465         hist_data->fields[key_idx] = hist_field;
5466
5467         key_size = ALIGN(key_size, sizeof(u64));
5468         hist_data->fields[key_idx]->size = key_size;
5469         hist_data->fields[key_idx]->offset = key_offset;
5470
5471         hist_data->key_size += key_size;
5472
5473         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
5474                 ret = -EINVAL;
5475                 goto out;
5476         }
5477
5478         hist_data->n_keys++;
5479         hist_data->n_fields++;
5480
5481         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
5482                 return -EINVAL;
5483
5484         ret = key_size;
5485  out:
5486         return ret;
5487 }
5488
5489 static int create_key_fields(struct hist_trigger_data *hist_data,
5490                              struct trace_event_file *file)
5491 {
5492         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
5493         char *fields_str, *field_str;
5494         int ret = -EINVAL;
5495
5496         fields_str = hist_data->attrs->keys_str;
5497         if (!fields_str)
5498                 goto out;
5499
5500         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
5501                 field_str = strsep(&fields_str, ",");
5502                 if (!field_str)
5503                         break;
5504                 ret = create_key_field(hist_data, i, key_offset,
5505                                        file, field_str);
5506                 if (ret < 0)
5507                         goto out;
5508                 key_offset += ret;
5509         }
5510         if (fields_str) {
5511                 ret = -EINVAL;
5512                 goto out;
5513         }
5514         ret = 0;
5515  out:
5516         return ret;
5517 }
5518
5519 static int create_var_fields(struct hist_trigger_data *hist_data,
5520                              struct trace_event_file *file)
5521 {
5522         unsigned int i, j = hist_data->n_vals;
5523         int ret = 0;
5524
5525         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
5526
5527         for (i = 0; i < n_vars; i++) {
5528                 char *var_name = hist_data->attrs->var_defs.name[i];
5529                 char *expr = hist_data->attrs->var_defs.expr[i];
5530
5531                 ret = create_var_field(hist_data, j++, file, var_name, expr);
5532                 if (ret)
5533                         goto out;
5534         }
5535  out:
5536         return ret;
5537 }
5538
5539 static void free_var_defs(struct hist_trigger_data *hist_data)
5540 {
5541         unsigned int i;
5542
5543         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
5544                 kfree(hist_data->attrs->var_defs.name[i]);
5545                 kfree(hist_data->attrs->var_defs.expr[i]);
5546         }
5547
5548         hist_data->attrs->var_defs.n_vars = 0;
5549 }
5550
5551 static int parse_var_defs(struct hist_trigger_data *hist_data)
5552 {
5553         struct trace_array *tr = hist_data->event_file->tr;
5554         char *s, *str, *var_name, *field_str;
5555         unsigned int i, j, n_vars = 0;
5556         int ret = 0;
5557
5558         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
5559                 str = hist_data->attrs->assignment_str[i];
5560                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
5561                         field_str = strsep(&str, ",");
5562                         if (!field_str)
5563                                 break;
5564
5565                         var_name = strsep(&field_str, "=");
5566                         if (!var_name || !field_str) {
5567                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
5568                                          errpos(var_name));
5569                                 ret = -EINVAL;
5570                                 goto free;
5571                         }
5572
5573                         if (n_vars == TRACING_MAP_VARS_MAX) {
5574                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
5575                                 ret = -EINVAL;
5576                                 goto free;
5577                         }
5578
5579                         s = kstrdup(var_name, GFP_KERNEL);
5580                         if (!s) {
5581                                 ret = -ENOMEM;
5582                                 goto free;
5583                         }
5584                         hist_data->attrs->var_defs.name[n_vars] = s;
5585
5586                         s = kstrdup(field_str, GFP_KERNEL);
5587                         if (!s) {
5588                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
5589                                 ret = -ENOMEM;
5590                                 goto free;
5591                         }
5592                         hist_data->attrs->var_defs.expr[n_vars++] = s;
5593
5594                         hist_data->attrs->var_defs.n_vars = n_vars;
5595                 }
5596         }
5597
5598         return ret;
5599  free:
5600         free_var_defs(hist_data);
5601
5602         return ret;
5603 }
5604
5605 static int create_hist_fields(struct hist_trigger_data *hist_data,
5606                               struct trace_event_file *file)
5607 {
5608         int ret;
5609
5610         ret = parse_var_defs(hist_data);
5611         if (ret)
5612                 goto out;
5613
5614         ret = create_val_fields(hist_data, file);
5615         if (ret)
5616                 goto out;
5617
5618         ret = create_var_fields(hist_data, file);
5619         if (ret)
5620                 goto out;
5621
5622         ret = create_key_fields(hist_data, file);
5623         if (ret)
5624                 goto out;
5625  out:
5626         free_var_defs(hist_data);
5627
5628         return ret;
5629 }
5630
5631 static int is_descending(struct trace_array *tr, const char *str)
5632 {
5633         if (!str)
5634                 return 0;
5635
5636         if (strcmp(str, "descending") == 0)
5637                 return 1;
5638
5639         if (strcmp(str, "ascending") == 0)
5640                 return 0;
5641
5642         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
5643
5644         return -EINVAL;
5645 }
5646
5647 static int create_sort_keys(struct hist_trigger_data *hist_data)
5648 {
5649         struct trace_array *tr = hist_data->event_file->tr;
5650         char *fields_str = hist_data->attrs->sort_key_str;
5651         struct tracing_map_sort_key *sort_key;
5652         int descending, ret = 0;
5653         unsigned int i, j, k;
5654
5655         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
5656
5657         if (!fields_str)
5658                 goto out;
5659
5660         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
5661                 struct hist_field *hist_field;
5662                 char *field_str, *field_name;
5663                 const char *test_name;
5664
5665                 sort_key = &hist_data->sort_keys[i];
5666
5667                 field_str = strsep(&fields_str, ",");
5668                 if (!field_str)
5669                         break;
5670
5671                 if (!*field_str) {
5672                         ret = -EINVAL;
5673                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
5674                         break;
5675                 }
5676
5677                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
5678                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
5679                         ret = -EINVAL;
5680                         break;
5681                 }
5682
5683                 field_name = strsep(&field_str, ".");
5684                 if (!field_name || !*field_name) {
5685                         ret = -EINVAL;
5686                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
5687                         break;
5688                 }
5689
5690                 if (strcmp(field_name, "hitcount") == 0) {
5691                         descending = is_descending(tr, field_str);
5692                         if (descending < 0) {
5693                                 ret = descending;
5694                                 break;
5695                         }
5696                         sort_key->descending = descending;
5697                         continue;
5698                 }
5699
5700                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
5701                         unsigned int idx;
5702
5703                         hist_field = hist_data->fields[j];
5704                         if (hist_field->flags & HIST_FIELD_FL_VAR)
5705                                 continue;
5706
5707                         idx = k++;
5708
5709                         test_name = hist_field_name(hist_field, 0);
5710
5711                         if (strcmp(field_name, test_name) == 0) {
5712                                 sort_key->field_idx = idx;
5713                                 descending = is_descending(tr, field_str);
5714                                 if (descending < 0) {
5715                                         ret = descending;
5716                                         goto out;
5717                                 }
5718                                 sort_key->descending = descending;
5719                                 break;
5720                         }
5721                 }
5722                 if (j == hist_data->n_fields) {
5723                         ret = -EINVAL;
5724                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
5725                         break;
5726                 }
5727         }
5728
5729         hist_data->n_sort_keys = i;
5730  out:
5731         return ret;
5732 }
5733
5734 static void destroy_actions(struct hist_trigger_data *hist_data)
5735 {
5736         unsigned int i;
5737
5738         for (i = 0; i < hist_data->n_actions; i++) {
5739                 struct action_data *data = hist_data->actions[i];
5740
5741                 if (data->handler == HANDLER_ONMATCH)
5742                         onmatch_destroy(data);
5743                 else if (data->handler == HANDLER_ONMAX ||
5744                          data->handler == HANDLER_ONCHANGE)
5745                         track_data_destroy(hist_data, data);
5746                 else
5747                         kfree(data);
5748         }
5749 }
5750
5751 static int parse_actions(struct hist_trigger_data *hist_data)
5752 {
5753         struct trace_array *tr = hist_data->event_file->tr;
5754         struct action_data *data;
5755         unsigned int i;
5756         int ret = 0;
5757         char *str;
5758         int len;
5759
5760         for (i = 0; i < hist_data->attrs->n_actions; i++) {
5761                 str = hist_data->attrs->action_str[i];
5762
5763                 if ((len = str_has_prefix(str, "onmatch("))) {
5764                         char *action_str = str + len;
5765
5766                         data = onmatch_parse(tr, action_str);
5767                         if (IS_ERR(data)) {
5768                                 ret = PTR_ERR(data);
5769                                 break;
5770                         }
5771                 } else if ((len = str_has_prefix(str, "onmax("))) {
5772                         char *action_str = str + len;
5773
5774                         data = track_data_parse(hist_data, action_str,
5775                                                 HANDLER_ONMAX);
5776                         if (IS_ERR(data)) {
5777                                 ret = PTR_ERR(data);
5778                                 break;
5779                         }
5780                 } else if ((len = str_has_prefix(str, "onchange("))) {
5781                         char *action_str = str + len;
5782
5783                         data = track_data_parse(hist_data, action_str,
5784                                                 HANDLER_ONCHANGE);
5785                         if (IS_ERR(data)) {
5786                                 ret = PTR_ERR(data);
5787                                 break;
5788                         }
5789                 } else {
5790                         ret = -EINVAL;
5791                         break;
5792                 }
5793
5794                 hist_data->actions[hist_data->n_actions++] = data;
5795         }
5796
5797         return ret;
5798 }
5799
5800 static int create_actions(struct hist_trigger_data *hist_data)
5801 {
5802         struct action_data *data;
5803         unsigned int i;
5804         int ret = 0;
5805
5806         for (i = 0; i < hist_data->attrs->n_actions; i++) {
5807                 data = hist_data->actions[i];
5808
5809                 if (data->handler == HANDLER_ONMATCH) {
5810                         ret = onmatch_create(hist_data, data);
5811                         if (ret)
5812                                 break;
5813                 } else if (data->handler == HANDLER_ONMAX ||
5814                            data->handler == HANDLER_ONCHANGE) {
5815                         ret = track_data_create(hist_data, data);
5816                         if (ret)
5817                                 break;
5818                 } else {
5819                         ret = -EINVAL;
5820                         break;
5821                 }
5822         }
5823
5824         return ret;
5825 }
5826
5827 static void print_actions(struct seq_file *m,
5828                           struct hist_trigger_data *hist_data,
5829                           struct tracing_map_elt *elt)
5830 {
5831         unsigned int i;
5832
5833         for (i = 0; i < hist_data->n_actions; i++) {
5834                 struct action_data *data = hist_data->actions[i];
5835
5836                 if (data->action == ACTION_SNAPSHOT)
5837                         continue;
5838
5839                 if (data->handler == HANDLER_ONMAX ||
5840                     data->handler == HANDLER_ONCHANGE)
5841                         track_data_print(m, hist_data, elt, data);
5842         }
5843 }
5844
5845 static void print_action_spec(struct seq_file *m,
5846                               struct hist_trigger_data *hist_data,
5847                               struct action_data *data)
5848 {
5849         unsigned int i;
5850
5851         if (data->action == ACTION_SAVE) {
5852                 for (i = 0; i < hist_data->n_save_vars; i++) {
5853                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
5854                         if (i < hist_data->n_save_vars - 1)
5855                                 seq_puts(m, ",");
5856                 }
5857         } else if (data->action == ACTION_TRACE) {
5858                 if (data->use_trace_keyword)
5859                         seq_printf(m, "%s", data->synth_event_name);
5860                 for (i = 0; i < data->n_params; i++) {
5861                         if (i || data->use_trace_keyword)
5862                                 seq_puts(m, ",");
5863                         seq_printf(m, "%s", data->params[i]);
5864                 }
5865         }
5866 }
5867
5868 static void print_track_data_spec(struct seq_file *m,
5869                                   struct hist_trigger_data *hist_data,
5870                                   struct action_data *data)
5871 {
5872         if (data->handler == HANDLER_ONMAX)
5873                 seq_puts(m, ":onmax(");
5874         else if (data->handler == HANDLER_ONCHANGE)
5875                 seq_puts(m, ":onchange(");
5876         seq_printf(m, "%s", data->track_data.var_str);
5877         seq_printf(m, ").%s(", data->action_name);
5878
5879         print_action_spec(m, hist_data, data);
5880
5881         seq_puts(m, ")");
5882 }
5883
5884 static void print_onmatch_spec(struct seq_file *m,
5885                                struct hist_trigger_data *hist_data,
5886                                struct action_data *data)
5887 {
5888         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
5889                    data->match_data.event);
5890
5891         seq_printf(m, "%s(", data->action_name);
5892
5893         print_action_spec(m, hist_data, data);
5894
5895         seq_puts(m, ")");
5896 }
5897
5898 static bool actions_match(struct hist_trigger_data *hist_data,
5899                           struct hist_trigger_data *hist_data_test)
5900 {
5901         unsigned int i, j;
5902
5903         if (hist_data->n_actions != hist_data_test->n_actions)
5904                 return false;
5905
5906         for (i = 0; i < hist_data->n_actions; i++) {
5907                 struct action_data *data = hist_data->actions[i];
5908                 struct action_data *data_test = hist_data_test->actions[i];
5909                 char *action_name, *action_name_test;
5910
5911                 if (data->handler != data_test->handler)
5912                         return false;
5913                 if (data->action != data_test->action)
5914                         return false;
5915
5916                 if (data->n_params != data_test->n_params)
5917                         return false;
5918
5919                 for (j = 0; j < data->n_params; j++) {
5920                         if (strcmp(data->params[j], data_test->params[j]) != 0)
5921                                 return false;
5922                 }
5923
5924                 if (data->use_trace_keyword)
5925                         action_name = data->synth_event_name;
5926                 else
5927                         action_name = data->action_name;
5928
5929                 if (data_test->use_trace_keyword)
5930                         action_name_test = data_test->synth_event_name;
5931                 else
5932                         action_name_test = data_test->action_name;
5933
5934                 if (strcmp(action_name, action_name_test) != 0)
5935                         return false;
5936
5937                 if (data->handler == HANDLER_ONMATCH) {
5938                         if (strcmp(data->match_data.event_system,
5939                                    data_test->match_data.event_system) != 0)
5940                                 return false;
5941                         if (strcmp(data->match_data.event,
5942                                    data_test->match_data.event) != 0)
5943                                 return false;
5944                 } else if (data->handler == HANDLER_ONMAX ||
5945                            data->handler == HANDLER_ONCHANGE) {
5946                         if (strcmp(data->track_data.var_str,
5947                                    data_test->track_data.var_str) != 0)
5948                                 return false;
5949                 }
5950         }
5951
5952         return true;
5953 }
5954
5955
5956 static void print_actions_spec(struct seq_file *m,
5957                                struct hist_trigger_data *hist_data)
5958 {
5959         unsigned int i;
5960
5961         for (i = 0; i < hist_data->n_actions; i++) {
5962                 struct action_data *data = hist_data->actions[i];
5963
5964                 if (data->handler == HANDLER_ONMATCH)
5965                         print_onmatch_spec(m, hist_data, data);
5966                 else if (data->handler == HANDLER_ONMAX ||
5967                          data->handler == HANDLER_ONCHANGE)
5968                         print_track_data_spec(m, hist_data, data);
5969         }
5970 }
5971
5972 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5973 {
5974         unsigned int i;
5975
5976         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5977                 kfree(hist_data->field_var_hists[i]->cmd);
5978                 kfree(hist_data->field_var_hists[i]);
5979         }
5980 }
5981
5982 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5983 {
5984         if (!hist_data)
5985                 return;
5986
5987         destroy_hist_trigger_attrs(hist_data->attrs);
5988         destroy_hist_fields(hist_data);
5989         tracing_map_destroy(hist_data->map);
5990
5991         destroy_actions(hist_data);
5992         destroy_field_vars(hist_data);
5993         destroy_field_var_hists(hist_data);
5994
5995         kfree(hist_data);
5996 }
5997
5998 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5999 {
6000         struct tracing_map *map = hist_data->map;
6001         struct ftrace_event_field *field;
6002         struct hist_field *hist_field;
6003         int i, idx = 0;
6004
6005         for_each_hist_field(i, hist_data) {
6006                 hist_field = hist_data->fields[i];
6007                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
6008                         tracing_map_cmp_fn_t cmp_fn;
6009
6010                         field = hist_field->field;
6011
6012                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
6013                                 cmp_fn = tracing_map_cmp_none;
6014                         else if (!field)
6015                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
6016                                                              hist_field->is_signed);
6017                         else if (is_string_field(field))
6018                                 cmp_fn = tracing_map_cmp_string;
6019                         else
6020                                 cmp_fn = tracing_map_cmp_num(field->size,
6021                                                              field->is_signed);
6022                         idx = tracing_map_add_key_field(map,
6023                                                         hist_field->offset,
6024                                                         cmp_fn);
6025                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
6026                         idx = tracing_map_add_sum_field(map);
6027
6028                 if (idx < 0)
6029                         return idx;
6030
6031                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6032                         idx = tracing_map_add_var(map);
6033                         if (idx < 0)
6034                                 return idx;
6035                         hist_field->var.idx = idx;
6036                         hist_field->var.hist_data = hist_data;
6037                 }
6038         }
6039
6040         return 0;
6041 }
6042
6043 static struct hist_trigger_data *
6044 create_hist_data(unsigned int map_bits,
6045                  struct hist_trigger_attrs *attrs,
6046                  struct trace_event_file *file,
6047                  bool remove)
6048 {
6049         const struct tracing_map_ops *map_ops = NULL;
6050         struct hist_trigger_data *hist_data;
6051         int ret = 0;
6052
6053         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
6054         if (!hist_data)
6055                 return ERR_PTR(-ENOMEM);
6056
6057         hist_data->attrs = attrs;
6058         hist_data->remove = remove;
6059         hist_data->event_file = file;
6060
6061         ret = parse_actions(hist_data);
6062         if (ret)
6063                 goto free;
6064
6065         ret = create_hist_fields(hist_data, file);
6066         if (ret)
6067                 goto free;
6068
6069         ret = create_sort_keys(hist_data);
6070         if (ret)
6071                 goto free;
6072
6073         map_ops = &hist_trigger_elt_data_ops;
6074
6075         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6076                                             map_ops, hist_data);
6077         if (IS_ERR(hist_data->map)) {
6078                 ret = PTR_ERR(hist_data->map);
6079                 hist_data->map = NULL;
6080                 goto free;
6081         }
6082
6083         ret = create_tracing_map_fields(hist_data);
6084         if (ret)
6085                 goto free;
6086  out:
6087         return hist_data;
6088  free:
6089         hist_data->attrs = NULL;
6090
6091         destroy_hist_data(hist_data);
6092
6093         hist_data = ERR_PTR(ret);
6094
6095         goto out;
6096 }
6097
6098 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
6099                                     struct tracing_map_elt *elt, void *rec,
6100                                     struct ring_buffer_event *rbe,
6101                                     u64 *var_ref_vals)
6102 {
6103         struct hist_elt_data *elt_data;
6104         struct hist_field *hist_field;
6105         unsigned int i, var_idx;
6106         u64 hist_val;
6107
6108         elt_data = elt->private_data;
6109         elt_data->var_ref_vals = var_ref_vals;
6110
6111         for_each_hist_val_field(i, hist_data) {
6112                 hist_field = hist_data->fields[i];
6113                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
6114                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6115                         var_idx = hist_field->var.idx;
6116                         tracing_map_set_var(elt, var_idx, hist_val);
6117                         continue;
6118                 }
6119                 tracing_map_update_sum(elt, i, hist_val);
6120         }
6121
6122         for_each_hist_key_field(i, hist_data) {
6123                 hist_field = hist_data->fields[i];
6124                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6125                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
6126                         var_idx = hist_field->var.idx;
6127                         tracing_map_set_var(elt, var_idx, hist_val);
6128                 }
6129         }
6130
6131         update_field_vars(hist_data, elt, rbe, rec);
6132 }
6133
6134 static inline void add_to_key(char *compound_key, void *key,
6135                               struct hist_field *key_field, void *rec)
6136 {
6137         size_t size = key_field->size;
6138
6139         if (key_field->flags & HIST_FIELD_FL_STRING) {
6140                 struct ftrace_event_field *field;
6141
6142                 field = key_field->field;
6143                 if (field->filter_type == FILTER_DYN_STRING)
6144                         size = *(u32 *)(rec + field->offset) >> 16;
6145                 else if (field->filter_type == FILTER_PTR_STRING)
6146                         size = strlen(key);
6147                 else if (field->filter_type == FILTER_STATIC_STRING)
6148                         size = field->size;
6149
6150                 /* ensure NULL-termination */
6151                 if (size > key_field->size - 1)
6152                         size = key_field->size - 1;
6153
6154                 strncpy(compound_key + key_field->offset, (char *)key, size);
6155         } else
6156                 memcpy(compound_key + key_field->offset, key, size);
6157 }
6158
6159 static void
6160 hist_trigger_actions(struct hist_trigger_data *hist_data,
6161                      struct tracing_map_elt *elt, void *rec,
6162                      struct ring_buffer_event *rbe, void *key,
6163                      u64 *var_ref_vals)
6164 {
6165         struct action_data *data;
6166         unsigned int i;
6167
6168         for (i = 0; i < hist_data->n_actions; i++) {
6169                 data = hist_data->actions[i];
6170                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
6171         }
6172 }
6173
6174 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
6175                                struct ring_buffer_event *rbe)
6176 {
6177         struct hist_trigger_data *hist_data = data->private_data;
6178         bool use_compound_key = (hist_data->n_keys > 1);
6179         unsigned long entries[HIST_STACKTRACE_DEPTH];
6180         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
6181         char compound_key[HIST_KEY_SIZE_MAX];
6182         struct tracing_map_elt *elt = NULL;
6183         struct hist_field *key_field;
6184         u64 field_contents;
6185         void *key = NULL;
6186         unsigned int i;
6187
6188         memset(compound_key, 0, hist_data->key_size);
6189
6190         for_each_hist_key_field(i, hist_data) {
6191                 key_field = hist_data->fields[i];
6192
6193                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
6194                         memset(entries, 0, HIST_STACKTRACE_SIZE);
6195                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
6196                                          HIST_STACKTRACE_SKIP);
6197                         key = entries;
6198                 } else {
6199                         field_contents = key_field->fn(key_field, elt, rbe, rec);
6200                         if (key_field->flags & HIST_FIELD_FL_STRING) {
6201                                 key = (void *)(unsigned long)field_contents;
6202                                 use_compound_key = true;
6203                         } else
6204                                 key = (void *)&field_contents;
6205                 }
6206
6207                 if (use_compound_key)
6208                         add_to_key(compound_key, key, key_field, rec);
6209         }
6210
6211         if (use_compound_key)
6212                 key = compound_key;
6213
6214         if (hist_data->n_var_refs &&
6215             !resolve_var_refs(hist_data, key, var_ref_vals, false))
6216                 return;
6217
6218         elt = tracing_map_insert(hist_data->map, key);
6219         if (!elt)
6220                 return;
6221
6222         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
6223
6224         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
6225                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
6226 }
6227
6228 static void hist_trigger_stacktrace_print(struct seq_file *m,
6229                                           unsigned long *stacktrace_entries,
6230                                           unsigned int max_entries)
6231 {
6232         char str[KSYM_SYMBOL_LEN];
6233         unsigned int spaces = 8;
6234         unsigned int i;
6235
6236         for (i = 0; i < max_entries; i++) {
6237                 if (!stacktrace_entries[i])
6238                         return;
6239
6240                 seq_printf(m, "%*c", 1 + spaces, ' ');
6241                 sprint_symbol(str, stacktrace_entries[i]);
6242                 seq_printf(m, "%s\n", str);
6243         }
6244 }
6245
6246 static void hist_trigger_print_key(struct seq_file *m,
6247                                    struct hist_trigger_data *hist_data,
6248                                    void *key,
6249                                    struct tracing_map_elt *elt)
6250 {
6251         struct hist_field *key_field;
6252         char str[KSYM_SYMBOL_LEN];
6253         bool multiline = false;
6254         const char *field_name;
6255         unsigned int i;
6256         u64 uval;
6257
6258         seq_puts(m, "{ ");
6259
6260         for_each_hist_key_field(i, hist_data) {
6261                 key_field = hist_data->fields[i];
6262
6263                 if (i > hist_data->n_vals)
6264                         seq_puts(m, ", ");
6265
6266                 field_name = hist_field_name(key_field, 0);
6267
6268                 if (key_field->flags & HIST_FIELD_FL_HEX) {
6269                         uval = *(u64 *)(key + key_field->offset);
6270                         seq_printf(m, "%s: %llx", field_name, uval);
6271                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
6272                         uval = *(u64 *)(key + key_field->offset);
6273                         sprint_symbol_no_offset(str, uval);
6274                         seq_printf(m, "%s: [%llx] %-45s", field_name,
6275                                    uval, str);
6276                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
6277                         uval = *(u64 *)(key + key_field->offset);
6278                         sprint_symbol(str, uval);
6279                         seq_printf(m, "%s: [%llx] %-55s", field_name,
6280                                    uval, str);
6281                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
6282                         struct hist_elt_data *elt_data = elt->private_data;
6283                         char *comm;
6284
6285                         if (WARN_ON_ONCE(!elt_data))
6286                                 return;
6287
6288                         comm = elt_data->comm;
6289
6290                         uval = *(u64 *)(key + key_field->offset);
6291                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
6292                                    comm, uval);
6293                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
6294                         const char *syscall_name;
6295
6296                         uval = *(u64 *)(key + key_field->offset);
6297                         syscall_name = get_syscall_name(uval);
6298                         if (!syscall_name)
6299                                 syscall_name = "unknown_syscall";
6300
6301                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
6302                                    syscall_name, uval);
6303                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
6304                         seq_puts(m, "stacktrace:\n");
6305                         hist_trigger_stacktrace_print(m,
6306                                                       key + key_field->offset,
6307                                                       HIST_STACKTRACE_DEPTH);
6308                         multiline = true;
6309                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
6310                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
6311                                    *(u64 *)(key + key_field->offset));
6312                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
6313                         seq_printf(m, "%s: %-50s", field_name,
6314                                    (char *)(key + key_field->offset));
6315                 } else {
6316                         uval = *(u64 *)(key + key_field->offset);
6317                         seq_printf(m, "%s: %10llu", field_name, uval);
6318                 }
6319         }
6320
6321         if (!multiline)
6322                 seq_puts(m, " ");
6323
6324         seq_puts(m, "}");
6325 }
6326
6327 static void hist_trigger_entry_print(struct seq_file *m,
6328                                      struct hist_trigger_data *hist_data,
6329                                      void *key,
6330                                      struct tracing_map_elt *elt)
6331 {
6332         const char *field_name;
6333         unsigned int i;
6334
6335         hist_trigger_print_key(m, hist_data, key, elt);
6336
6337         seq_printf(m, " hitcount: %10llu",
6338                    tracing_map_read_sum(elt, HITCOUNT_IDX));
6339
6340         for (i = 1; i < hist_data->n_vals; i++) {
6341                 field_name = hist_field_name(hist_data->fields[i], 0);
6342
6343                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
6344                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
6345                         continue;
6346
6347                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
6348                         seq_printf(m, "  %s: %10llx", field_name,
6349                                    tracing_map_read_sum(elt, i));
6350                 } else {
6351                         seq_printf(m, "  %s: %10llu", field_name,
6352                                    tracing_map_read_sum(elt, i));
6353                 }
6354         }
6355
6356         print_actions(m, hist_data, elt);
6357
6358         seq_puts(m, "\n");
6359 }
6360
6361 static int print_entries(struct seq_file *m,
6362                          struct hist_trigger_data *hist_data)
6363 {
6364         struct tracing_map_sort_entry **sort_entries = NULL;
6365         struct tracing_map *map = hist_data->map;
6366         int i, n_entries;
6367
6368         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
6369                                              hist_data->n_sort_keys,
6370                                              &sort_entries);
6371         if (n_entries < 0)
6372                 return n_entries;
6373
6374         for (i = 0; i < n_entries; i++)
6375                 hist_trigger_entry_print(m, hist_data,
6376                                          sort_entries[i]->key,
6377                                          sort_entries[i]->elt);
6378
6379         tracing_map_destroy_sort_entries(sort_entries, n_entries);
6380
6381         return n_entries;
6382 }
6383
6384 static void hist_trigger_show(struct seq_file *m,
6385                               struct event_trigger_data *data, int n)
6386 {
6387         struct hist_trigger_data *hist_data;
6388         int n_entries;
6389
6390         if (n > 0)
6391                 seq_puts(m, "\n\n");
6392
6393         seq_puts(m, "# event histogram\n#\n# trigger info: ");
6394         data->ops->print(m, data->ops, data);
6395         seq_puts(m, "#\n\n");
6396
6397         hist_data = data->private_data;
6398         n_entries = print_entries(m, hist_data);
6399         if (n_entries < 0)
6400                 n_entries = 0;
6401
6402         track_data_snapshot_print(m, hist_data);
6403
6404         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
6405                    (u64)atomic64_read(&hist_data->map->hits),
6406                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
6407 }
6408
6409 static int hist_show(struct seq_file *m, void *v)
6410 {
6411         struct event_trigger_data *data;
6412         struct trace_event_file *event_file;
6413         int n = 0, ret = 0;
6414
6415         mutex_lock(&event_mutex);
6416
6417         event_file = event_file_data(m->private);
6418         if (unlikely(!event_file)) {
6419                 ret = -ENODEV;
6420                 goto out_unlock;
6421         }
6422
6423         list_for_each_entry(data, &event_file->triggers, list) {
6424                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
6425                         hist_trigger_show(m, data, n++);
6426         }
6427
6428  out_unlock:
6429         mutex_unlock(&event_mutex);
6430
6431         return ret;
6432 }
6433
6434 static int event_hist_open(struct inode *inode, struct file *file)
6435 {
6436         int ret;
6437
6438         ret = security_locked_down(LOCKDOWN_TRACEFS);
6439         if (ret)
6440                 return ret;
6441
6442         return single_open(file, hist_show, file);
6443 }
6444
6445 const struct file_operations event_hist_fops = {
6446         .open = event_hist_open,
6447         .read = seq_read,
6448         .llseek = seq_lseek,
6449         .release = single_release,
6450 };
6451
6452 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
6453 {
6454         const char *field_name = hist_field_name(hist_field, 0);
6455
6456         if (hist_field->var.name)
6457                 seq_printf(m, "%s=", hist_field->var.name);
6458
6459         if (hist_field->flags & HIST_FIELD_FL_CPU)
6460                 seq_puts(m, "cpu");
6461         else if (field_name) {
6462                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
6463                     hist_field->flags & HIST_FIELD_FL_ALIAS)
6464                         seq_putc(m, '$');
6465                 seq_printf(m, "%s", field_name);
6466         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
6467                 seq_puts(m, "common_timestamp");
6468
6469         if (hist_field->flags) {
6470                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
6471                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
6472                         const char *flags = get_hist_field_flags(hist_field);
6473
6474                         if (flags)
6475                                 seq_printf(m, ".%s", flags);
6476                 }
6477         }
6478 }
6479
6480 static int event_hist_trigger_print(struct seq_file *m,
6481                                     struct event_trigger_ops *ops,
6482                                     struct event_trigger_data *data)
6483 {
6484         struct hist_trigger_data *hist_data = data->private_data;
6485         struct hist_field *field;
6486         bool have_var = false;
6487         unsigned int i;
6488
6489         seq_puts(m, "hist:");
6490
6491         if (data->name)
6492                 seq_printf(m, "%s:", data->name);
6493
6494         seq_puts(m, "keys=");
6495
6496         for_each_hist_key_field(i, hist_data) {
6497                 field = hist_data->fields[i];
6498
6499                 if (i > hist_data->n_vals)
6500                         seq_puts(m, ",");
6501
6502                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
6503                         seq_puts(m, "stacktrace");
6504                 else
6505                         hist_field_print(m, field);
6506         }
6507
6508         seq_puts(m, ":vals=");
6509
6510         for_each_hist_val_field(i, hist_data) {
6511                 field = hist_data->fields[i];
6512                 if (field->flags & HIST_FIELD_FL_VAR) {
6513                         have_var = true;
6514                         continue;
6515                 }
6516
6517                 if (i == HITCOUNT_IDX)
6518                         seq_puts(m, "hitcount");
6519                 else {
6520                         seq_puts(m, ",");
6521                         hist_field_print(m, field);
6522                 }
6523         }
6524
6525         if (have_var) {
6526                 unsigned int n = 0;
6527
6528                 seq_puts(m, ":");
6529
6530                 for_each_hist_val_field(i, hist_data) {
6531                         field = hist_data->fields[i];
6532
6533                         if (field->flags & HIST_FIELD_FL_VAR) {
6534                                 if (n++)
6535                                         seq_puts(m, ",");
6536                                 hist_field_print(m, field);
6537                         }
6538                 }
6539         }
6540
6541         seq_puts(m, ":sort=");
6542
6543         for (i = 0; i < hist_data->n_sort_keys; i++) {
6544                 struct tracing_map_sort_key *sort_key;
6545                 unsigned int idx, first_key_idx;
6546
6547                 /* skip VAR vals */
6548                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
6549
6550                 sort_key = &hist_data->sort_keys[i];
6551                 idx = sort_key->field_idx;
6552
6553                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
6554                         return -EINVAL;
6555
6556                 if (i > 0)
6557                         seq_puts(m, ",");
6558
6559                 if (idx == HITCOUNT_IDX)
6560                         seq_puts(m, "hitcount");
6561                 else {
6562                         if (idx >= first_key_idx)
6563                                 idx += hist_data->n_vars;
6564                         hist_field_print(m, hist_data->fields[idx]);
6565                 }
6566
6567                 if (sort_key->descending)
6568                         seq_puts(m, ".descending");
6569         }
6570         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6571         if (hist_data->enable_timestamps)
6572                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6573
6574         print_actions_spec(m, hist_data);
6575
6576         if (data->filter_str)
6577                 seq_printf(m, " if %s", data->filter_str);
6578
6579         if (data->paused)
6580                 seq_puts(m, " [paused]");
6581         else
6582                 seq_puts(m, " [active]");
6583
6584         seq_putc(m, '\n');
6585
6586         return 0;
6587 }
6588
6589 static int event_hist_trigger_init(struct event_trigger_ops *ops,
6590                                    struct event_trigger_data *data)
6591 {
6592         struct hist_trigger_data *hist_data = data->private_data;
6593
6594         if (!data->ref && hist_data->attrs->name)
6595                 save_named_trigger(hist_data->attrs->name, data);
6596
6597         data->ref++;
6598
6599         return 0;
6600 }
6601
6602 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6603 {
6604         struct trace_event_file *file;
6605         unsigned int i;
6606         char *cmd;
6607         int ret;
6608
6609         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6610                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6611                 cmd = hist_data->field_var_hists[i]->cmd;
6612                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
6613                                               "!hist", "hist", cmd);
6614         }
6615 }
6616
6617 static void event_hist_trigger_free(struct event_trigger_ops *ops,
6618                                     struct event_trigger_data *data)
6619 {
6620         struct hist_trigger_data *hist_data = data->private_data;
6621
6622         if (WARN_ON_ONCE(data->ref <= 0))
6623                 return;
6624
6625         data->ref--;
6626         if (!data->ref) {
6627                 if (data->name)
6628                         del_named_trigger(data);
6629
6630                 trigger_data_free(data);
6631
6632                 remove_hist_vars(hist_data);
6633
6634                 unregister_field_var_hists(hist_data);
6635
6636                 destroy_hist_data(hist_data);
6637         }
6638 }
6639
6640 static struct event_trigger_ops event_hist_trigger_ops = {
6641         .func                   = event_hist_trigger,
6642         .print                  = event_hist_trigger_print,
6643         .init                   = event_hist_trigger_init,
6644         .free                   = event_hist_trigger_free,
6645 };
6646
6647 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
6648                                          struct event_trigger_data *data)
6649 {
6650         data->ref++;
6651
6652         save_named_trigger(data->named_data->name, data);
6653
6654         event_hist_trigger_init(ops, data->named_data);
6655
6656         return 0;
6657 }
6658
6659 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
6660                                           struct event_trigger_data *data)
6661 {
6662         if (WARN_ON_ONCE(data->ref <= 0))
6663                 return;
6664
6665         event_hist_trigger_free(ops, data->named_data);
6666
6667         data->ref--;
6668         if (!data->ref) {
6669                 del_named_trigger(data);
6670                 trigger_data_free(data);
6671         }
6672 }
6673
6674 static struct event_trigger_ops event_hist_trigger_named_ops = {
6675         .func                   = event_hist_trigger,
6676         .print                  = event_hist_trigger_print,
6677         .init                   = event_hist_trigger_named_init,
6678         .free                   = event_hist_trigger_named_free,
6679 };
6680
6681 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6682                                                             char *param)
6683 {
6684         return &event_hist_trigger_ops;
6685 }
6686
6687 static void hist_clear(struct event_trigger_data *data)
6688 {
6689         struct hist_trigger_data *hist_data = data->private_data;
6690
6691         if (data->name)
6692                 pause_named_trigger(data);
6693
6694         tracepoint_synchronize_unregister();
6695
6696         tracing_map_clear(hist_data->map);
6697
6698         if (data->name)
6699                 unpause_named_trigger(data);
6700 }
6701
6702 static bool compatible_field(struct ftrace_event_field *field,
6703                              struct ftrace_event_field *test_field)
6704 {
6705         if (field == test_field)
6706                 return true;
6707         if (field == NULL || test_field == NULL)
6708                 return false;
6709         if (strcmp(field->name, test_field->name) != 0)
6710                 return false;
6711         if (strcmp(field->type, test_field->type) != 0)
6712                 return false;
6713         if (field->size != test_field->size)
6714                 return false;
6715         if (field->is_signed != test_field->is_signed)
6716                 return false;
6717
6718         return true;
6719 }
6720
6721 static bool hist_trigger_match(struct event_trigger_data *data,
6722                                struct event_trigger_data *data_test,
6723                                struct event_trigger_data *named_data,
6724                                bool ignore_filter)
6725 {
6726         struct tracing_map_sort_key *sort_key, *sort_key_test;
6727         struct hist_trigger_data *hist_data, *hist_data_test;
6728         struct hist_field *key_field, *key_field_test;
6729         unsigned int i;
6730
6731         if (named_data && (named_data != data_test) &&
6732             (named_data != data_test->named_data))
6733                 return false;
6734
6735         if (!named_data && is_named_trigger(data_test))
6736                 return false;
6737
6738         hist_data = data->private_data;
6739         hist_data_test = data_test->private_data;
6740
6741         if (hist_data->n_vals != hist_data_test->n_vals ||
6742             hist_data->n_fields != hist_data_test->n_fields ||
6743             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6744                 return false;
6745
6746         if (!ignore_filter) {
6747                 if ((data->filter_str && !data_test->filter_str) ||
6748                    (!data->filter_str && data_test->filter_str))
6749                         return false;
6750         }
6751
6752         for_each_hist_field(i, hist_data) {
6753                 key_field = hist_data->fields[i];
6754                 key_field_test = hist_data_test->fields[i];
6755
6756                 if (key_field->flags != key_field_test->flags)
6757                         return false;
6758                 if (!compatible_field(key_field->field, key_field_test->field))
6759                         return false;
6760                 if (key_field->offset != key_field_test->offset)
6761                         return false;
6762                 if (key_field->size != key_field_test->size)
6763                         return false;
6764                 if (key_field->is_signed != key_field_test->is_signed)
6765                         return false;
6766                 if (!!key_field->var.name != !!key_field_test->var.name)
6767                         return false;
6768                 if (key_field->var.name &&
6769                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6770                         return false;
6771         }
6772
6773         for (i = 0; i < hist_data->n_sort_keys; i++) {
6774                 sort_key = &hist_data->sort_keys[i];
6775                 sort_key_test = &hist_data_test->sort_keys[i];
6776
6777                 if (sort_key->field_idx != sort_key_test->field_idx ||
6778                     sort_key->descending != sort_key_test->descending)
6779                         return false;
6780         }
6781
6782         if (!ignore_filter && data->filter_str &&
6783             (strcmp(data->filter_str, data_test->filter_str) != 0))
6784                 return false;
6785
6786         if (!actions_match(hist_data, hist_data_test))
6787                 return false;
6788
6789         return true;
6790 }
6791
6792 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
6793                                  struct event_trigger_data *data,
6794                                  struct trace_event_file *file)
6795 {
6796         struct hist_trigger_data *hist_data = data->private_data;
6797         struct event_trigger_data *test, *named_data = NULL;
6798         struct trace_array *tr = file->tr;
6799         int ret = 0;
6800
6801         if (hist_data->attrs->name) {
6802                 named_data = find_named_trigger(hist_data->attrs->name);
6803                 if (named_data) {
6804                         if (!hist_trigger_match(data, named_data, named_data,
6805                                                 true)) {
6806                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6807                                 ret = -EINVAL;
6808                                 goto out;
6809                         }
6810                 }
6811         }
6812
6813         if (hist_data->attrs->name && !named_data)
6814                 goto new;
6815
6816         lockdep_assert_held(&event_mutex);
6817
6818         list_for_each_entry(test, &file->triggers, list) {
6819                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6820                         if (!hist_trigger_match(data, test, named_data, false))
6821                                 continue;
6822                         if (hist_data->attrs->pause)
6823                                 test->paused = true;
6824                         else if (hist_data->attrs->cont)
6825                                 test->paused = false;
6826                         else if (hist_data->attrs->clear)
6827                                 hist_clear(test);
6828                         else {
6829                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6830                                 ret = -EEXIST;
6831                         }
6832                         goto out;
6833                 }
6834         }
6835  new:
6836         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6837                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6838                 ret = -ENOENT;
6839                 goto out;
6840         }
6841
6842         if (hist_data->attrs->pause)
6843                 data->paused = true;
6844
6845         if (named_data) {
6846                 data->private_data = named_data->private_data;
6847                 set_named_trigger_data(data, named_data);
6848                 data->ops = &event_hist_trigger_named_ops;
6849         }
6850
6851         if (data->ops->init) {
6852                 ret = data->ops->init(data->ops, data);
6853                 if (ret < 0)
6854                         goto out;
6855         }
6856
6857         if (hist_data->enable_timestamps) {
6858                 char *clock = hist_data->attrs->clock;
6859
6860                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6861                 if (ret) {
6862                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6863                         goto out;
6864                 }
6865
6866                 tracing_set_time_stamp_abs(file->tr, true);
6867         }
6868
6869         if (named_data)
6870                 destroy_hist_data(hist_data);
6871
6872         ret++;
6873  out:
6874         return ret;
6875 }
6876
6877 static int hist_trigger_enable(struct event_trigger_data *data,
6878                                struct trace_event_file *file)
6879 {
6880         int ret = 0;
6881
6882         list_add_tail_rcu(&data->list, &file->triggers);
6883
6884         update_cond_flag(file);
6885
6886         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6887                 list_del_rcu(&data->list);
6888                 update_cond_flag(file);
6889                 ret--;
6890         }
6891
6892         return ret;
6893 }
6894
6895 static bool have_hist_trigger_match(struct event_trigger_data *data,
6896                                     struct trace_event_file *file)
6897 {
6898         struct hist_trigger_data *hist_data = data->private_data;
6899         struct event_trigger_data *test, *named_data = NULL;
6900         bool match = false;
6901
6902         lockdep_assert_held(&event_mutex);
6903
6904         if (hist_data->attrs->name)
6905                 named_data = find_named_trigger(hist_data->attrs->name);
6906
6907         list_for_each_entry(test, &file->triggers, list) {
6908                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6909                         if (hist_trigger_match(data, test, named_data, false)) {
6910                                 match = true;
6911                                 break;
6912                         }
6913                 }
6914         }
6915
6916         return match;
6917 }
6918
6919 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6920                                     struct trace_event_file *file)
6921 {
6922         struct hist_trigger_data *hist_data = data->private_data;
6923         struct event_trigger_data *test, *named_data = NULL;
6924
6925         lockdep_assert_held(&event_mutex);
6926
6927         if (hist_data->attrs->name)
6928                 named_data = find_named_trigger(hist_data->attrs->name);
6929
6930         list_for_each_entry(test, &file->triggers, list) {
6931                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6932                         if (!hist_trigger_match(data, test, named_data, false))
6933                                 continue;
6934                         hist_data = test->private_data;
6935                         if (check_var_refs(hist_data))
6936                                 return true;
6937                         break;
6938                 }
6939         }
6940
6941         return false;
6942 }
6943
6944 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6945                                     struct event_trigger_data *data,
6946                                     struct trace_event_file *file)
6947 {
6948         struct hist_trigger_data *hist_data = data->private_data;
6949         struct event_trigger_data *test, *named_data = NULL;
6950         bool unregistered = false;
6951
6952         lockdep_assert_held(&event_mutex);
6953
6954         if (hist_data->attrs->name)
6955                 named_data = find_named_trigger(hist_data->attrs->name);
6956
6957         list_for_each_entry(test, &file->triggers, list) {
6958                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6959                         if (!hist_trigger_match(data, test, named_data, false))
6960                                 continue;
6961                         unregistered = true;
6962                         list_del_rcu(&test->list);
6963                         trace_event_trigger_enable_disable(file, 0);
6964                         update_cond_flag(file);
6965                         break;
6966                 }
6967         }
6968
6969         if (unregistered && test->ops->free)
6970                 test->ops->free(test->ops, test);
6971
6972         if (hist_data->enable_timestamps) {
6973                 if (!hist_data->remove || unregistered)
6974                         tracing_set_time_stamp_abs(file->tr, false);
6975         }
6976 }
6977
6978 static bool hist_file_check_refs(struct trace_event_file *file)
6979 {
6980         struct hist_trigger_data *hist_data;
6981         struct event_trigger_data *test;
6982
6983         lockdep_assert_held(&event_mutex);
6984
6985         list_for_each_entry(test, &file->triggers, list) {
6986                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6987                         hist_data = test->private_data;
6988                         if (check_var_refs(hist_data))
6989                                 return true;
6990                 }
6991         }
6992
6993         return false;
6994 }
6995
6996 static void hist_unreg_all(struct trace_event_file *file)
6997 {
6998         struct event_trigger_data *test, *n;
6999         struct hist_trigger_data *hist_data;
7000         struct synth_event *se;
7001         const char *se_name;
7002
7003         lockdep_assert_held(&event_mutex);
7004
7005         if (hist_file_check_refs(file))
7006                 return;
7007
7008         list_for_each_entry_safe(test, n, &file->triggers, list) {
7009                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7010                         hist_data = test->private_data;
7011                         list_del_rcu(&test->list);
7012                         trace_event_trigger_enable_disable(file, 0);
7013
7014                         se_name = trace_event_name(file->event_call);
7015                         se = find_synth_event(se_name);
7016                         if (se)
7017                                 se->ref--;
7018
7019                         update_cond_flag(file);
7020                         if (hist_data->enable_timestamps)
7021                                 tracing_set_time_stamp_abs(file->tr, false);
7022                         if (test->ops->free)
7023                                 test->ops->free(test->ops, test);
7024                 }
7025         }
7026 }
7027
7028 static int event_hist_trigger_func(struct event_command *cmd_ops,
7029                                    struct trace_event_file *file,
7030                                    char *glob, char *cmd, char *param)
7031 {
7032         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
7033         struct event_trigger_data *trigger_data;
7034         struct hist_trigger_attrs *attrs;
7035         struct event_trigger_ops *trigger_ops;
7036         struct hist_trigger_data *hist_data;
7037         struct synth_event *se;
7038         const char *se_name;
7039         bool remove = false;
7040         char *trigger, *p;
7041         int ret = 0;
7042
7043         lockdep_assert_held(&event_mutex);
7044
7045         if (glob && strlen(glob)) {
7046                 hist_err_clear();
7047                 last_cmd_set(file, param);
7048         }
7049
7050         if (!param)
7051                 return -EINVAL;
7052
7053         if (glob[0] == '!')
7054                 remove = true;
7055
7056         /*
7057          * separate the trigger from the filter (k:v [if filter])
7058          * allowing for whitespace in the trigger
7059          */
7060         p = trigger = param;
7061         do {
7062                 p = strstr(p, "if");
7063                 if (!p)
7064                         break;
7065                 if (p == param)
7066                         return -EINVAL;
7067                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
7068                         p++;
7069                         continue;
7070                 }
7071                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
7072                         return -EINVAL;
7073                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
7074                         p++;
7075                         continue;
7076                 }
7077                 break;
7078         } while (p);
7079
7080         if (!p)
7081                 param = NULL;
7082         else {
7083                 *(p - 1) = '\0';
7084                 param = strstrip(p);
7085                 trigger = strstrip(trigger);
7086         }
7087
7088         attrs = parse_hist_trigger_attrs(file->tr, trigger);
7089         if (IS_ERR(attrs))
7090                 return PTR_ERR(attrs);
7091
7092         if (attrs->map_bits)
7093                 hist_trigger_bits = attrs->map_bits;
7094
7095         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7096         if (IS_ERR(hist_data)) {
7097                 destroy_hist_trigger_attrs(attrs);
7098                 return PTR_ERR(hist_data);
7099         }
7100
7101         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
7102
7103         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
7104         if (!trigger_data) {
7105                 ret = -ENOMEM;
7106                 goto out_free;
7107         }
7108
7109         trigger_data->count = -1;
7110         trigger_data->ops = trigger_ops;
7111         trigger_data->cmd_ops = cmd_ops;
7112
7113         INIT_LIST_HEAD(&trigger_data->list);
7114         RCU_INIT_POINTER(trigger_data->filter, NULL);
7115
7116         trigger_data->private_data = hist_data;
7117
7118         /* if param is non-empty, it's supposed to be a filter */
7119         if (param && cmd_ops->set_filter) {
7120                 ret = cmd_ops->set_filter(param, trigger_data, file);
7121                 if (ret < 0)
7122                         goto out_free;
7123         }
7124
7125         if (remove) {
7126                 if (!have_hist_trigger_match(trigger_data, file))
7127                         goto out_free;
7128
7129                 if (hist_trigger_check_refs(trigger_data, file)) {
7130                         ret = -EBUSY;
7131                         goto out_free;
7132                 }
7133
7134                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7135                 se_name = trace_event_name(file->event_call);
7136                 se = find_synth_event(se_name);
7137                 if (se)
7138                         se->ref--;
7139                 ret = 0;
7140                 goto out_free;
7141         }
7142
7143         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
7144         /*
7145          * The above returns on success the # of triggers registered,
7146          * but if it didn't register any it returns zero.  Consider no
7147          * triggers registered a failure too.
7148          */
7149         if (!ret) {
7150                 if (!(attrs->pause || attrs->cont || attrs->clear))
7151                         ret = -ENOENT;
7152                 goto out_free;
7153         } else if (ret < 0)
7154                 goto out_free;
7155
7156         if (get_named_trigger_data(trigger_data))
7157                 goto enable;
7158
7159         if (has_hist_vars(hist_data))
7160                 save_hist_vars(hist_data);
7161
7162         ret = create_actions(hist_data);
7163         if (ret)
7164                 goto out_unreg;
7165
7166         ret = tracing_map_init(hist_data->map);
7167         if (ret)
7168                 goto out_unreg;
7169 enable:
7170         ret = hist_trigger_enable(trigger_data, file);
7171         if (ret)
7172                 goto out_unreg;
7173
7174         se_name = trace_event_name(file->event_call);
7175         se = find_synth_event(se_name);
7176         if (se)
7177                 se->ref++;
7178         /* Just return zero, not the number of registered triggers */
7179         ret = 0;
7180  out:
7181         if (ret == 0)
7182                 hist_err_clear();
7183
7184         return ret;
7185  out_unreg:
7186         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7187  out_free:
7188         if (cmd_ops->set_filter)
7189                 cmd_ops->set_filter(NULL, trigger_data, NULL);
7190
7191         remove_hist_vars(hist_data);
7192
7193         kfree(trigger_data);
7194
7195         destroy_hist_data(hist_data);
7196         goto out;
7197 }
7198
7199 static struct event_command trigger_hist_cmd = {
7200         .name                   = "hist",
7201         .trigger_type           = ETT_EVENT_HIST,
7202         .flags                  = EVENT_CMD_FL_NEEDS_REC,
7203         .func                   = event_hist_trigger_func,
7204         .reg                    = hist_register_trigger,
7205         .unreg                  = hist_unregister_trigger,
7206         .unreg_all              = hist_unreg_all,
7207         .get_trigger_ops        = event_hist_get_trigger_ops,
7208         .set_filter             = set_trigger_filter,
7209 };
7210
7211 __init int register_trigger_hist_cmd(void)
7212 {
7213         int ret;
7214
7215         ret = register_event_command(&trigger_hist_cmd);
7216         WARN_ON(ret < 0);
7217
7218         return ret;
7219 }
7220
7221 static void
7222 hist_enable_trigger(struct event_trigger_data *data, void *rec,
7223                     struct ring_buffer_event *event)
7224 {
7225         struct enable_trigger_data *enable_data = data->private_data;
7226         struct event_trigger_data *test;
7227
7228         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
7229                                 lockdep_is_held(&event_mutex)) {
7230                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7231                         if (enable_data->enable)
7232                                 test->paused = false;
7233                         else
7234                                 test->paused = true;
7235                 }
7236         }
7237 }
7238
7239 static void
7240 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
7241                           struct ring_buffer_event *event)
7242 {
7243         if (!data->count)
7244                 return;
7245
7246         if (data->count != -1)
7247                 (data->count)--;
7248
7249         hist_enable_trigger(data, rec, event);
7250 }
7251
7252 static struct event_trigger_ops hist_enable_trigger_ops = {
7253         .func                   = hist_enable_trigger,
7254         .print                  = event_enable_trigger_print,
7255         .init                   = event_trigger_init,
7256         .free                   = event_enable_trigger_free,
7257 };
7258
7259 static struct event_trigger_ops hist_enable_count_trigger_ops = {
7260         .func                   = hist_enable_count_trigger,
7261         .print                  = event_enable_trigger_print,
7262         .init                   = event_trigger_init,
7263         .free                   = event_enable_trigger_free,
7264 };
7265
7266 static struct event_trigger_ops hist_disable_trigger_ops = {
7267         .func                   = hist_enable_trigger,
7268         .print                  = event_enable_trigger_print,
7269         .init                   = event_trigger_init,
7270         .free                   = event_enable_trigger_free,
7271 };
7272
7273 static struct event_trigger_ops hist_disable_count_trigger_ops = {
7274         .func                   = hist_enable_count_trigger,
7275         .print                  = event_enable_trigger_print,
7276         .init                   = event_trigger_init,
7277         .free                   = event_enable_trigger_free,
7278 };
7279
7280 static struct event_trigger_ops *
7281 hist_enable_get_trigger_ops(char *cmd, char *param)
7282 {
7283         struct event_trigger_ops *ops;
7284         bool enable;
7285
7286         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
7287
7288         if (enable)
7289                 ops = param ? &hist_enable_count_trigger_ops :
7290                         &hist_enable_trigger_ops;
7291         else
7292                 ops = param ? &hist_disable_count_trigger_ops :
7293                         &hist_disable_trigger_ops;
7294
7295         return ops;
7296 }
7297
7298 static void hist_enable_unreg_all(struct trace_event_file *file)
7299 {
7300         struct event_trigger_data *test, *n;
7301
7302         list_for_each_entry_safe(test, n, &file->triggers, list) {
7303                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
7304                         list_del_rcu(&test->list);
7305                         update_cond_flag(file);
7306                         trace_event_trigger_enable_disable(file, 0);
7307                         if (test->ops->free)
7308                                 test->ops->free(test->ops, test);
7309                 }
7310         }
7311 }
7312
7313 static struct event_command trigger_hist_enable_cmd = {
7314         .name                   = ENABLE_HIST_STR,
7315         .trigger_type           = ETT_HIST_ENABLE,
7316         .func                   = event_enable_trigger_func,
7317         .reg                    = event_enable_register_trigger,
7318         .unreg                  = event_enable_unregister_trigger,
7319         .unreg_all              = hist_enable_unreg_all,
7320         .get_trigger_ops        = hist_enable_get_trigger_ops,
7321         .set_filter             = set_trigger_filter,
7322 };
7323
7324 static struct event_command trigger_hist_disable_cmd = {
7325         .name                   = DISABLE_HIST_STR,
7326         .trigger_type           = ETT_HIST_ENABLE,
7327         .func                   = event_enable_trigger_func,
7328         .reg                    = event_enable_register_trigger,
7329         .unreg                  = event_enable_unregister_trigger,
7330         .unreg_all              = hist_enable_unreg_all,
7331         .get_trigger_ops        = hist_enable_get_trigger_ops,
7332         .set_filter             = set_trigger_filter,
7333 };
7334
7335 static __init void unregister_trigger_hist_enable_disable_cmds(void)
7336 {
7337         unregister_event_command(&trigger_hist_enable_cmd);
7338         unregister_event_command(&trigger_hist_disable_cmd);
7339 }
7340
7341 __init int register_trigger_hist_enable_disable_cmds(void)
7342 {
7343         int ret;
7344
7345         ret = register_event_command(&trigger_hist_enable_cmd);
7346         if (WARN_ON(ret < 0))
7347                 return ret;
7348         ret = register_event_command(&trigger_hist_disable_cmd);
7349         if (WARN_ON(ret < 0))
7350                 unregister_trigger_hist_enable_disable_cmds();
7351
7352         return ret;
7353 }
7354
7355 static __init int trace_events_hist_init(void)
7356 {
7357         struct dentry *entry = NULL;
7358         struct dentry *d_tracer;
7359         int err = 0;
7360
7361         err = dyn_event_register(&synth_event_ops);
7362         if (err) {
7363                 pr_warn("Could not register synth_event_ops\n");
7364                 return err;
7365         }
7366
7367         d_tracer = tracing_init_dentry();
7368         if (IS_ERR(d_tracer)) {
7369                 err = PTR_ERR(d_tracer);
7370                 goto err;
7371         }
7372
7373         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
7374                                     NULL, &synth_events_fops);
7375         if (!entry) {
7376                 err = -ENODEV;
7377                 goto err;
7378         }
7379
7380         return err;
7381  err:
7382         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
7383
7384         return err;
7385 }
7386
7387 fs_initcall(trace_events_hist_init);