kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18 #include "trace_dynevent.h"
  19
  20 #define SYNTH_SYSTEM            "synthetic"
  21 #define SYNTH_FIELDS_MAX        16
  22
  23 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  24
  25 #define ERRORS                                                          \
  26         C(NONE,                 "No error"),                            \
  27         C(DUPLICATE_VAR,        "Variable already defined"),            \
  28         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  29         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  30         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  31         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  32         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  33         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  34         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  35         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  36         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  37         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  38         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  39         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  40         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  41         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  42         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  43         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  44         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  45         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  46         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  47         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  48         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  49         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  50         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  51         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  52         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  53         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  54         C(INVALID_PARAM,        "Invalid action param"),                \
  55         C(ACTION_NOT_FOUND,     "No action found"),                     \
  56         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  57         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  58         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  59         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  60         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  61         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  62         C(INVALID_REF_KEY,      "Using variable references as keys not supported"), \
  63         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  64         C(FIELD_NOT_FOUND,      "Couldn't find field"),
  65
  66 #undef C
  67 #define C(a, b)         HIST_ERR_##a
  68
  69 enum { ERRORS };
  70
  71 #undef C
  72 #define C(a, b)         b
  73
  74 static const char *err_text[] = { ERRORS };
  75
  76 struct hist_field;
  77
  78 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  79                                 struct tracing_map_elt *elt,
  80                                 struct ring_buffer_event *rbe,
  81                                 void *event);
  82
  83 #define HIST_FIELD_OPERANDS_MAX 2
  84 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  85 #define HIST_ACTIONS_MAX        8
  86
  87 enum field_op_id {
  88         FIELD_OP_NONE,
  89         FIELD_OP_PLUS,
  90         FIELD_OP_MINUS,
  91         FIELD_OP_UNARY_MINUS,
  92 };
  93
  94 /*
  95  * A hist_var (histogram variable) contains variable information for
  96  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
  97  * flag set.  A hist_var has a variable name e.g. ts0, and is
  98  * associated with a given histogram trigger, as specified by
  99  * hist_data.  The hist_var idx is the unique index assigned to the
 100  * variable by the hist trigger's tracing_map.  The idx is what is
 101  * used to set a variable's value and, by a variable reference, to
 102  * retrieve it.
 103  */
 104 struct hist_var {
 105         char                            *name;
 106         struct hist_trigger_data        *hist_data;
 107         unsigned int                    idx;
 108 };
 109
 110 struct hist_field {
 111         struct ftrace_event_field       *field;
 112         unsigned long                   flags;
 113         hist_field_fn_t                 fn;
 114         unsigned int                    size;
 115         unsigned int                    offset;
 116         unsigned int                    is_signed;
 117         const char                      *type;
 118         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 119         struct hist_trigger_data        *hist_data;
 120
 121         /*
 122          * Variable fields contain variable-specific info in var.
 123          */
 124         struct hist_var                 var;
 125         enum field_op_id                operator;
 126         char                            *system;
 127         char                            *event_name;
 128
 129         /*
 130          * The name field is used for EXPR and VAR_REF fields.  VAR
 131          * fields contain the variable name in var.name.
 132          */
 133         char                            *name;
 134
 135         /*
 136          * When a histogram trigger is hit, if it has any references
 137          * to variables, the values of those variables are collected
 138          * into a var_ref_vals array by resolve_var_refs().  The
 139          * current value of each variable is read from the tracing_map
 140          * using the hist field's hist_var.idx and entered into the
 141          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 142          */
 143         unsigned int                    var_ref_idx;
 144         bool                            read_once;
 145 };
 146
 147 static u64 hist_field_none(struct hist_field *field,
 148                            struct tracing_map_elt *elt,
 149                            struct ring_buffer_event *rbe,
 150                            void *event)
 151 {
 152         return 0;
 153 }
 154
 155 static u64 hist_field_counter(struct hist_field *field,
 156                               struct tracing_map_elt *elt,
 157                               struct ring_buffer_event *rbe,
 158                               void *event)
 159 {
 160         return 1;
 161 }
 162
 163 static u64 hist_field_string(struct hist_field *hist_field,
 164                              struct tracing_map_elt *elt,
 165                              struct ring_buffer_event *rbe,
 166                              void *event)
 167 {
 168         char *addr = (char *)(event + hist_field->field->offset);
 169
 170         return (u64)(unsigned long)addr;
 171 }
 172
 173 static u64 hist_field_dynstring(struct hist_field *hist_field,
 174                                 struct tracing_map_elt *elt,
 175                                 struct ring_buffer_event *rbe,
 176                                 void *event)
 177 {
 178         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 179         int str_loc = str_item & 0xffff;
 180         char *addr = (char *)(event + str_loc);
 181
 182         return (u64)(unsigned long)addr;
 183 }
 184
 185 static u64 hist_field_pstring(struct hist_field *hist_field,
 186                               struct tracing_map_elt *elt,
 187                               struct ring_buffer_event *rbe,
 188                               void *event)
 189 {
 190         char **addr = (char **)(event + hist_field->field->offset);
 191
 192         return (u64)(unsigned long)*addr;
 193 }
 194
 195 static u64 hist_field_log2(struct hist_field *hist_field,
 196                            struct tracing_map_elt *elt,
 197                            struct ring_buffer_event *rbe,
 198                            void *event)
 199 {
 200         struct hist_field *operand = hist_field->operands[0];
 201
 202         u64 val = operand->fn(operand, elt, rbe, event);
 203
 204         return (u64) ilog2(roundup_pow_of_two(val));
 205 }
 206
 207 static u64 hist_field_plus(struct hist_field *hist_field,
 208                            struct tracing_map_elt *elt,
 209                            struct ring_buffer_event *rbe,
 210                            void *event)
 211 {
 212         struct hist_field *operand1 = hist_field->operands[0];
 213         struct hist_field *operand2 = hist_field->operands[1];
 214
 215         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 216         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 217
 218         return val1 + val2;
 219 }
 220
 221 static u64 hist_field_minus(struct hist_field *hist_field,
 222                             struct tracing_map_elt *elt,
 223                             struct ring_buffer_event *rbe,
 224                             void *event)
 225 {
 226         struct hist_field *operand1 = hist_field->operands[0];
 227         struct hist_field *operand2 = hist_field->operands[1];
 228
 229         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 230         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 231
 232         return val1 - val2;
 233 }
 234
 235 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 236                                   struct tracing_map_elt *elt,
 237                                   struct ring_buffer_event *rbe,
 238                                   void *event)
 239 {
 240         struct hist_field *operand = hist_field->operands[0];
 241
 242         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 243         u64 val = (u64)-sval;
 244
 245         return val;
 246 }
 247
 248 #define DEFINE_HIST_FIELD_FN(type)                                      \
 249         static u64 hist_field_##type(struct hist_field *hist_field,     \
 250                                      struct tracing_map_elt *elt,       \
 251                                      struct ring_buffer_event *rbe,     \
 252                                      void *event)                       \
 253 {                                                                       \
 254         type *addr = (type *)(event + hist_field->field->offset);       \
 255                                                                         \
 256         return (u64)(unsigned long)*addr;                               \
 257 }
 258
 259 DEFINE_HIST_FIELD_FN(s64);
 260 DEFINE_HIST_FIELD_FN(u64);
 261 DEFINE_HIST_FIELD_FN(s32);
 262 DEFINE_HIST_FIELD_FN(u32);
 263 DEFINE_HIST_FIELD_FN(s16);
 264 DEFINE_HIST_FIELD_FN(u16);
 265 DEFINE_HIST_FIELD_FN(s8);
 266 DEFINE_HIST_FIELD_FN(u8);
 267
 268 #define for_each_hist_field(i, hist_data)       \
 269         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 270
 271 #define for_each_hist_val_field(i, hist_data)   \
 272         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 273
 274 #define for_each_hist_key_field(i, hist_data)   \
 275         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 276
 277 #define HIST_STACKTRACE_DEPTH   16
 278 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 279 #define HIST_STACKTRACE_SKIP    5
 280
 281 #define HITCOUNT_IDX            0
 282 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 283
 284 enum hist_field_flags {
 285         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 286         HIST_FIELD_FL_KEY               = 1 << 1,
 287         HIST_FIELD_FL_STRING            = 1 << 2,
 288         HIST_FIELD_FL_HEX               = 1 << 3,
 289         HIST_FIELD_FL_SYM               = 1 << 4,
 290         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 291         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 292         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 293         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 294         HIST_FIELD_FL_LOG2              = 1 << 9,
 295         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 296         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 297         HIST_FIELD_FL_VAR               = 1 << 12,
 298         HIST_FIELD_FL_EXPR              = 1 << 13,
 299         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 300         HIST_FIELD_FL_CPU               = 1 << 15,
 301         HIST_FIELD_FL_ALIAS             = 1 << 16,
 302 };
 303
 304 struct var_defs {
 305         unsigned int    n_vars;
 306         char            *name[TRACING_MAP_VARS_MAX];
 307         char            *expr[TRACING_MAP_VARS_MAX];
 308 };
 309
 310 struct hist_trigger_attrs {
 311         char            *keys_str;
 312         char            *vals_str;
 313         char            *sort_key_str;
 314         char            *name;
 315         char            *clock;
 316         bool            pause;
 317         bool            cont;
 318         bool            clear;
 319         bool            ts_in_usecs;
 320         unsigned int    map_bits;
 321
 322         char            *assignment_str[TRACING_MAP_VARS_MAX];
 323         unsigned int    n_assignments;
 324
 325         char            *action_str[HIST_ACTIONS_MAX];
 326         unsigned int    n_actions;
 327
 328         struct var_defs var_defs;
 329 };
 330
 331 struct field_var {
 332         struct hist_field       *var;
 333         struct hist_field       *val;
 334 };
 335
 336 struct field_var_hist {
 337         struct hist_trigger_data        *hist_data;
 338         char                            *cmd;
 339 };
 340
 341 struct hist_trigger_data {
 342         struct hist_field               *fields[HIST_FIELDS_MAX];
 343         unsigned int                    n_vals;
 344         unsigned int                    n_keys;
 345         unsigned int                    n_fields;
 346         unsigned int                    n_vars;
 347         unsigned int                    key_size;
 348         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 349         unsigned int                    n_sort_keys;
 350         struct trace_event_file         *event_file;
 351         struct hist_trigger_attrs       *attrs;
 352         struct tracing_map              *map;
 353         bool                            enable_timestamps;
 354         bool                            remove;
 355         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 356         unsigned int                    n_var_refs;
 357
 358         struct action_data              *actions[HIST_ACTIONS_MAX];
 359         unsigned int                    n_actions;
 360
 361         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 362         unsigned int                    n_field_vars;
 363         unsigned int                    n_field_var_str;
 364         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 365         unsigned int                    n_field_var_hists;
 366
 367         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 368         unsigned int                    n_save_vars;
 369         unsigned int                    n_save_var_str;
 370 };
 371
 372 static int synth_event_create(int argc, const char **argv);
 373 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 374 static int synth_event_release(struct dyn_event *ev);
 375 static bool synth_event_is_busy(struct dyn_event *ev);
 376 static bool synth_event_match(const char *system, const char *event,
 377                               struct dyn_event *ev);
 378
 379 static struct dyn_event_operations synth_event_ops = {
 380         .create = synth_event_create,
 381         .show = synth_event_show,
 382         .is_busy = synth_event_is_busy,
 383         .free = synth_event_release,
 384         .match = synth_event_match,
 385 };
 386
 387 struct synth_field {
 388         char *type;
 389         char *name;
 390         size_t size;
 391         bool is_signed;
 392         bool is_string;
 393 };
 394
 395 struct synth_event {
 396         struct dyn_event                        devent;
 397         int                                     ref;
 398         char                                    *name;
 399         struct synth_field                      **fields;
 400         unsigned int                            n_fields;
 401         unsigned int                            n_u64;
 402         struct trace_event_class                class;
 403         struct trace_event_call                 call;
 404         struct tracepoint                       *tp;
 405 };
 406
 407 static bool is_synth_event(struct dyn_event *ev)
 408 {
 409         return ev->ops == &synth_event_ops;
 410 }
 411
 412 static struct synth_event *to_synth_event(struct dyn_event *ev)
 413 {
 414         return container_of(ev, struct synth_event, devent);
 415 }
 416
 417 static bool synth_event_is_busy(struct dyn_event *ev)
 418 {
 419         struct synth_event *event = to_synth_event(ev);
 420
 421         return event->ref != 0;
 422 }
 423
 424 static bool synth_event_match(const char *system, const char *event,
 425                               struct dyn_event *ev)
 426 {
 427         struct synth_event *sev = to_synth_event(ev);
 428
 429         return strcmp(sev->name, event) == 0 &&
 430                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 431 }
 432
 433 struct action_data;
 434
 435 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 436                              struct tracing_map_elt *elt, void *rec,
 437                              struct ring_buffer_event *rbe, void *key,
 438                              struct action_data *data, u64 *var_ref_vals);
 439
 440 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 441
 442 enum handler_id {
 443         HANDLER_ONMATCH = 1,
 444         HANDLER_ONMAX,
 445         HANDLER_ONCHANGE,
 446 };
 447
 448 enum action_id {
 449         ACTION_SAVE = 1,
 450         ACTION_TRACE,
 451         ACTION_SNAPSHOT,
 452 };
 453
 454 struct action_data {
 455         enum handler_id         handler;
 456         enum action_id          action;
 457         char                    *action_name;
 458         action_fn_t             fn;
 459
 460         unsigned int            n_params;
 461         char                    *params[SYNTH_FIELDS_MAX];
 462
 463         /*
 464          * When a histogram trigger is hit, the values of any
 465          * references to variables, including variables being passed
 466          * as parameters to synthetic events, are collected into a
 467          * var_ref_vals array.  This var_ref_idx is the index of the
 468          * first param in the array to be passed to the synthetic
 469          * event invocation.
 470          */
 471         unsigned int            var_ref_idx;
 472         struct synth_event      *synth_event;
 473         bool                    use_trace_keyword;
 474         char                    *synth_event_name;
 475
 476         union {
 477                 struct {
 478                         char                    *event;
 479                         char                    *event_system;
 480                 } match_data;
 481
 482                 struct {
 483                         /*
 484                          * var_str contains the $-unstripped variable
 485                          * name referenced by var_ref, and used when
 486                          * printing the action.  Because var_ref
 487                          * creation is deferred to create_actions(),
 488                          * we need a per-action way to save it until
 489                          * then, thus var_str.
 490                          */
 491                         char                    *var_str;
 492
 493                         /*
 494                          * var_ref refers to the variable being
 495                          * tracked e.g onmax($var).
 496                          */
 497                         struct hist_field       *var_ref;
 498
 499                         /*
 500                          * track_var contains the 'invisible' tracking
 501                          * variable created to keep the current
 502                          * e.g. max value.
 503                          */
 504                         struct hist_field       *track_var;
 505
 506                         check_track_val_fn_t    check_val;
 507                         action_fn_t             save_data;
 508                 } track_data;
 509         };
 510 };
 511
 512 struct track_data {
 513         u64                             track_val;
 514         bool                            updated;
 515
 516         unsigned int                    key_len;
 517         void                            *key;
 518         struct tracing_map_elt          elt;
 519
 520         struct action_data              *action_data;
 521         struct hist_trigger_data        *hist_data;
 522 };
 523
 524 struct hist_elt_data {
 525         char *comm;
 526         u64 *var_ref_vals;
 527         char *field_var_str[SYNTH_FIELDS_MAX];
 528 };
 529
 530 struct snapshot_context {
 531         struct tracing_map_elt  *elt;
 532         void                    *key;
 533 };
 534
 535 static void track_data_free(struct track_data *track_data)
 536 {
 537         struct hist_elt_data *elt_data;
 538
 539         if (!track_data)
 540                 return;
 541
 542         kfree(track_data->key);
 543
 544         elt_data = track_data->elt.private_data;
 545         if (elt_data) {
 546                 kfree(elt_data->comm);
 547                 kfree(elt_data);
 548         }
 549
 550         kfree(track_data);
 551 }
 552
 553 static struct track_data *track_data_alloc(unsigned int key_len,
 554                                            struct action_data *action_data,
 555                                            struct hist_trigger_data *hist_data)
 556 {
 557         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 558         struct hist_elt_data *elt_data;
 559
 560         if (!data)
 561                 return ERR_PTR(-ENOMEM);
 562
 563         data->key = kzalloc(key_len, GFP_KERNEL);
 564         if (!data->key) {
 565                 track_data_free(data);
 566                 return ERR_PTR(-ENOMEM);
 567         }
 568
 569         data->key_len = key_len;
 570         data->action_data = action_data;
 571         data->hist_data = hist_data;
 572
 573         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 574         if (!elt_data) {
 575                 track_data_free(data);
 576                 return ERR_PTR(-ENOMEM);
 577         }
 578         data->elt.private_data = elt_data;
 579
 580         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 581         if (!elt_data->comm) {
 582                 track_data_free(data);
 583                 return ERR_PTR(-ENOMEM);
 584         }
 585
 586         return data;
 587 }
 588
 589 static char last_cmd[MAX_FILTER_STR_VAL];
 590 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 591
 592 static int errpos(char *str)
 593 {
 594         return err_pos(last_cmd, str);
 595 }
 596
 597 static void last_cmd_set(struct trace_event_file *file, char *str)
 598 {
 599         const char *system = NULL, *name = NULL;
 600         struct trace_event_call *call;
 601
 602         if (!str)
 603                 return;
 604
 605         strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 606
 607         if (file) {
 608                 call = file->event_call;
 609
 610                 system = call->class->system;
 611                 if (system) {
 612                         name = trace_event_name(call);
 613                         if (!name)
 614                                 system = NULL;
 615                 }
 616         }
 617
 618         if (system)
 619                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 620 }
 621
 622 static void hist_err(u8 err_type, u8 err_pos)
 623 {
 624         tracing_log_err(last_cmd_loc, last_cmd, err_text, err_type, err_pos);
 625 }
 626
 627 static void hist_err_clear(void)
 628 {
 629         last_cmd[0] = '\0';
 630         last_cmd_loc[0] = '\0';
 631 }
 632
 633 struct synth_trace_event {
 634         struct trace_entry      ent;
 635         u64                     fields[];
 636 };
 637
 638 static int synth_event_define_fields(struct trace_event_call *call)
 639 {
 640         struct synth_trace_event trace;
 641         int offset = offsetof(typeof(trace), fields);
 642         struct synth_event *event = call->data;
 643         unsigned int i, size, n_u64;
 644         char *name, *type;
 645         bool is_signed;
 646         int ret = 0;
 647
 648         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 649                 size = event->fields[i]->size;
 650                 is_signed = event->fields[i]->is_signed;
 651                 type = event->fields[i]->type;
 652                 name = event->fields[i]->name;
 653                 ret = trace_define_field(call, type, name, offset, size,
 654                                          is_signed, FILTER_OTHER);
 655                 if (ret)
 656                         break;
 657
 658                 if (event->fields[i]->is_string) {
 659                         offset += STR_VAR_LEN_MAX;
 660                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 661                 } else {
 662                         offset += sizeof(u64);
 663                         n_u64++;
 664                 }
 665         }
 666
 667         event->n_u64 = n_u64;
 668
 669         return ret;
 670 }
 671
 672 static bool synth_field_signed(char *type)
 673 {
 674         if (str_has_prefix(type, "u"))
 675                 return false;
 676
 677         return true;
 678 }
 679
 680 static int synth_field_is_string(char *type)
 681 {
 682         if (strstr(type, "char[") != NULL)
 683                 return true;
 684
 685         return false;
 686 }
 687
 688 static int synth_field_string_size(char *type)
 689 {
 690         char buf[4], *end, *start;
 691         unsigned int len;
 692         int size, err;
 693
 694         start = strstr(type, "char[");
 695         if (start == NULL)
 696                 return -EINVAL;
 697         start += sizeof("char[") - 1;
 698
 699         end = strchr(type, ']');
 700         if (!end || end < start)
 701                 return -EINVAL;
 702
 703         len = end - start;
 704         if (len > 3)
 705                 return -EINVAL;
 706
 707         strncpy(buf, start, len);
 708         buf[len] = '\0';
 709
 710         err = kstrtouint(buf, 0, &size);
 711         if (err)
 712                 return err;
 713
 714         if (size > STR_VAR_LEN_MAX)
 715                 return -EINVAL;
 716
 717         return size;
 718 }
 719
 720 static int synth_field_size(char *type)
 721 {
 722         int size = 0;
 723
 724         if (strcmp(type, "s64") == 0)
 725                 size = sizeof(s64);
 726         else if (strcmp(type, "u64") == 0)
 727                 size = sizeof(u64);
 728         else if (strcmp(type, "s32") == 0)
 729                 size = sizeof(s32);
 730         else if (strcmp(type, "u32") == 0)
 731                 size = sizeof(u32);
 732         else if (strcmp(type, "s16") == 0)
 733                 size = sizeof(s16);
 734         else if (strcmp(type, "u16") == 0)
 735                 size = sizeof(u16);
 736         else if (strcmp(type, "s8") == 0)
 737                 size = sizeof(s8);
 738         else if (strcmp(type, "u8") == 0)
 739                 size = sizeof(u8);
 740         else if (strcmp(type, "char") == 0)
 741                 size = sizeof(char);
 742         else if (strcmp(type, "unsigned char") == 0)
 743                 size = sizeof(unsigned char);
 744         else if (strcmp(type, "int") == 0)
 745                 size = sizeof(int);
 746         else if (strcmp(type, "unsigned int") == 0)
 747                 size = sizeof(unsigned int);
 748         else if (strcmp(type, "long") == 0)
 749                 size = sizeof(long);
 750         else if (strcmp(type, "unsigned long") == 0)
 751                 size = sizeof(unsigned long);
 752         else if (strcmp(type, "pid_t") == 0)
 753                 size = sizeof(pid_t);
 754         else if (synth_field_is_string(type))
 755                 size = synth_field_string_size(type);
 756
 757         return size;
 758 }
 759
 760 static const char *synth_field_fmt(char *type)
 761 {
 762         const char *fmt = "%llu";
 763
 764         if (strcmp(type, "s64") == 0)
 765                 fmt = "%lld";
 766         else if (strcmp(type, "u64") == 0)
 767                 fmt = "%llu";
 768         else if (strcmp(type, "s32") == 0)
 769                 fmt = "%d";
 770         else if (strcmp(type, "u32") == 0)
 771                 fmt = "%u";
 772         else if (strcmp(type, "s16") == 0)
 773                 fmt = "%d";
 774         else if (strcmp(type, "u16") == 0)
 775                 fmt = "%u";
 776         else if (strcmp(type, "s8") == 0)
 777                 fmt = "%d";
 778         else if (strcmp(type, "u8") == 0)
 779                 fmt = "%u";
 780         else if (strcmp(type, "char") == 0)
 781                 fmt = "%d";
 782         else if (strcmp(type, "unsigned char") == 0)
 783                 fmt = "%u";
 784         else if (strcmp(type, "int") == 0)
 785                 fmt = "%d";
 786         else if (strcmp(type, "unsigned int") == 0)
 787                 fmt = "%u";
 788         else if (strcmp(type, "long") == 0)
 789                 fmt = "%ld";
 790         else if (strcmp(type, "unsigned long") == 0)
 791                 fmt = "%lu";
 792         else if (strcmp(type, "pid_t") == 0)
 793                 fmt = "%d";
 794         else if (synth_field_is_string(type))
 795                 fmt = "%s";
 796
 797         return fmt;
 798 }
 799
 800 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 801                                            int flags,
 802                                            struct trace_event *event)
 803 {
 804         struct trace_array *tr = iter->tr;
 805         struct trace_seq *s = &iter->seq;
 806         struct synth_trace_event *entry;
 807         struct synth_event *se;
 808         unsigned int i, n_u64;
 809         char print_fmt[32];
 810         const char *fmt;
 811
 812         entry = (struct synth_trace_event *)iter->ent;
 813         se = container_of(event, struct synth_event, call.event);
 814
 815         trace_seq_printf(s, "%s: ", se->name);
 816
 817         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 818                 if (trace_seq_has_overflowed(s))
 819                         goto end;
 820
 821                 fmt = synth_field_fmt(se->fields[i]->type);
 822
 823                 /* parameter types */
 824                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 825                         trace_seq_printf(s, "%s ", fmt);
 826
 827                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 828
 829                 /* parameter values */
 830                 if (se->fields[i]->is_string) {
 831                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 832                                          (char *)&entry->fields[n_u64],
 833                                          i == se->n_fields - 1 ? "" : " ");
 834                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 835                 } else {
 836                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 837                                          entry->fields[n_u64],
 838                                          i == se->n_fields - 1 ? "" : " ");
 839                         n_u64++;
 840                 }
 841         }
 842 end:
 843         trace_seq_putc(s, '\n');
 844
 845         return trace_handle_return(s);
 846 }
 847
 848 static struct trace_event_functions synth_event_funcs = {
 849         .trace          = print_synth_event
 850 };
 851
 852 static notrace void trace_event_raw_event_synth(void *__data,
 853                                                 u64 *var_ref_vals,
 854                                                 unsigned int var_ref_idx)
 855 {
 856         struct trace_event_file *trace_file = __data;
 857         struct synth_trace_event *entry;
 858         struct trace_event_buffer fbuffer;
 859         struct ring_buffer *buffer;
 860         struct synth_event *event;
 861         unsigned int i, n_u64;
 862         int fields_size = 0;
 863
 864         event = trace_file->event_call->data;
 865
 866         if (trace_trigger_soft_disabled(trace_file))
 867                 return;
 868
 869         fields_size = event->n_u64 * sizeof(u64);
 870
 871         /*
 872          * Avoid ring buffer recursion detection, as this event
 873          * is being performed within another event.
 874          */
 875         buffer = trace_file->tr->trace_buffer.buffer;
 876         ring_buffer_nest_start(buffer);
 877
 878         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 879                                            sizeof(*entry) + fields_size);
 880         if (!entry)
 881                 goto out;
 882
 883         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 884                 if (event->fields[i]->is_string) {
 885                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 886                         char *str_field = (char *)&entry->fields[n_u64];
 887
 888                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 889                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 890                 } else {
 891                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 892                         n_u64++;
 893                 }
 894         }
 895
 896         trace_event_buffer_commit(&fbuffer);
 897 out:
 898         ring_buffer_nest_end(buffer);
 899 }
 900
 901 static void free_synth_event_print_fmt(struct trace_event_call *call)
 902 {
 903         if (call) {
 904                 kfree(call->print_fmt);
 905                 call->print_fmt = NULL;
 906         }
 907 }
 908
 909 static int __set_synth_event_print_fmt(struct synth_event *event,
 910                                        char *buf, int len)
 911 {
 912         const char *fmt;
 913         int pos = 0;
 914         int i;
 915
 916         /* When len=0, we just calculate the needed length */
 917 #define LEN_OR_ZERO (len ? len - pos : 0)
 918
 919         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 920         for (i = 0; i < event->n_fields; i++) {
 921                 fmt = synth_field_fmt(event->fields[i]->type);
 922                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 923                                 event->fields[i]->name, fmt,
 924                                 i == event->n_fields - 1 ? "" : ", ");
 925         }
 926         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 927
 928         for (i = 0; i < event->n_fields; i++) {
 929                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 930                                 ", REC->%s", event->fields[i]->name);
 931         }
 932
 933 #undef LEN_OR_ZERO
 934
 935         /* return the length of print_fmt */
 936         return pos;
 937 }
 938
 939 static int set_synth_event_print_fmt(struct trace_event_call *call)
 940 {
 941         struct synth_event *event = call->data;
 942         char *print_fmt;
 943         int len;
 944
 945         /* First: called with 0 length to calculate the needed length */
 946         len = __set_synth_event_print_fmt(event, NULL, 0);
 947
 948         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 949         if (!print_fmt)
 950                 return -ENOMEM;
 951
 952         /* Second: actually write the @print_fmt */
 953         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 954         call->print_fmt = print_fmt;
 955
 956         return 0;
 957 }
 958
 959 static void free_synth_field(struct synth_field *field)
 960 {
 961         kfree(field->type);
 962         kfree(field->name);
 963         kfree(field);
 964 }
 965
 966 static struct synth_field *parse_synth_field(int argc, const char **argv,
 967                                              int *consumed)
 968 {
 969         struct synth_field *field;
 970         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 971         int len, ret = 0;
 972
 973         if (field_type[0] == ';')
 974                 field_type++;
 975
 976         if (!strcmp(field_type, "unsigned")) {
 977                 if (argc < 3)
 978                         return ERR_PTR(-EINVAL);
 979                 prefix = "unsigned ";
 980                 field_type = argv[1];
 981                 field_name = argv[2];
 982                 *consumed = 3;
 983         } else {
 984                 field_name = argv[1];
 985                 *consumed = 2;
 986         }
 987
 988         field = kzalloc(sizeof(*field), GFP_KERNEL);
 989         if (!field)
 990                 return ERR_PTR(-ENOMEM);
 991
 992         len = strlen(field_name);
 993         array = strchr(field_name, '[');
 994         if (array)
 995                 len -= strlen(array);
 996         else if (field_name[len - 1] == ';')
 997                 len--;
 998
 999         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1000         if (!field->name) {
1001                 ret = -ENOMEM;
1002                 goto free;
1003         }
1004
1005         if (field_type[0] == ';')
1006                 field_type++;
1007         len = strlen(field_type) + 1;
1008         if (array)
1009                 len += strlen(array);
1010         if (prefix)
1011                 len += strlen(prefix);
1012
1013         field->type = kzalloc(len, GFP_KERNEL);
1014         if (!field->type) {
1015                 ret = -ENOMEM;
1016                 goto free;
1017         }
1018         if (prefix)
1019                 strcat(field->type, prefix);
1020         strcat(field->type, field_type);
1021         if (array) {
1022                 strcat(field->type, array);
1023                 if (field->type[len - 1] == ';')
1024                         field->type[len - 1] = '\0';
1025         }
1026
1027         field->size = synth_field_size(field->type);
1028         if (!field->size) {
1029                 ret = -EINVAL;
1030                 goto free;
1031         }
1032
1033         if (synth_field_is_string(field->type))
1034                 field->is_string = true;
1035
1036         field->is_signed = synth_field_signed(field->type);
1037
1038  out:
1039         return field;
1040  free:
1041         free_synth_field(field);
1042         field = ERR_PTR(ret);
1043         goto out;
1044 }
1045
1046 static void free_synth_tracepoint(struct tracepoint *tp)
1047 {
1048         if (!tp)
1049                 return;
1050
1051         kfree(tp->name);
1052         kfree(tp);
1053 }
1054
1055 static struct tracepoint *alloc_synth_tracepoint(char *name)
1056 {
1057         struct tracepoint *tp;
1058
1059         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1060         if (!tp)
1061                 return ERR_PTR(-ENOMEM);
1062
1063         tp->name = kstrdup(name, GFP_KERNEL);
1064         if (!tp->name) {
1065                 kfree(tp);
1066                 return ERR_PTR(-ENOMEM);
1067         }
1068
1069         return tp;
1070 }
1071
1072 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1073                                     unsigned int var_ref_idx);
1074
1075 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1076                                unsigned int var_ref_idx)
1077 {
1078         struct tracepoint *tp = event->tp;
1079
1080         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1081                 struct tracepoint_func *probe_func_ptr;
1082                 synth_probe_func_t probe_func;
1083                 void *__data;
1084
1085                 if (!(cpu_online(raw_smp_processor_id())))
1086                         return;
1087
1088                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1089                 if (probe_func_ptr) {
1090                         do {
1091                                 probe_func = probe_func_ptr->func;
1092                                 __data = probe_func_ptr->data;
1093                                 probe_func(__data, var_ref_vals, var_ref_idx);
1094                         } while ((++probe_func_ptr)->func);
1095                 }
1096         }
1097 }
1098
1099 static struct synth_event *find_synth_event(const char *name)
1100 {
1101         struct dyn_event *pos;
1102         struct synth_event *event;
1103
1104         for_each_dyn_event(pos) {
1105                 if (!is_synth_event(pos))
1106                         continue;
1107                 event = to_synth_event(pos);
1108                 if (strcmp(event->name, name) == 0)
1109                         return event;
1110         }
1111
1112         return NULL;
1113 }
1114
1115 static int register_synth_event(struct synth_event *event)
1116 {
1117         struct trace_event_call *call = &event->call;
1118         int ret = 0;
1119
1120         event->call.class = &event->class;
1121         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1122         if (!event->class.system) {
1123                 ret = -ENOMEM;
1124                 goto out;
1125         }
1126
1127         event->tp = alloc_synth_tracepoint(event->name);
1128         if (IS_ERR(event->tp)) {
1129                 ret = PTR_ERR(event->tp);
1130                 event->tp = NULL;
1131                 goto out;
1132         }
1133
1134         INIT_LIST_HEAD(&call->class->fields);
1135         call->event.funcs = &synth_event_funcs;
1136         call->class->define_fields = synth_event_define_fields;
1137
1138         ret = register_trace_event(&call->event);
1139         if (!ret) {
1140                 ret = -ENODEV;
1141                 goto out;
1142         }
1143         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1144         call->class->reg = trace_event_reg;
1145         call->class->probe = trace_event_raw_event_synth;
1146         call->data = event;
1147         call->tp = event->tp;
1148
1149         ret = trace_add_event_call(call);
1150         if (ret) {
1151                 pr_warn("Failed to register synthetic event: %s\n",
1152                         trace_event_name(call));
1153                 goto err;
1154         }
1155
1156         ret = set_synth_event_print_fmt(call);
1157         if (ret < 0) {
1158                 trace_remove_event_call(call);
1159                 goto err;
1160         }
1161  out:
1162         return ret;
1163  err:
1164         unregister_trace_event(&call->event);
1165         goto out;
1166 }
1167
1168 static int unregister_synth_event(struct synth_event *event)
1169 {
1170         struct trace_event_call *call = &event->call;
1171         int ret;
1172
1173         ret = trace_remove_event_call(call);
1174
1175         return ret;
1176 }
1177
1178 static void free_synth_event(struct synth_event *event)
1179 {
1180         unsigned int i;
1181
1182         if (!event)
1183                 return;
1184
1185         for (i = 0; i < event->n_fields; i++)
1186                 free_synth_field(event->fields[i]);
1187
1188         kfree(event->fields);
1189         kfree(event->name);
1190         kfree(event->class.system);
1191         free_synth_tracepoint(event->tp);
1192         free_synth_event_print_fmt(&event->call);
1193         kfree(event);
1194 }
1195
1196 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1197                                              struct synth_field **fields)
1198 {
1199         struct synth_event *event;
1200         unsigned int i;
1201
1202         event = kzalloc(sizeof(*event), GFP_KERNEL);
1203         if (!event) {
1204                 event = ERR_PTR(-ENOMEM);
1205                 goto out;
1206         }
1207
1208         event->name = kstrdup(name, GFP_KERNEL);
1209         if (!event->name) {
1210                 kfree(event);
1211                 event = ERR_PTR(-ENOMEM);
1212                 goto out;
1213         }
1214
1215         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1216         if (!event->fields) {
1217                 free_synth_event(event);
1218                 event = ERR_PTR(-ENOMEM);
1219                 goto out;
1220         }
1221
1222         dyn_event_init(&event->devent, &synth_event_ops);
1223
1224         for (i = 0; i < n_fields; i++)
1225                 event->fields[i] = fields[i];
1226
1227         event->n_fields = n_fields;
1228  out:
1229         return event;
1230 }
1231
1232 static void action_trace(struct hist_trigger_data *hist_data,
1233                          struct tracing_map_elt *elt, void *rec,
1234                          struct ring_buffer_event *rbe, void *key,
1235                          struct action_data *data, u64 *var_ref_vals)
1236 {
1237         struct synth_event *event = data->synth_event;
1238
1239         trace_synth(event, var_ref_vals, data->var_ref_idx);
1240 }
1241
1242 struct hist_var_data {
1243         struct list_head list;
1244         struct hist_trigger_data *hist_data;
1245 };
1246
1247 static int __create_synth_event(int argc, const char *name, const char **argv)
1248 {
1249         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1250         struct synth_event *event = NULL;
1251         int i, consumed = 0, n_fields = 0, ret = 0;
1252
1253         /*
1254          * Argument syntax:
1255          *  - Add synthetic event: <event_name> field[;field] ...
1256          *  - Remove synthetic event: !<event_name> field[;field] ...
1257          *      where 'field' = type field_name
1258          */
1259
1260         if (name[0] == '\0' || argc < 1)
1261                 return -EINVAL;
1262
1263         mutex_lock(&event_mutex);
1264
1265         event = find_synth_event(name);
1266         if (event) {
1267                 ret = -EEXIST;
1268                 goto out;
1269         }
1270
1271         for (i = 0; i < argc - 1; i++) {
1272                 if (strcmp(argv[i], ";") == 0)
1273                         continue;
1274                 if (n_fields == SYNTH_FIELDS_MAX) {
1275                         ret = -EINVAL;
1276                         goto err;
1277                 }
1278
1279                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1280                 if (IS_ERR(field)) {
1281                         ret = PTR_ERR(field);
1282                         goto err;
1283                 }
1284                 fields[n_fields++] = field;
1285                 i += consumed - 1;
1286         }
1287
1288         if (i < argc && strcmp(argv[i], ";") != 0) {
1289                 ret = -EINVAL;
1290                 goto err;
1291         }
1292
1293         event = alloc_synth_event(name, n_fields, fields);
1294         if (IS_ERR(event)) {
1295                 ret = PTR_ERR(event);
1296                 event = NULL;
1297                 goto err;
1298         }
1299         ret = register_synth_event(event);
1300         if (!ret)
1301                 dyn_event_add(&event->devent);
1302         else
1303                 free_synth_event(event);
1304  out:
1305         mutex_unlock(&event_mutex);
1306
1307         return ret;
1308  err:
1309         for (i = 0; i < n_fields; i++)
1310                 free_synth_field(fields[i]);
1311
1312         goto out;
1313 }
1314
1315 static int create_or_delete_synth_event(int argc, char **argv)
1316 {
1317         const char *name = argv[0];
1318         struct synth_event *event = NULL;
1319         int ret;
1320
1321         /* trace_run_command() ensures argc != 0 */
1322         if (name[0] == '!') {
1323                 mutex_lock(&event_mutex);
1324                 event = find_synth_event(name + 1);
1325                 if (event) {
1326                         if (event->ref)
1327                                 ret = -EBUSY;
1328                         else {
1329                                 ret = unregister_synth_event(event);
1330                                 if (!ret) {
1331                                         dyn_event_remove(&event->devent);
1332                                         free_synth_event(event);
1333                                 }
1334                         }
1335                 } else
1336                         ret = -ENOENT;
1337                 mutex_unlock(&event_mutex);
1338                 return ret;
1339         }
1340
1341         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1342         return ret == -ECANCELED ? -EINVAL : ret;
1343 }
1344
1345 static int synth_event_create(int argc, const char **argv)
1346 {
1347         const char *name = argv[0];
1348         int len;
1349
1350         if (name[0] != 's' || name[1] != ':')
1351                 return -ECANCELED;
1352         name += 2;
1353
1354         /* This interface accepts group name prefix */
1355         if (strchr(name, '/')) {
1356                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1357                 if (len == 0)
1358                         return -EINVAL;
1359                 name += len;
1360         }
1361         return __create_synth_event(argc - 1, name, argv + 1);
1362 }
1363
1364 static int synth_event_release(struct dyn_event *ev)
1365 {
1366         struct synth_event *event = to_synth_event(ev);
1367         int ret;
1368
1369         if (event->ref)
1370                 return -EBUSY;
1371
1372         ret = unregister_synth_event(event);
1373         if (ret)
1374                 return ret;
1375
1376         dyn_event_remove(ev);
1377         free_synth_event(event);
1378         return 0;
1379 }
1380
1381 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1382 {
1383         struct synth_field *field;
1384         unsigned int i;
1385
1386         seq_printf(m, "%s\t", event->name);
1387
1388         for (i = 0; i < event->n_fields; i++) {
1389                 field = event->fields[i];
1390
1391                 /* parameter values */
1392                 seq_printf(m, "%s %s%s", field->type, field->name,
1393                            i == event->n_fields - 1 ? "" : "; ");
1394         }
1395
1396         seq_putc(m, '\n');
1397
1398         return 0;
1399 }
1400
1401 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1402 {
1403         struct synth_event *event = to_synth_event(ev);
1404
1405         seq_printf(m, "s:%s/", event->class.system);
1406
1407         return __synth_event_show(m, event);
1408 }
1409
1410 static int synth_events_seq_show(struct seq_file *m, void *v)
1411 {
1412         struct dyn_event *ev = v;
1413
1414         if (!is_synth_event(ev))
1415                 return 0;
1416
1417         return __synth_event_show(m, to_synth_event(ev));
1418 }
1419
1420 static const struct seq_operations synth_events_seq_op = {
1421         .start  = dyn_event_seq_start,
1422         .next   = dyn_event_seq_next,
1423         .stop   = dyn_event_seq_stop,
1424         .show   = synth_events_seq_show,
1425 };
1426
1427 static int synth_events_open(struct inode *inode, struct file *file)
1428 {
1429         int ret;
1430
1431         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1432                 ret = dyn_events_release_all(&synth_event_ops);
1433                 if (ret < 0)
1434                         return ret;
1435         }
1436
1437         return seq_open(file, &synth_events_seq_op);
1438 }
1439
1440 static ssize_t synth_events_write(struct file *file,
1441                                   const char __user *buffer,
1442                                   size_t count, loff_t *ppos)
1443 {
1444         return trace_parse_run_command(file, buffer, count, ppos,
1445                                        create_or_delete_synth_event);
1446 }
1447
1448 static const struct file_operations synth_events_fops = {
1449         .open           = synth_events_open,
1450         .write          = synth_events_write,
1451         .read           = seq_read,
1452         .llseek         = seq_lseek,
1453         .release        = seq_release,
1454 };
1455
1456 static u64 hist_field_timestamp(struct hist_field *hist_field,
1457                                 struct tracing_map_elt *elt,
1458                                 struct ring_buffer_event *rbe,
1459                                 void *event)
1460 {
1461         struct hist_trigger_data *hist_data = hist_field->hist_data;
1462         struct trace_array *tr = hist_data->event_file->tr;
1463
1464         u64 ts = ring_buffer_event_time_stamp(rbe);
1465
1466         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1467                 ts = ns2usecs(ts);
1468
1469         return ts;
1470 }
1471
1472 static u64 hist_field_cpu(struct hist_field *hist_field,
1473                           struct tracing_map_elt *elt,
1474                           struct ring_buffer_event *rbe,
1475                           void *event)
1476 {
1477         int cpu = smp_processor_id();
1478
1479         return cpu;
1480 }
1481
1482 /**
1483  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1484  * @hist_field: The VAR_REF field to check
1485  * @var_data: The hist trigger that owns the variable
1486  * @var_idx: The trigger variable identifier
1487  *
1488  * Check the given VAR_REF field to see whether or not it references
1489  * the given variable associated with the given trigger.
1490  *
1491  * Return: The VAR_REF field if it does reference the variable, NULL if not
1492  */
1493 static struct hist_field *
1494 check_field_for_var_ref(struct hist_field *hist_field,
1495                         struct hist_trigger_data *var_data,
1496                         unsigned int var_idx)
1497 {
1498         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1499
1500         if (hist_field && hist_field->var.idx == var_idx &&
1501             hist_field->var.hist_data == var_data)
1502                 return hist_field;
1503
1504         return NULL;
1505 }
1506
1507 /**
1508  * find_var_ref - Check if a trigger has a reference to a trigger variable
1509  * @hist_data: The hist trigger that might have a reference to the variable
1510  * @var_data: The hist trigger that owns the variable
1511  * @var_idx: The trigger variable identifier
1512  *
1513  * Check the list of var_refs[] on the first hist trigger to see
1514  * whether any of them are references to the variable on the second
1515  * trigger.
1516  *
1517  * Return: The VAR_REF field referencing the variable if so, NULL if not
1518  */
1519 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1520                                        struct hist_trigger_data *var_data,
1521                                        unsigned int var_idx)
1522 {
1523         struct hist_field *hist_field;
1524         unsigned int i;
1525
1526         for (i = 0; i < hist_data->n_var_refs; i++) {
1527                 hist_field = hist_data->var_refs[i];
1528                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1529                         return hist_field;
1530         }
1531
1532         return NULL;
1533 }
1534
1535 /**
1536  * find_any_var_ref - Check if there is a reference to a given trigger variable
1537  * @hist_data: The hist trigger
1538  * @var_idx: The trigger variable identifier
1539  *
1540  * Check to see whether the given variable is currently referenced by
1541  * any other trigger.
1542  *
1543  * The trigger the variable is defined on is explicitly excluded - the
1544  * assumption being that a self-reference doesn't prevent a trigger
1545  * from being removed.
1546  *
1547  * Return: The VAR_REF field referencing the variable if so, NULL if not
1548  */
1549 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1550                                            unsigned int var_idx)
1551 {
1552         struct trace_array *tr = hist_data->event_file->tr;
1553         struct hist_field *found = NULL;
1554         struct hist_var_data *var_data;
1555
1556         list_for_each_entry(var_data, &tr->hist_vars, list) {
1557                 if (var_data->hist_data == hist_data)
1558                         continue;
1559                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1560                 if (found)
1561                         break;
1562         }
1563
1564         return found;
1565 }
1566
1567 /**
1568  * check_var_refs - Check if there is a reference to any of trigger's variables
1569  * @hist_data: The hist trigger
1570  *
1571  * A trigger can define one or more variables.  If any one of them is
1572  * currently referenced by any other trigger, this function will
1573  * determine that.
1574
1575  * Typically used to determine whether or not a trigger can be removed
1576  * - if there are any references to a trigger's variables, it cannot.
1577  *
1578  * Return: True if there is a reference to any of trigger's variables
1579  */
1580 static bool check_var_refs(struct hist_trigger_data *hist_data)
1581 {
1582         struct hist_field *field;
1583         bool found = false;
1584         int i;
1585
1586         for_each_hist_field(i, hist_data) {
1587                 field = hist_data->fields[i];
1588                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1589                         if (find_any_var_ref(hist_data, field->var.idx)) {
1590                                 found = true;
1591                                 break;
1592                         }
1593                 }
1594         }
1595
1596         return found;
1597 }
1598
1599 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1600 {
1601         struct trace_array *tr = hist_data->event_file->tr;
1602         struct hist_var_data *var_data, *found = NULL;
1603
1604         list_for_each_entry(var_data, &tr->hist_vars, list) {
1605                 if (var_data->hist_data == hist_data) {
1606                         found = var_data;
1607                         break;
1608                 }
1609         }
1610
1611         return found;
1612 }
1613
1614 static bool field_has_hist_vars(struct hist_field *hist_field,
1615                                 unsigned int level)
1616 {
1617         int i;
1618
1619         if (level > 3)
1620                 return false;
1621
1622         if (!hist_field)
1623                 return false;
1624
1625         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1626             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1627                 return true;
1628
1629         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1630                 struct hist_field *operand;
1631
1632                 operand = hist_field->operands[i];
1633                 if (field_has_hist_vars(operand, level + 1))
1634                         return true;
1635         }
1636
1637         return false;
1638 }
1639
1640 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1641 {
1642         struct hist_field *hist_field;
1643         int i;
1644
1645         for_each_hist_field(i, hist_data) {
1646                 hist_field = hist_data->fields[i];
1647                 if (field_has_hist_vars(hist_field, 0))
1648                         return true;
1649         }
1650
1651         return false;
1652 }
1653
1654 static int save_hist_vars(struct hist_trigger_data *hist_data)
1655 {
1656         struct trace_array *tr = hist_data->event_file->tr;
1657         struct hist_var_data *var_data;
1658
1659         var_data = find_hist_vars(hist_data);
1660         if (var_data)
1661                 return 0;
1662
1663         if (trace_array_get(tr) < 0)
1664                 return -ENODEV;
1665
1666         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1667         if (!var_data) {
1668                 trace_array_put(tr);
1669                 return -ENOMEM;
1670         }
1671
1672         var_data->hist_data = hist_data;
1673         list_add(&var_data->list, &tr->hist_vars);
1674
1675         return 0;
1676 }
1677
1678 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1679 {
1680         struct trace_array *tr = hist_data->event_file->tr;
1681         struct hist_var_data *var_data;
1682
1683         var_data = find_hist_vars(hist_data);
1684         if (!var_data)
1685                 return;
1686
1687         if (WARN_ON(check_var_refs(hist_data)))
1688                 return;
1689
1690         list_del(&var_data->list);
1691
1692         kfree(var_data);
1693
1694         trace_array_put(tr);
1695 }
1696
1697 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1698                                          const char *var_name)
1699 {
1700         struct hist_field *hist_field, *found = NULL;
1701         int i;
1702
1703         for_each_hist_field(i, hist_data) {
1704                 hist_field = hist_data->fields[i];
1705                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1706                     strcmp(hist_field->var.name, var_name) == 0) {
1707                         found = hist_field;
1708                         break;
1709                 }
1710         }
1711
1712         return found;
1713 }
1714
1715 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1716                                    struct trace_event_file *file,
1717                                    const char *var_name)
1718 {
1719         struct hist_trigger_data *test_data;
1720         struct event_trigger_data *test;
1721         struct hist_field *hist_field;
1722
1723         hist_field = find_var_field(hist_data, var_name);
1724         if (hist_field)
1725                 return hist_field;
1726
1727         list_for_each_entry_rcu(test, &file->triggers, list) {
1728                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1729                         test_data = test->private_data;
1730                         hist_field = find_var_field(test_data, var_name);
1731                         if (hist_field)
1732                                 return hist_field;
1733                 }
1734         }
1735
1736         return NULL;
1737 }
1738
1739 static struct trace_event_file *find_var_file(struct trace_array *tr,
1740                                               char *system,
1741                                               char *event_name,
1742                                               char *var_name)
1743 {
1744         struct hist_trigger_data *var_hist_data;
1745         struct hist_var_data *var_data;
1746         struct trace_event_file *file, *found = NULL;
1747
1748         if (system)
1749                 return find_event_file(tr, system, event_name);
1750
1751         list_for_each_entry(var_data, &tr->hist_vars, list) {
1752                 var_hist_data = var_data->hist_data;
1753                 file = var_hist_data->event_file;
1754                 if (file == found)
1755                         continue;
1756
1757                 if (find_var_field(var_hist_data, var_name)) {
1758                         if (found) {
1759                                 hist_err(HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1760                                 return NULL;
1761                         }
1762
1763                         found = file;
1764                 }
1765         }
1766
1767         return found;
1768 }
1769
1770 static struct hist_field *find_file_var(struct trace_event_file *file,
1771                                         const char *var_name)
1772 {
1773         struct hist_trigger_data *test_data;
1774         struct event_trigger_data *test;
1775         struct hist_field *hist_field;
1776
1777         list_for_each_entry_rcu(test, &file->triggers, list) {
1778                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1779                         test_data = test->private_data;
1780                         hist_field = find_var_field(test_data, var_name);
1781                         if (hist_field)
1782                                 return hist_field;
1783                 }
1784         }
1785
1786         return NULL;
1787 }
1788
1789 static struct hist_field *
1790 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1791 {
1792         struct trace_array *tr = hist_data->event_file->tr;
1793         struct hist_field *hist_field, *found = NULL;
1794         struct trace_event_file *file;
1795         unsigned int i;
1796
1797         for (i = 0; i < hist_data->n_actions; i++) {
1798                 struct action_data *data = hist_data->actions[i];
1799
1800                 if (data->handler == HANDLER_ONMATCH) {
1801                         char *system = data->match_data.event_system;
1802                         char *event_name = data->match_data.event;
1803
1804                         file = find_var_file(tr, system, event_name, var_name);
1805                         if (!file)
1806                                 continue;
1807                         hist_field = find_file_var(file, var_name);
1808                         if (hist_field) {
1809                                 if (found) {
1810                                         hist_err(HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1811                                         return ERR_PTR(-EINVAL);
1812                                 }
1813
1814                                 found = hist_field;
1815                         }
1816                 }
1817         }
1818         return found;
1819 }
1820
1821 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1822                                          char *system,
1823                                          char *event_name,
1824                                          char *var_name)
1825 {
1826         struct trace_array *tr = hist_data->event_file->tr;
1827         struct hist_field *hist_field = NULL;
1828         struct trace_event_file *file;
1829
1830         if (!system || !event_name) {
1831                 hist_field = find_match_var(hist_data, var_name);
1832                 if (IS_ERR(hist_field))
1833                         return NULL;
1834                 if (hist_field)
1835                         return hist_field;
1836         }
1837
1838         file = find_var_file(tr, system, event_name, var_name);
1839         if (!file)
1840                 return NULL;
1841
1842         hist_field = find_file_var(file, var_name);
1843
1844         return hist_field;
1845 }
1846
1847 static u64 hist_field_var_ref(struct hist_field *hist_field,
1848                               struct tracing_map_elt *elt,
1849                               struct ring_buffer_event *rbe,
1850                               void *event)
1851 {
1852         struct hist_elt_data *elt_data;
1853         u64 var_val = 0;
1854
1855         elt_data = elt->private_data;
1856         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1857
1858         return var_val;
1859 }
1860
1861 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1862                              u64 *var_ref_vals, bool self)
1863 {
1864         struct hist_trigger_data *var_data;
1865         struct tracing_map_elt *var_elt;
1866         struct hist_field *hist_field;
1867         unsigned int i, var_idx;
1868         bool resolved = true;
1869         u64 var_val = 0;
1870
1871         for (i = 0; i < hist_data->n_var_refs; i++) {
1872                 hist_field = hist_data->var_refs[i];
1873                 var_idx = hist_field->var.idx;
1874                 var_data = hist_field->var.hist_data;
1875
1876                 if (var_data == NULL) {
1877                         resolved = false;
1878                         break;
1879                 }
1880
1881                 if ((self && var_data != hist_data) ||
1882                     (!self && var_data == hist_data))
1883                         continue;
1884
1885                 var_elt = tracing_map_lookup(var_data->map, key);
1886                 if (!var_elt) {
1887                         resolved = false;
1888                         break;
1889                 }
1890
1891                 if (!tracing_map_var_set(var_elt, var_idx)) {
1892                         resolved = false;
1893                         break;
1894                 }
1895
1896                 if (self || !hist_field->read_once)
1897                         var_val = tracing_map_read_var(var_elt, var_idx);
1898                 else
1899                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1900
1901                 var_ref_vals[i] = var_val;
1902         }
1903
1904         return resolved;
1905 }
1906
1907 static const char *hist_field_name(struct hist_field *field,
1908                                    unsigned int level)
1909 {
1910         const char *field_name = "";
1911
1912         if (level > 1)
1913                 return field_name;
1914
1915         if (field->field)
1916                 field_name = field->field->name;
1917         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1918                  field->flags & HIST_FIELD_FL_ALIAS)
1919                 field_name = hist_field_name(field->operands[0], ++level);
1920         else if (field->flags & HIST_FIELD_FL_CPU)
1921                 field_name = "cpu";
1922         else if (field->flags & HIST_FIELD_FL_EXPR ||
1923                  field->flags & HIST_FIELD_FL_VAR_REF) {
1924                 if (field->system) {
1925                         static char full_name[MAX_FILTER_STR_VAL];
1926
1927                         strcat(full_name, field->system);
1928                         strcat(full_name, ".");
1929                         strcat(full_name, field->event_name);
1930                         strcat(full_name, ".");
1931                         strcat(full_name, field->name);
1932                         field_name = full_name;
1933                 } else
1934                         field_name = field->name;
1935         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1936                 field_name = "common_timestamp";
1937
1938         if (field_name == NULL)
1939                 field_name = "";
1940
1941         return field_name;
1942 }
1943
1944 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1945 {
1946         hist_field_fn_t fn = NULL;
1947
1948         switch (field_size) {
1949         case 8:
1950                 if (field_is_signed)
1951                         fn = hist_field_s64;
1952                 else
1953                         fn = hist_field_u64;
1954                 break;
1955         case 4:
1956                 if (field_is_signed)
1957                         fn = hist_field_s32;
1958                 else
1959                         fn = hist_field_u32;
1960                 break;
1961         case 2:
1962                 if (field_is_signed)
1963                         fn = hist_field_s16;
1964                 else
1965                         fn = hist_field_u16;
1966                 break;
1967         case 1:
1968                 if (field_is_signed)
1969                         fn = hist_field_s8;
1970                 else
1971                         fn = hist_field_u8;
1972                 break;
1973         }
1974
1975         return fn;
1976 }
1977
1978 static int parse_map_size(char *str)
1979 {
1980         unsigned long size, map_bits;
1981         int ret;
1982
1983         strsep(&str, "=");
1984         if (!str) {
1985                 ret = -EINVAL;
1986                 goto out;
1987         }
1988
1989         ret = kstrtoul(str, 0, &size);
1990         if (ret)
1991                 goto out;
1992
1993         map_bits = ilog2(roundup_pow_of_two(size));
1994         if (map_bits < TRACING_MAP_BITS_MIN ||
1995             map_bits > TRACING_MAP_BITS_MAX)
1996                 ret = -EINVAL;
1997         else
1998                 ret = map_bits;
1999  out:
2000         return ret;
2001 }
2002
2003 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2004 {
2005         unsigned int i;
2006
2007         if (!attrs)
2008                 return;
2009
2010         for (i = 0; i < attrs->n_assignments; i++)
2011                 kfree(attrs->assignment_str[i]);
2012
2013         for (i = 0; i < attrs->n_actions; i++)
2014                 kfree(attrs->action_str[i]);
2015
2016         kfree(attrs->name);
2017         kfree(attrs->sort_key_str);
2018         kfree(attrs->keys_str);
2019         kfree(attrs->vals_str);
2020         kfree(attrs->clock);
2021         kfree(attrs);
2022 }
2023
2024 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2025 {
2026         int ret = -EINVAL;
2027
2028         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2029                 return ret;
2030
2031         if ((str_has_prefix(str, "onmatch(")) ||
2032             (str_has_prefix(str, "onmax(")) ||
2033             (str_has_prefix(str, "onchange("))) {
2034                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2035                 if (!attrs->action_str[attrs->n_actions]) {
2036                         ret = -ENOMEM;
2037                         return ret;
2038                 }
2039                 attrs->n_actions++;
2040                 ret = 0;
2041         }
2042         return ret;
2043 }
2044
2045 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
2046 {
2047         int ret = 0;
2048
2049         if ((str_has_prefix(str, "key=")) ||
2050             (str_has_prefix(str, "keys="))) {
2051                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
2052                 if (!attrs->keys_str) {
2053                         ret = -ENOMEM;
2054                         goto out;
2055                 }
2056         } else if ((str_has_prefix(str, "val=")) ||
2057                    (str_has_prefix(str, "vals=")) ||
2058                    (str_has_prefix(str, "values="))) {
2059                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
2060                 if (!attrs->vals_str) {
2061                         ret = -ENOMEM;
2062                         goto out;
2063                 }
2064         } else if (str_has_prefix(str, "sort=")) {
2065                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2066                 if (!attrs->sort_key_str) {
2067                         ret = -ENOMEM;
2068                         goto out;
2069                 }
2070         } else if (str_has_prefix(str, "name=")) {
2071                 attrs->name = kstrdup(str, GFP_KERNEL);
2072                 if (!attrs->name) {
2073                         ret = -ENOMEM;
2074                         goto out;
2075                 }
2076         } else if (str_has_prefix(str, "clock=")) {
2077                 strsep(&str, "=");
2078                 if (!str) {
2079                         ret = -EINVAL;
2080                         goto out;
2081                 }
2082
2083                 str = strstrip(str);
2084                 attrs->clock = kstrdup(str, GFP_KERNEL);
2085                 if (!attrs->clock) {
2086                         ret = -ENOMEM;
2087                         goto out;
2088                 }
2089         } else if (str_has_prefix(str, "size=")) {
2090                 int map_bits = parse_map_size(str);
2091
2092                 if (map_bits < 0) {
2093                         ret = map_bits;
2094                         goto out;
2095                 }
2096                 attrs->map_bits = map_bits;
2097         } else {
2098                 char *assignment;
2099
2100                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2101                         hist_err(HIST_ERR_TOO_MANY_VARS, errpos(str));
2102                         ret = -EINVAL;
2103                         goto out;
2104                 }
2105
2106                 assignment = kstrdup(str, GFP_KERNEL);
2107                 if (!assignment) {
2108                         ret = -ENOMEM;
2109                         goto out;
2110                 }
2111
2112                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2113         }
2114  out:
2115         return ret;
2116 }
2117
2118 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
2119 {
2120         struct hist_trigger_attrs *attrs;
2121         int ret = 0;
2122
2123         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2124         if (!attrs)
2125                 return ERR_PTR(-ENOMEM);
2126
2127         while (trigger_str) {
2128                 char *str = strsep(&trigger_str, ":");
2129
2130                 if (strchr(str, '=')) {
2131                         ret = parse_assignment(str, attrs);
2132                         if (ret)
2133                                 goto free;
2134                 } else if (strcmp(str, "pause") == 0)
2135                         attrs->pause = true;
2136                 else if ((strcmp(str, "cont") == 0) ||
2137                          (strcmp(str, "continue") == 0))
2138                         attrs->cont = true;
2139                 else if (strcmp(str, "clear") == 0)
2140                         attrs->clear = true;
2141                 else {
2142                         ret = parse_action(str, attrs);
2143                         if (ret)
2144                                 goto free;
2145                 }
2146         }
2147
2148         if (!attrs->keys_str) {
2149                 ret = -EINVAL;
2150                 goto free;
2151         }
2152
2153         if (!attrs->clock) {
2154                 attrs->clock = kstrdup("global", GFP_KERNEL);
2155                 if (!attrs->clock) {
2156                         ret = -ENOMEM;
2157                         goto free;
2158                 }
2159         }
2160
2161         return attrs;
2162  free:
2163         destroy_hist_trigger_attrs(attrs);
2164
2165         return ERR_PTR(ret);
2166 }
2167
2168 static inline void save_comm(char *comm, struct task_struct *task)
2169 {
2170         if (!task->pid) {
2171                 strcpy(comm, "<idle>");
2172                 return;
2173         }
2174
2175         if (WARN_ON_ONCE(task->pid < 0)) {
2176                 strcpy(comm, "<XXX>");
2177                 return;
2178         }
2179
2180         strncpy(comm, task->comm, TASK_COMM_LEN);
2181 }
2182
2183 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2184 {
2185         unsigned int i;
2186
2187         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2188                 kfree(elt_data->field_var_str[i]);
2189
2190         kfree(elt_data->comm);
2191         kfree(elt_data);
2192 }
2193
2194 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2195 {
2196         struct hist_elt_data *elt_data = elt->private_data;
2197
2198         hist_elt_data_free(elt_data);
2199 }
2200
2201 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2202 {
2203         struct hist_trigger_data *hist_data = elt->map->private_data;
2204         unsigned int size = TASK_COMM_LEN;
2205         struct hist_elt_data *elt_data;
2206         struct hist_field *key_field;
2207         unsigned int i, n_str;
2208
2209         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2210         if (!elt_data)
2211                 return -ENOMEM;
2212
2213         for_each_hist_key_field(i, hist_data) {
2214                 key_field = hist_data->fields[i];
2215
2216                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2217                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2218                         if (!elt_data->comm) {
2219                                 kfree(elt_data);
2220                                 return -ENOMEM;
2221                         }
2222                         break;
2223                 }
2224         }
2225
2226         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2227
2228         size = STR_VAR_LEN_MAX;
2229
2230         for (i = 0; i < n_str; i++) {
2231                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2232                 if (!elt_data->field_var_str[i]) {
2233                         hist_elt_data_free(elt_data);
2234                         return -ENOMEM;
2235                 }
2236         }
2237
2238         elt->private_data = elt_data;
2239
2240         return 0;
2241 }
2242
2243 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2244 {
2245         struct hist_elt_data *elt_data = elt->private_data;
2246
2247         if (elt_data->comm)
2248                 save_comm(elt_data->comm, current);
2249 }
2250
2251 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2252         .elt_alloc      = hist_trigger_elt_data_alloc,
2253         .elt_free       = hist_trigger_elt_data_free,
2254         .elt_init       = hist_trigger_elt_data_init,
2255 };
2256
2257 static const char *get_hist_field_flags(struct hist_field *hist_field)
2258 {
2259         const char *flags_str = NULL;
2260
2261         if (hist_field->flags & HIST_FIELD_FL_HEX)
2262                 flags_str = "hex";
2263         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2264                 flags_str = "sym";
2265         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2266                 flags_str = "sym-offset";
2267         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2268                 flags_str = "execname";
2269         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2270                 flags_str = "syscall";
2271         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2272                 flags_str = "log2";
2273         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2274                 flags_str = "usecs";
2275
2276         return flags_str;
2277 }
2278
2279 static void expr_field_str(struct hist_field *field, char *expr)
2280 {
2281         if (field->flags & HIST_FIELD_FL_VAR_REF)
2282                 strcat(expr, "$");
2283
2284         strcat(expr, hist_field_name(field, 0));
2285
2286         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2287                 const char *flags_str = get_hist_field_flags(field);
2288
2289                 if (flags_str) {
2290                         strcat(expr, ".");
2291                         strcat(expr, flags_str);
2292                 }
2293         }
2294 }
2295
2296 static char *expr_str(struct hist_field *field, unsigned int level)
2297 {
2298         char *expr;
2299
2300         if (level > 1)
2301                 return NULL;
2302
2303         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2304         if (!expr)
2305                 return NULL;
2306
2307         if (!field->operands[0]) {
2308                 expr_field_str(field, expr);
2309                 return expr;
2310         }
2311
2312         if (field->operator == FIELD_OP_UNARY_MINUS) {
2313                 char *subexpr;
2314
2315                 strcat(expr, "-(");
2316                 subexpr = expr_str(field->operands[0], ++level);
2317                 if (!subexpr) {
2318                         kfree(expr);
2319                         return NULL;
2320                 }
2321                 strcat(expr, subexpr);
2322                 strcat(expr, ")");
2323
2324                 kfree(subexpr);
2325
2326                 return expr;
2327         }
2328
2329         expr_field_str(field->operands[0], expr);
2330
2331         switch (field->operator) {
2332         case FIELD_OP_MINUS:
2333                 strcat(expr, "-");
2334                 break;
2335         case FIELD_OP_PLUS:
2336                 strcat(expr, "+");
2337                 break;
2338         default:
2339                 kfree(expr);
2340                 return NULL;
2341         }
2342
2343         expr_field_str(field->operands[1], expr);
2344
2345         return expr;
2346 }
2347
2348 static int contains_operator(char *str)
2349 {
2350         enum field_op_id field_op = FIELD_OP_NONE;
2351         char *op;
2352
2353         op = strpbrk(str, "+-");
2354         if (!op)
2355                 return FIELD_OP_NONE;
2356
2357         switch (*op) {
2358         case '-':
2359                 if (*str == '-')
2360                         field_op = FIELD_OP_UNARY_MINUS;
2361                 else
2362                         field_op = FIELD_OP_MINUS;
2363                 break;
2364         case '+':
2365                 field_op = FIELD_OP_PLUS;
2366                 break;
2367         default:
2368                 break;
2369         }
2370
2371         return field_op;
2372 }
2373
2374 static void __destroy_hist_field(struct hist_field *hist_field)
2375 {
2376         kfree(hist_field->var.name);
2377         kfree(hist_field->name);
2378         kfree(hist_field->type);
2379
2380         kfree(hist_field);
2381 }
2382
2383 static void destroy_hist_field(struct hist_field *hist_field,
2384                                unsigned int level)
2385 {
2386         unsigned int i;
2387
2388         if (level > 3)
2389                 return;
2390
2391         if (!hist_field)
2392                 return;
2393
2394         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2395                 return; /* var refs will be destroyed separately */
2396
2397         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2398                 destroy_hist_field(hist_field->operands[i], level + 1);
2399
2400         __destroy_hist_field(hist_field);
2401 }
2402
2403 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2404                                             struct ftrace_event_field *field,
2405                                             unsigned long flags,
2406                                             char *var_name)
2407 {
2408         struct hist_field *hist_field;
2409
2410         if (field && is_function_field(field))
2411                 return NULL;
2412
2413         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2414         if (!hist_field)
2415                 return NULL;
2416
2417         hist_field->hist_data = hist_data;
2418
2419         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2420                 goto out; /* caller will populate */
2421
2422         if (flags & HIST_FIELD_FL_VAR_REF) {
2423                 hist_field->fn = hist_field_var_ref;
2424                 goto out;
2425         }
2426
2427         if (flags & HIST_FIELD_FL_HITCOUNT) {
2428                 hist_field->fn = hist_field_counter;
2429                 hist_field->size = sizeof(u64);
2430                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2431                 if (!hist_field->type)
2432                         goto free;
2433                 goto out;
2434         }
2435
2436         if (flags & HIST_FIELD_FL_STACKTRACE) {
2437                 hist_field->fn = hist_field_none;
2438                 goto out;
2439         }
2440
2441         if (flags & HIST_FIELD_FL_LOG2) {
2442                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2443                 hist_field->fn = hist_field_log2;
2444                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2445                 hist_field->size = hist_field->operands[0]->size;
2446                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2447                 if (!hist_field->type)
2448                         goto free;
2449                 goto out;
2450         }
2451
2452         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2453                 hist_field->fn = hist_field_timestamp;
2454                 hist_field->size = sizeof(u64);
2455                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2456                 if (!hist_field->type)
2457                         goto free;
2458                 goto out;
2459         }
2460
2461         if (flags & HIST_FIELD_FL_CPU) {
2462                 hist_field->fn = hist_field_cpu;
2463                 hist_field->size = sizeof(int);
2464                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2465                 if (!hist_field->type)
2466                         goto free;
2467                 goto out;
2468         }
2469
2470         if (WARN_ON_ONCE(!field))
2471                 goto out;
2472
2473         if (is_string_field(field)) {
2474                 flags |= HIST_FIELD_FL_STRING;
2475
2476                 hist_field->size = MAX_FILTER_STR_VAL;
2477                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2478                 if (!hist_field->type)
2479                         goto free;
2480
2481                 if (field->filter_type == FILTER_STATIC_STRING)
2482                         hist_field->fn = hist_field_string;
2483                 else if (field->filter_type == FILTER_DYN_STRING)
2484                         hist_field->fn = hist_field_dynstring;
2485                 else
2486                         hist_field->fn = hist_field_pstring;
2487         } else {
2488                 hist_field->size = field->size;
2489                 hist_field->is_signed = field->is_signed;
2490                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2491                 if (!hist_field->type)
2492                         goto free;
2493
2494                 hist_field->fn = select_value_fn(field->size,
2495                                                  field->is_signed);
2496                 if (!hist_field->fn) {
2497                         destroy_hist_field(hist_field, 0);
2498                         return NULL;
2499                 }
2500         }
2501  out:
2502         hist_field->field = field;
2503         hist_field->flags = flags;
2504
2505         if (var_name) {
2506                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2507                 if (!hist_field->var.name)
2508                         goto free;
2509         }
2510
2511         return hist_field;
2512  free:
2513         destroy_hist_field(hist_field, 0);
2514         return NULL;
2515 }
2516
2517 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2518 {
2519         unsigned int i;
2520
2521         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2522                 if (hist_data->fields[i]) {
2523                         destroy_hist_field(hist_data->fields[i], 0);
2524                         hist_data->fields[i] = NULL;
2525                 }
2526         }
2527
2528         for (i = 0; i < hist_data->n_var_refs; i++) {
2529                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2530                 __destroy_hist_field(hist_data->var_refs[i]);
2531                 hist_data->var_refs[i] = NULL;
2532         }
2533 }
2534
2535 static int init_var_ref(struct hist_field *ref_field,
2536                         struct hist_field *var_field,
2537                         char *system, char *event_name)
2538 {
2539         int err = 0;
2540
2541         ref_field->var.idx = var_field->var.idx;
2542         ref_field->var.hist_data = var_field->hist_data;
2543         ref_field->size = var_field->size;
2544         ref_field->is_signed = var_field->is_signed;
2545         ref_field->flags |= var_field->flags &
2546                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2547
2548         if (system) {
2549                 ref_field->system = kstrdup(system, GFP_KERNEL);
2550                 if (!ref_field->system)
2551                         return -ENOMEM;
2552         }
2553
2554         if (event_name) {
2555                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2556                 if (!ref_field->event_name) {
2557                         err = -ENOMEM;
2558                         goto free;
2559                 }
2560         }
2561
2562         if (var_field->var.name) {
2563                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2564                 if (!ref_field->name) {
2565                         err = -ENOMEM;
2566                         goto free;
2567                 }
2568         } else if (var_field->name) {
2569                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2570                 if (!ref_field->name) {
2571                         err = -ENOMEM;
2572                         goto free;
2573                 }
2574         }
2575
2576         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2577         if (!ref_field->type) {
2578                 err = -ENOMEM;
2579                 goto free;
2580         }
2581  out:
2582         return err;
2583  free:
2584         kfree(ref_field->system);
2585         kfree(ref_field->event_name);
2586         kfree(ref_field->name);
2587
2588         goto out;
2589 }
2590
2591 /**
2592  * create_var_ref - Create a variable reference and attach it to trigger
2593  * @hist_data: The trigger that will be referencing the variable
2594  * @var_field: The VAR field to create a reference to
2595  * @system: The optional system string
2596  * @event_name: The optional event_name string
2597  *
2598  * Given a variable hist_field, create a VAR_REF hist_field that
2599  * represents a reference to it.
2600  *
2601  * This function also adds the reference to the trigger that
2602  * now references the variable.
2603  *
2604  * Return: The VAR_REF field if successful, NULL if not
2605  */
2606 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2607                                          struct hist_field *var_field,
2608                                          char *system, char *event_name)
2609 {
2610         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2611         struct hist_field *ref_field;
2612
2613         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2614         if (ref_field) {
2615                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2616                         destroy_hist_field(ref_field, 0);
2617                         return NULL;
2618                 }
2619
2620                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2621                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2622         }
2623
2624         return ref_field;
2625 }
2626
2627 static bool is_var_ref(char *var_name)
2628 {
2629         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2630                 return false;
2631
2632         return true;
2633 }
2634
2635 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2636                                  char *var_name)
2637 {
2638         char *name, *field;
2639         unsigned int i;
2640
2641         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2642                 name = hist_data->attrs->var_defs.name[i];
2643
2644                 if (strcmp(var_name, name) == 0) {
2645                         field = hist_data->attrs->var_defs.expr[i];
2646                         if (contains_operator(field) || is_var_ref(field))
2647                                 continue;
2648                         return field;
2649                 }
2650         }
2651
2652         return NULL;
2653 }
2654
2655 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2656                                  char *system, char *event_name,
2657                                  char *var_name)
2658 {
2659         struct trace_event_call *call;
2660
2661         if (system && event_name) {
2662                 call = hist_data->event_file->event_call;
2663
2664                 if (strcmp(system, call->class->system) != 0)
2665                         return NULL;
2666
2667                 if (strcmp(event_name, trace_event_name(call)) != 0)
2668                         return NULL;
2669         }
2670
2671         if (!!system != !!event_name)
2672                 return NULL;
2673
2674         if (!is_var_ref(var_name))
2675                 return NULL;
2676
2677         var_name++;
2678
2679         return field_name_from_var(hist_data, var_name);
2680 }
2681
2682 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2683                                         char *system, char *event_name,
2684                                         char *var_name)
2685 {
2686         struct hist_field *var_field = NULL, *ref_field = NULL;
2687
2688         if (!is_var_ref(var_name))
2689                 return NULL;
2690
2691         var_name++;
2692
2693         var_field = find_event_var(hist_data, system, event_name, var_name);
2694         if (var_field)
2695                 ref_field = create_var_ref(hist_data, var_field,
2696                                            system, event_name);
2697
2698         if (!ref_field)
2699                 hist_err(HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2700
2701         return ref_field;
2702 }
2703
2704 static struct ftrace_event_field *
2705 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2706             char *field_str, unsigned long *flags)
2707 {
2708         struct ftrace_event_field *field = NULL;
2709         char *field_name, *modifier, *str;
2710
2711         modifier = str = kstrdup(field_str, GFP_KERNEL);
2712         if (!modifier)
2713                 return ERR_PTR(-ENOMEM);
2714
2715         field_name = strsep(&modifier, ".");
2716         if (modifier) {
2717                 if (strcmp(modifier, "hex") == 0)
2718                         *flags |= HIST_FIELD_FL_HEX;
2719                 else if (strcmp(modifier, "sym") == 0)
2720                         *flags |= HIST_FIELD_FL_SYM;
2721                 else if (strcmp(modifier, "sym-offset") == 0)
2722                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2723                 else if ((strcmp(modifier, "execname") == 0) &&
2724                          (strcmp(field_name, "common_pid") == 0))
2725                         *flags |= HIST_FIELD_FL_EXECNAME;
2726                 else if (strcmp(modifier, "syscall") == 0)
2727                         *flags |= HIST_FIELD_FL_SYSCALL;
2728                 else if (strcmp(modifier, "log2") == 0)
2729                         *flags |= HIST_FIELD_FL_LOG2;
2730                 else if (strcmp(modifier, "usecs") == 0)
2731                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2732                 else {
2733                         hist_err(HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2734                         field = ERR_PTR(-EINVAL);
2735                         goto out;
2736                 }
2737         }
2738
2739         if (strcmp(field_name, "common_timestamp") == 0) {
2740                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2741                 hist_data->enable_timestamps = true;
2742                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2743                         hist_data->attrs->ts_in_usecs = true;
2744         } else if (strcmp(field_name, "cpu") == 0)
2745                 *flags |= HIST_FIELD_FL_CPU;
2746         else {
2747                 field = trace_find_event_field(file->event_call, field_name);
2748                 if (!field || !field->size) {
2749                         hist_err(HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2750                         field = ERR_PTR(-EINVAL);
2751                         goto out;
2752                 }
2753         }
2754  out:
2755         kfree(str);
2756
2757         return field;
2758 }
2759
2760 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2761                                        struct hist_field *var_ref,
2762                                        char *var_name)
2763 {
2764         struct hist_field *alias = NULL;
2765         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2766
2767         alias = create_hist_field(hist_data, NULL, flags, var_name);
2768         if (!alias)
2769                 return NULL;
2770
2771         alias->fn = var_ref->fn;
2772         alias->operands[0] = var_ref;
2773
2774         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2775                 destroy_hist_field(alias, 0);
2776                 return NULL;
2777         }
2778
2779         return alias;
2780 }
2781
2782 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2783                                      struct trace_event_file *file, char *str,
2784                                      unsigned long *flags, char *var_name)
2785 {
2786         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2787         struct ftrace_event_field *field = NULL;
2788         struct hist_field *hist_field = NULL;
2789         int ret = 0;
2790
2791         s = strchr(str, '.');
2792         if (s) {
2793                 s = strchr(++s, '.');
2794                 if (s) {
2795                         ref_system = strsep(&str, ".");
2796                         if (!str) {
2797                                 ret = -EINVAL;
2798                                 goto out;
2799                         }
2800                         ref_event = strsep(&str, ".");
2801                         if (!str) {
2802                                 ret = -EINVAL;
2803                                 goto out;
2804                         }
2805                         ref_var = str;
2806                 }
2807         }
2808
2809         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2810         if (!s) {
2811                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2812                 if (hist_field) {
2813                         if (var_name) {
2814                                 hist_field = create_alias(hist_data, hist_field, var_name);
2815                                 if (!hist_field) {
2816                                         ret = -ENOMEM;
2817                                         goto out;
2818                                 }
2819                         }
2820                         return hist_field;
2821                 }
2822         } else
2823                 str = s;
2824
2825         field = parse_field(hist_data, file, str, flags);
2826         if (IS_ERR(field)) {
2827                 ret = PTR_ERR(field);
2828                 goto out;
2829         }
2830
2831         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2832         if (!hist_field) {
2833                 ret = -ENOMEM;
2834                 goto out;
2835         }
2836
2837         return hist_field;
2838  out:
2839         return ERR_PTR(ret);
2840 }
2841
2842 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2843                                      struct trace_event_file *file,
2844                                      char *str, unsigned long flags,
2845                                      char *var_name, unsigned int level);
2846
2847 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2848                                       struct trace_event_file *file,
2849                                       char *str, unsigned long flags,
2850                                       char *var_name, unsigned int level)
2851 {
2852         struct hist_field *operand1, *expr = NULL;
2853         unsigned long operand_flags;
2854         int ret = 0;
2855         char *s;
2856
2857         /* we support only -(xxx) i.e. explicit parens required */
2858
2859         if (level > 3) {
2860                 hist_err(HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2861                 ret = -EINVAL;
2862                 goto free;
2863         }
2864
2865         str++; /* skip leading '-' */
2866
2867         s = strchr(str, '(');
2868         if (s)
2869                 str++;
2870         else {
2871                 ret = -EINVAL;
2872                 goto free;
2873         }
2874
2875         s = strrchr(str, ')');
2876         if (s)
2877                 *s = '\0';
2878         else {
2879                 ret = -EINVAL; /* no closing ')' */
2880                 goto free;
2881         }
2882
2883         flags |= HIST_FIELD_FL_EXPR;
2884         expr = create_hist_field(hist_data, NULL, flags, var_name);
2885         if (!expr) {
2886                 ret = -ENOMEM;
2887                 goto free;
2888         }
2889
2890         operand_flags = 0;
2891         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2892         if (IS_ERR(operand1)) {
2893                 ret = PTR_ERR(operand1);
2894                 goto free;
2895         }
2896
2897         expr->flags |= operand1->flags &
2898                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2899         expr->fn = hist_field_unary_minus;
2900         expr->operands[0] = operand1;
2901         expr->operator = FIELD_OP_UNARY_MINUS;
2902         expr->name = expr_str(expr, 0);
2903         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2904         if (!expr->type) {
2905                 ret = -ENOMEM;
2906                 goto free;
2907         }
2908
2909         return expr;
2910  free:
2911         destroy_hist_field(expr, 0);
2912         return ERR_PTR(ret);
2913 }
2914
2915 static int check_expr_operands(struct hist_field *operand1,
2916                                struct hist_field *operand2)
2917 {
2918         unsigned long operand1_flags = operand1->flags;
2919         unsigned long operand2_flags = operand2->flags;
2920
2921         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2922             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2923                 struct hist_field *var;
2924
2925                 var = find_var_field(operand1->var.hist_data, operand1->name);
2926                 if (!var)
2927                         return -EINVAL;
2928                 operand1_flags = var->flags;
2929         }
2930
2931         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2932             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2933                 struct hist_field *var;
2934
2935                 var = find_var_field(operand2->var.hist_data, operand2->name);
2936                 if (!var)
2937                         return -EINVAL;
2938                 operand2_flags = var->flags;
2939         }
2940
2941         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2942             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2943                 hist_err(HIST_ERR_TIMESTAMP_MISMATCH, 0);
2944                 return -EINVAL;
2945         }
2946
2947         return 0;
2948 }
2949
2950 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2951                                      struct trace_event_file *file,
2952                                      char *str, unsigned long flags,
2953                                      char *var_name, unsigned int level)
2954 {
2955         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2956         unsigned long operand_flags;
2957         int field_op, ret = -EINVAL;
2958         char *sep, *operand1_str;
2959
2960         if (level > 3) {
2961                 hist_err(HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2962                 return ERR_PTR(-EINVAL);
2963         }
2964
2965         field_op = contains_operator(str);
2966
2967         if (field_op == FIELD_OP_NONE)
2968                 return parse_atom(hist_data, file, str, &flags, var_name);
2969
2970         if (field_op == FIELD_OP_UNARY_MINUS)
2971                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2972
2973         switch (field_op) {
2974         case FIELD_OP_MINUS:
2975                 sep = "-";
2976                 break;
2977         case FIELD_OP_PLUS:
2978                 sep = "+";
2979                 break;
2980         default:
2981                 goto free;
2982         }
2983
2984         operand1_str = strsep(&str, sep);
2985         if (!operand1_str || !str)
2986                 goto free;
2987
2988         operand_flags = 0;
2989         operand1 = parse_atom(hist_data, file, operand1_str,
2990                               &operand_flags, NULL);
2991         if (IS_ERR(operand1)) {
2992                 ret = PTR_ERR(operand1);
2993                 operand1 = NULL;
2994                 goto free;
2995         }
2996
2997         /* rest of string could be another expression e.g. b+c in a+b+c */
2998         operand_flags = 0;
2999         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3000         if (IS_ERR(operand2)) {
3001                 ret = PTR_ERR(operand2);
3002                 operand2 = NULL;
3003                 goto free;
3004         }
3005
3006         ret = check_expr_operands(operand1, operand2);
3007         if (ret)
3008                 goto free;
3009
3010         flags |= HIST_FIELD_FL_EXPR;
3011
3012         flags |= operand1->flags &
3013                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3014
3015         expr = create_hist_field(hist_data, NULL, flags, var_name);
3016         if (!expr) {
3017                 ret = -ENOMEM;
3018                 goto free;
3019         }
3020
3021         operand1->read_once = true;
3022         operand2->read_once = true;
3023
3024         expr->operands[0] = operand1;
3025         expr->operands[1] = operand2;
3026         expr->operator = field_op;
3027         expr->name = expr_str(expr, 0);
3028         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3029         if (!expr->type) {
3030                 ret = -ENOMEM;
3031                 goto free;
3032         }
3033
3034         switch (field_op) {
3035         case FIELD_OP_MINUS:
3036                 expr->fn = hist_field_minus;
3037                 break;
3038         case FIELD_OP_PLUS:
3039                 expr->fn = hist_field_plus;
3040                 break;
3041         default:
3042                 ret = -EINVAL;
3043                 goto free;
3044         }
3045
3046         return expr;
3047  free:
3048         destroy_hist_field(operand1, 0);
3049         destroy_hist_field(operand2, 0);
3050         destroy_hist_field(expr, 0);
3051
3052         return ERR_PTR(ret);
3053 }
3054
3055 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3056                                  struct trace_event_file *file)
3057 {
3058         struct event_trigger_data *test;
3059
3060         list_for_each_entry_rcu(test, &file->triggers, list) {
3061                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3062                         if (test->private_data == hist_data)
3063                                 return test->filter_str;
3064                 }
3065         }
3066
3067         return NULL;
3068 }
3069
3070 static struct event_command trigger_hist_cmd;
3071 static int event_hist_trigger_func(struct event_command *cmd_ops,
3072                                    struct trace_event_file *file,
3073                                    char *glob, char *cmd, char *param);
3074
3075 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3076                             struct hist_trigger_data *hist_data,
3077                             unsigned int n_keys)
3078 {
3079         struct hist_field *target_hist_field, *hist_field;
3080         unsigned int n, i, j;
3081
3082         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3083                 return false;
3084
3085         i = hist_data->n_vals;
3086         j = target_hist_data->n_vals;
3087
3088         for (n = 0; n < n_keys; n++) {
3089                 hist_field = hist_data->fields[i + n];
3090                 target_hist_field = target_hist_data->fields[j + n];
3091
3092                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3093                         return false;
3094                 if (hist_field->size != target_hist_field->size)
3095                         return false;
3096                 if (hist_field->is_signed != target_hist_field->is_signed)
3097                         return false;
3098         }
3099
3100         return true;
3101 }
3102
3103 static struct hist_trigger_data *
3104 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3105                      struct trace_event_file *file)
3106 {
3107         struct hist_trigger_data *hist_data;
3108         struct event_trigger_data *test;
3109         unsigned int n_keys;
3110
3111         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3112
3113         list_for_each_entry_rcu(test, &file->triggers, list) {
3114                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3115                         hist_data = test->private_data;
3116
3117                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3118                                 return hist_data;
3119                 }
3120         }
3121
3122         return NULL;
3123 }
3124
3125 static struct trace_event_file *event_file(struct trace_array *tr,
3126                                            char *system, char *event_name)
3127 {
3128         struct trace_event_file *file;
3129
3130         file = __find_event_file(tr, system, event_name);
3131         if (!file)
3132                 return ERR_PTR(-EINVAL);
3133
3134         return file;
3135 }
3136
3137 static struct hist_field *
3138 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3139                          char *system, char *event_name, char *field_name)
3140 {
3141         struct hist_field *event_var;
3142         char *synthetic_name;
3143
3144         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3145         if (!synthetic_name)
3146                 return ERR_PTR(-ENOMEM);
3147
3148         strcpy(synthetic_name, "synthetic_");
3149         strcat(synthetic_name, field_name);
3150
3151         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3152
3153         kfree(synthetic_name);
3154
3155         return event_var;
3156 }
3157
3158 /**
3159  * create_field_var_hist - Automatically create a histogram and var for a field
3160  * @target_hist_data: The target hist trigger
3161  * @subsys_name: Optional subsystem name
3162  * @event_name: Optional event name
3163  * @field_name: The name of the field (and the resulting variable)
3164  *
3165  * Hist trigger actions fetch data from variables, not directly from
3166  * events.  However, for convenience, users are allowed to directly
3167  * specify an event field in an action, which will be automatically
3168  * converted into a variable on their behalf.
3169
3170  * If a user specifies a field on an event that isn't the event the
3171  * histogram currently being defined (the target event histogram), the
3172  * only way that can be accomplished is if a new hist trigger is
3173  * created and the field variable defined on that.
3174  *
3175  * This function creates a new histogram compatible with the target
3176  * event (meaning a histogram with the same key as the target
3177  * histogram), and creates a variable for the specified field, but
3178  * with 'synthetic_' prepended to the variable name in order to avoid
3179  * collision with normal field variables.
3180  *
3181  * Return: The variable created for the field.
3182  */
3183 static struct hist_field *
3184 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3185                       char *subsys_name, char *event_name, char *field_name)
3186 {
3187         struct trace_array *tr = target_hist_data->event_file->tr;
3188         struct hist_field *event_var = ERR_PTR(-EINVAL);
3189         struct hist_trigger_data *hist_data;
3190         unsigned int i, n, first = true;
3191         struct field_var_hist *var_hist;
3192         struct trace_event_file *file;
3193         struct hist_field *key_field;
3194         char *saved_filter;
3195         char *cmd;
3196         int ret;
3197
3198         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3199                 hist_err(HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3200                 return ERR_PTR(-EINVAL);
3201         }
3202
3203         file = event_file(tr, subsys_name, event_name);
3204
3205         if (IS_ERR(file)) {
3206                 hist_err(HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3207                 ret = PTR_ERR(file);
3208                 return ERR_PTR(ret);
3209         }
3210
3211         /*
3212          * Look for a histogram compatible with target.  We'll use the
3213          * found histogram specification to create a new matching
3214          * histogram with our variable on it.  target_hist_data is not
3215          * yet a registered histogram so we can't use that.
3216          */
3217         hist_data = find_compatible_hist(target_hist_data, file);
3218         if (!hist_data) {
3219                 hist_err(HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3220                 return ERR_PTR(-EINVAL);
3221         }
3222
3223         /* See if a synthetic field variable has already been created */
3224         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3225                                              event_name, field_name);
3226         if (!IS_ERR_OR_NULL(event_var))
3227                 return event_var;
3228
3229         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3230         if (!var_hist)
3231                 return ERR_PTR(-ENOMEM);
3232
3233         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3234         if (!cmd) {
3235                 kfree(var_hist);
3236                 return ERR_PTR(-ENOMEM);
3237         }
3238
3239         /* Use the same keys as the compatible histogram */
3240         strcat(cmd, "keys=");
3241
3242         for_each_hist_key_field(i, hist_data) {
3243                 key_field = hist_data->fields[i];
3244                 if (!first)
3245                         strcat(cmd, ",");
3246                 strcat(cmd, key_field->field->name);
3247                 first = false;
3248         }
3249
3250         /* Create the synthetic field variable specification */
3251         strcat(cmd, ":synthetic_");
3252         strcat(cmd, field_name);
3253         strcat(cmd, "=");
3254         strcat(cmd, field_name);
3255
3256         /* Use the same filter as the compatible histogram */
3257         saved_filter = find_trigger_filter(hist_data, file);
3258         if (saved_filter) {
3259                 strcat(cmd, " if ");
3260                 strcat(cmd, saved_filter);
3261         }
3262
3263         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3264         if (!var_hist->cmd) {
3265                 kfree(cmd);
3266                 kfree(var_hist);
3267                 return ERR_PTR(-ENOMEM);
3268         }
3269
3270         /* Save the compatible histogram information */
3271         var_hist->hist_data = hist_data;
3272
3273         /* Create the new histogram with our variable */
3274         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3275                                       "", "hist", cmd);
3276         if (ret) {
3277                 kfree(cmd);
3278                 kfree(var_hist->cmd);
3279                 kfree(var_hist);
3280                 hist_err(HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3281                 return ERR_PTR(ret);
3282         }
3283
3284         kfree(cmd);
3285
3286         /* If we can't find the variable, something went wrong */
3287         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3288                                              event_name, field_name);
3289         if (IS_ERR_OR_NULL(event_var)) {
3290                 kfree(var_hist->cmd);
3291                 kfree(var_hist);
3292                 hist_err(HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3293                 return ERR_PTR(-EINVAL);
3294         }
3295
3296         n = target_hist_data->n_field_var_hists;
3297         target_hist_data->field_var_hists[n] = var_hist;
3298         target_hist_data->n_field_var_hists++;
3299
3300         return event_var;
3301 }
3302
3303 static struct hist_field *
3304 find_target_event_var(struct hist_trigger_data *hist_data,
3305                       char *subsys_name, char *event_name, char *var_name)
3306 {
3307         struct trace_event_file *file = hist_data->event_file;
3308         struct hist_field *hist_field = NULL;
3309
3310         if (subsys_name) {
3311                 struct trace_event_call *call;
3312
3313                 if (!event_name)
3314                         return NULL;
3315
3316                 call = file->event_call;
3317
3318                 if (strcmp(subsys_name, call->class->system) != 0)
3319                         return NULL;
3320
3321                 if (strcmp(event_name, trace_event_name(call)) != 0)
3322                         return NULL;
3323         }
3324
3325         hist_field = find_var_field(hist_data, var_name);
3326
3327         return hist_field;
3328 }
3329
3330 static inline void __update_field_vars(struct tracing_map_elt *elt,
3331                                        struct ring_buffer_event *rbe,
3332                                        void *rec,
3333                                        struct field_var **field_vars,
3334                                        unsigned int n_field_vars,
3335                                        unsigned int field_var_str_start)
3336 {
3337         struct hist_elt_data *elt_data = elt->private_data;
3338         unsigned int i, j, var_idx;
3339         u64 var_val;
3340
3341         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3342                 struct field_var *field_var = field_vars[i];
3343                 struct hist_field *var = field_var->var;
3344                 struct hist_field *val = field_var->val;
3345
3346                 var_val = val->fn(val, elt, rbe, rec);
3347                 var_idx = var->var.idx;
3348
3349                 if (val->flags & HIST_FIELD_FL_STRING) {
3350                         char *str = elt_data->field_var_str[j++];
3351                         char *val_str = (char *)(uintptr_t)var_val;
3352
3353                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3354                         var_val = (u64)(uintptr_t)str;
3355                 }
3356                 tracing_map_set_var(elt, var_idx, var_val);
3357         }
3358 }
3359
3360 static void update_field_vars(struct hist_trigger_data *hist_data,
3361                               struct tracing_map_elt *elt,
3362                               struct ring_buffer_event *rbe,
3363                               void *rec)
3364 {
3365         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3366                             hist_data->n_field_vars, 0);
3367 }
3368
3369 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3370                                  struct tracing_map_elt *elt, void *rec,
3371                                  struct ring_buffer_event *rbe, void *key,
3372                                  struct action_data *data, u64 *var_ref_vals)
3373 {
3374         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3375                             hist_data->n_save_vars, hist_data->n_field_var_str);
3376 }
3377
3378 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3379                                      struct trace_event_file *file,
3380                                      char *name, int size, const char *type)
3381 {
3382         struct hist_field *var;
3383         int idx;
3384
3385         if (find_var(hist_data, file, name) && !hist_data->remove) {
3386                 var = ERR_PTR(-EINVAL);
3387                 goto out;
3388         }
3389
3390         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3391         if (!var) {
3392                 var = ERR_PTR(-ENOMEM);
3393                 goto out;
3394         }
3395
3396         idx = tracing_map_add_var(hist_data->map);
3397         if (idx < 0) {
3398                 kfree(var);
3399                 var = ERR_PTR(-EINVAL);
3400                 goto out;
3401         }
3402
3403         var->flags = HIST_FIELD_FL_VAR;
3404         var->var.idx = idx;
3405         var->var.hist_data = var->hist_data = hist_data;
3406         var->size = size;
3407         var->var.name = kstrdup(name, GFP_KERNEL);
3408         var->type = kstrdup(type, GFP_KERNEL);
3409         if (!var->var.name || !var->type) {
3410                 kfree(var->var.name);
3411                 kfree(var->type);
3412                 kfree(var);
3413                 var = ERR_PTR(-ENOMEM);
3414         }
3415  out:
3416         return var;
3417 }
3418
3419 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3420                                           struct trace_event_file *file,
3421                                           char *field_name)
3422 {
3423         struct hist_field *val = NULL, *var = NULL;
3424         unsigned long flags = HIST_FIELD_FL_VAR;
3425         struct field_var *field_var;
3426         int ret = 0;
3427
3428         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3429                 hist_err(HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3430                 ret = -EINVAL;
3431                 goto err;
3432         }
3433
3434         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3435         if (IS_ERR(val)) {
3436                 hist_err(HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3437                 ret = PTR_ERR(val);
3438                 goto err;
3439         }
3440
3441         var = create_var(hist_data, file, field_name, val->size, val->type);
3442         if (IS_ERR(var)) {
3443                 hist_err(HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3444                 kfree(val);
3445                 ret = PTR_ERR(var);
3446                 goto err;
3447         }
3448
3449         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3450         if (!field_var) {
3451                 kfree(val);
3452                 kfree(var);
3453                 ret =  -ENOMEM;
3454                 goto err;
3455         }
3456
3457         field_var->var = var;
3458         field_var->val = val;
3459  out:
3460         return field_var;
3461  err:
3462         field_var = ERR_PTR(ret);
3463         goto out;
3464 }
3465
3466 /**
3467  * create_target_field_var - Automatically create a variable for a field
3468  * @target_hist_data: The target hist trigger
3469  * @subsys_name: Optional subsystem name
3470  * @event_name: Optional event name
3471  * @var_name: The name of the field (and the resulting variable)
3472  *
3473  * Hist trigger actions fetch data from variables, not directly from
3474  * events.  However, for convenience, users are allowed to directly
3475  * specify an event field in an action, which will be automatically
3476  * converted into a variable on their behalf.
3477
3478  * This function creates a field variable with the name var_name on
3479  * the hist trigger currently being defined on the target event.  If
3480  * subsys_name and event_name are specified, this function simply
3481  * verifies that they do in fact match the target event subsystem and
3482  * event name.
3483  *
3484  * Return: The variable created for the field.
3485  */
3486 static struct field_var *
3487 create_target_field_var(struct hist_trigger_data *target_hist_data,
3488                         char *subsys_name, char *event_name, char *var_name)
3489 {
3490         struct trace_event_file *file = target_hist_data->event_file;
3491
3492         if (subsys_name) {
3493                 struct trace_event_call *call;
3494
3495                 if (!event_name)
3496                         return NULL;
3497
3498                 call = file->event_call;
3499
3500                 if (strcmp(subsys_name, call->class->system) != 0)
3501                         return NULL;
3502
3503                 if (strcmp(event_name, trace_event_name(call)) != 0)
3504                         return NULL;
3505         }
3506
3507         return create_field_var(target_hist_data, file, var_name);
3508 }
3509
3510 static bool check_track_val_max(u64 track_val, u64 var_val)
3511 {
3512         if (var_val <= track_val)
3513                 return false;
3514
3515         return true;
3516 }
3517
3518 static bool check_track_val_changed(u64 track_val, u64 var_val)
3519 {
3520         if (var_val == track_val)
3521                 return false;
3522
3523         return true;
3524 }
3525
3526 static u64 get_track_val(struct hist_trigger_data *hist_data,
3527                          struct tracing_map_elt *elt,
3528                          struct action_data *data)
3529 {
3530         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3531         u64 track_val;
3532
3533         track_val = tracing_map_read_var(elt, track_var_idx);
3534
3535         return track_val;
3536 }
3537
3538 static void save_track_val(struct hist_trigger_data *hist_data,
3539                            struct tracing_map_elt *elt,
3540                            struct action_data *data, u64 var_val)
3541 {
3542         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3543
3544         tracing_map_set_var(elt, track_var_idx, var_val);
3545 }
3546
3547 static void save_track_data(struct hist_trigger_data *hist_data,
3548                             struct tracing_map_elt *elt, void *rec,
3549                             struct ring_buffer_event *rbe, void *key,
3550                             struct action_data *data, u64 *var_ref_vals)
3551 {
3552         if (data->track_data.save_data)
3553                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3554 }
3555
3556 static bool check_track_val(struct tracing_map_elt *elt,
3557                             struct action_data *data,
3558                             u64 var_val)
3559 {
3560         struct hist_trigger_data *hist_data;
3561         u64 track_val;
3562
3563         hist_data = data->track_data.track_var->hist_data;
3564         track_val = get_track_val(hist_data, elt, data);
3565
3566         return data->track_data.check_val(track_val, var_val);
3567 }
3568
3569 #ifdef CONFIG_TRACER_SNAPSHOT
3570 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3571 {
3572         /* called with tr->max_lock held */
3573         struct track_data *track_data = tr->cond_snapshot->cond_data;
3574         struct hist_elt_data *elt_data, *track_elt_data;
3575         struct snapshot_context *context = cond_data;
3576         u64 track_val;
3577
3578         if (!track_data)
3579                 return false;
3580
3581         track_val = get_track_val(track_data->hist_data, context->elt,
3582                                   track_data->action_data);
3583
3584         track_data->track_val = track_val;
3585         memcpy(track_data->key, context->key, track_data->key_len);
3586
3587         elt_data = context->elt->private_data;
3588         track_elt_data = track_data->elt.private_data;
3589         if (elt_data->comm)
3590                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3591
3592         track_data->updated = true;
3593
3594         return true;
3595 }
3596
3597 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3598                                      struct tracing_map_elt *elt, void *rec,
3599                                      struct ring_buffer_event *rbe, void *key,
3600                                      struct action_data *data,
3601                                      u64 *var_ref_vals)
3602 {
3603         struct trace_event_file *file = hist_data->event_file;
3604         struct snapshot_context context;
3605
3606         context.elt = elt;
3607         context.key = key;
3608
3609         tracing_snapshot_cond(file->tr, &context);
3610 }
3611
3612 static void hist_trigger_print_key(struct seq_file *m,
3613                                    struct hist_trigger_data *hist_data,
3614                                    void *key,
3615                                    struct tracing_map_elt *elt);
3616
3617 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3618 {
3619         unsigned int i;
3620
3621         if (!hist_data->n_actions)
3622                 return NULL;
3623
3624         for (i = 0; i < hist_data->n_actions; i++) {
3625                 struct action_data *data = hist_data->actions[i];
3626
3627                 if (data->action == ACTION_SNAPSHOT)
3628                         return data;
3629         }
3630
3631         return NULL;
3632 }
3633
3634 static void track_data_snapshot_print(struct seq_file *m,
3635                                       struct hist_trigger_data *hist_data)
3636 {
3637         struct trace_event_file *file = hist_data->event_file;
3638         struct track_data *track_data;
3639         struct action_data *action;
3640
3641         track_data = tracing_cond_snapshot_data(file->tr);
3642         if (!track_data)
3643                 return;
3644
3645         if (!track_data->updated)
3646                 return;
3647
3648         action = snapshot_action(hist_data);
3649         if (!action)
3650                 return;
3651
3652         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3653         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3654                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3655                    action->track_data.var_str, track_data->track_val);
3656
3657         seq_puts(m, "\ttriggered by event with key: ");
3658         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3659         seq_putc(m, '\n');
3660 }
3661 #else
3662 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3663 {
3664         return false;
3665 }
3666 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3667                                      struct tracing_map_elt *elt, void *rec,
3668                                      struct ring_buffer_event *rbe, void *key,
3669                                      struct action_data *data,
3670                                      u64 *var_ref_vals) {}
3671 static void track_data_snapshot_print(struct seq_file *m,
3672                                       struct hist_trigger_data *hist_data) {}
3673 #endif /* CONFIG_TRACER_SNAPSHOT */
3674
3675 static void track_data_print(struct seq_file *m,
3676                              struct hist_trigger_data *hist_data,
3677                              struct tracing_map_elt *elt,
3678                              struct action_data *data)
3679 {
3680         u64 track_val = get_track_val(hist_data, elt, data);
3681         unsigned int i, save_var_idx;
3682
3683         if (data->handler == HANDLER_ONMAX)
3684                 seq_printf(m, "\n\tmax: %10llu", track_val);
3685         else if (data->handler == HANDLER_ONCHANGE)
3686                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3687
3688         if (data->action == ACTION_SNAPSHOT)
3689                 return;
3690
3691         for (i = 0; i < hist_data->n_save_vars; i++) {
3692                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3693                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3694                 u64 val;
3695
3696                 save_var_idx = save_var->var.idx;
3697
3698                 val = tracing_map_read_var(elt, save_var_idx);
3699
3700                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3701                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3702                                    (char *)(uintptr_t)(val));
3703                 } else
3704                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3705         }
3706 }
3707
3708 static void ontrack_action(struct hist_trigger_data *hist_data,
3709                            struct tracing_map_elt *elt, void *rec,
3710                            struct ring_buffer_event *rbe, void *key,
3711                            struct action_data *data, u64 *var_ref_vals)
3712 {
3713         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3714
3715         if (check_track_val(elt, data, var_val)) {
3716                 save_track_val(hist_data, elt, data, var_val);
3717                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3718         }
3719 }
3720
3721 static void action_data_destroy(struct action_data *data)
3722 {
3723         unsigned int i;
3724
3725         lockdep_assert_held(&event_mutex);
3726
3727         kfree(data->action_name);
3728
3729         for (i = 0; i < data->n_params; i++)
3730                 kfree(data->params[i]);
3731
3732         if (data->synth_event)
3733                 data->synth_event->ref--;
3734
3735         kfree(data->synth_event_name);
3736
3737         kfree(data);
3738 }
3739
3740 static void track_data_destroy(struct hist_trigger_data *hist_data,
3741                                struct action_data *data)
3742 {
3743         struct trace_event_file *file = hist_data->event_file;
3744
3745         destroy_hist_field(data->track_data.track_var, 0);
3746
3747         if (data->action == ACTION_SNAPSHOT) {
3748                 struct track_data *track_data;
3749
3750                 track_data = tracing_cond_snapshot_data(file->tr);
3751                 if (track_data && track_data->hist_data == hist_data) {
3752                         tracing_snapshot_cond_disable(file->tr);
3753                         track_data_free(track_data);
3754                 }
3755         }
3756
3757         kfree(data->track_data.var_str);
3758
3759         action_data_destroy(data);
3760 }
3761
3762 static int action_create(struct hist_trigger_data *hist_data,
3763                          struct action_data *data);
3764
3765 static int track_data_create(struct hist_trigger_data *hist_data,
3766                              struct action_data *data)
3767 {
3768         struct hist_field *var_field, *ref_field, *track_var = NULL;
3769         struct trace_event_file *file = hist_data->event_file;
3770         char *track_data_var_str;
3771         int ret = 0;
3772
3773         track_data_var_str = data->track_data.var_str;
3774         if (track_data_var_str[0] != '$') {
3775                 hist_err(HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3776                 return -EINVAL;
3777         }
3778         track_data_var_str++;
3779
3780         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3781         if (!var_field) {
3782                 hist_err(HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3783                 return -EINVAL;
3784         }
3785
3786         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3787         if (!ref_field)
3788                 return -ENOMEM;
3789
3790         data->track_data.var_ref = ref_field;
3791
3792         if (data->handler == HANDLER_ONMAX)
3793                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3794         if (IS_ERR(track_var)) {
3795                 hist_err(HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3796                 ret = PTR_ERR(track_var);
3797                 goto out;
3798         }
3799
3800         if (data->handler == HANDLER_ONCHANGE)
3801                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3802         if (IS_ERR(track_var)) {
3803                 hist_err(HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3804                 ret = PTR_ERR(track_var);
3805                 goto out;
3806         }
3807         data->track_data.track_var = track_var;
3808
3809         ret = action_create(hist_data, data);
3810  out:
3811         return ret;
3812 }
3813
3814 static int parse_action_params(char *params, struct action_data *data)
3815 {
3816         char *param, *saved_param;
3817         bool first_param = true;
3818         int ret = 0;
3819
3820         while (params) {
3821                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3822                         hist_err(HIST_ERR_TOO_MANY_PARAMS, 0);
3823                         goto out;
3824                 }
3825
3826                 param = strsep(&params, ",");
3827                 if (!param) {
3828                         hist_err(HIST_ERR_PARAM_NOT_FOUND, 0);
3829                         ret = -EINVAL;
3830                         goto out;
3831                 }
3832
3833                 param = strstrip(param);
3834                 if (strlen(param) < 2) {
3835                         hist_err(HIST_ERR_INVALID_PARAM, errpos(param));
3836                         ret = -EINVAL;
3837                         goto out;
3838                 }
3839
3840                 saved_param = kstrdup(param, GFP_KERNEL);
3841                 if (!saved_param) {
3842                         ret = -ENOMEM;
3843                         goto out;
3844                 }
3845
3846                 if (first_param && data->use_trace_keyword) {
3847                         data->synth_event_name = saved_param;
3848                         first_param = false;
3849                         continue;
3850                 }
3851                 first_param = false;
3852
3853                 data->params[data->n_params++] = saved_param;
3854         }
3855  out:
3856         return ret;
3857 }
3858
3859 static int action_parse(char *str, struct action_data *data,
3860                         enum handler_id handler)
3861 {
3862         char *action_name;
3863         int ret = 0;
3864
3865         strsep(&str, ".");
3866         if (!str) {
3867                 hist_err(HIST_ERR_ACTION_NOT_FOUND, 0);
3868                 ret = -EINVAL;
3869                 goto out;
3870         }
3871
3872         action_name = strsep(&str, "(");
3873         if (!action_name || !str) {
3874                 hist_err(HIST_ERR_ACTION_NOT_FOUND, 0);
3875                 ret = -EINVAL;
3876                 goto out;
3877         }
3878
3879         if (str_has_prefix(action_name, "save")) {
3880                 char *params = strsep(&str, ")");
3881
3882                 if (!params) {
3883                         hist_err(HIST_ERR_NO_SAVE_PARAMS, 0);
3884                         ret = -EINVAL;
3885                         goto out;
3886                 }
3887
3888                 ret = parse_action_params(params, data);
3889                 if (ret)
3890                         goto out;
3891
3892                 if (handler == HANDLER_ONMAX)
3893                         data->track_data.check_val = check_track_val_max;
3894                 else if (handler == HANDLER_ONCHANGE)
3895                         data->track_data.check_val = check_track_val_changed;
3896                 else {
3897                         hist_err(HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3898                         ret = -EINVAL;
3899                         goto out;
3900                 }
3901
3902                 data->track_data.save_data = save_track_data_vars;
3903                 data->fn = ontrack_action;
3904                 data->action = ACTION_SAVE;
3905         } else if (str_has_prefix(action_name, "snapshot")) {
3906                 char *params = strsep(&str, ")");
3907
3908                 if (!str) {
3909                         hist_err(HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3910                         ret = -EINVAL;
3911                         goto out;
3912                 }
3913
3914                 if (handler == HANDLER_ONMAX)
3915                         data->track_data.check_val = check_track_val_max;
3916                 else if (handler == HANDLER_ONCHANGE)
3917                         data->track_data.check_val = check_track_val_changed;
3918                 else {
3919                         hist_err(HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3920                         ret = -EINVAL;
3921                         goto out;
3922                 }
3923
3924                 data->track_data.save_data = save_track_data_snapshot;
3925                 data->fn = ontrack_action;
3926                 data->action = ACTION_SNAPSHOT;
3927         } else {
3928                 char *params = strsep(&str, ")");
3929
3930                 if (str_has_prefix(action_name, "trace"))
3931                         data->use_trace_keyword = true;
3932
3933                 if (params) {
3934                         ret = parse_action_params(params, data);
3935                         if (ret)
3936                                 goto out;
3937                 }
3938
3939                 if (handler == HANDLER_ONMAX)
3940                         data->track_data.check_val = check_track_val_max;
3941                 else if (handler == HANDLER_ONCHANGE)
3942                         data->track_data.check_val = check_track_val_changed;
3943
3944                 if (handler != HANDLER_ONMATCH) {
3945                         data->track_data.save_data = action_trace;
3946                         data->fn = ontrack_action;
3947                 } else
3948                         data->fn = action_trace;
3949
3950                 data->action = ACTION_TRACE;
3951         }
3952
3953         data->action_name = kstrdup(action_name, GFP_KERNEL);
3954         if (!data->action_name) {
3955                 ret = -ENOMEM;
3956                 goto out;
3957         }
3958
3959         data->handler = handler;
3960  out:
3961         return ret;
3962 }
3963
3964 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3965                                             char *str, enum handler_id handler)
3966 {
3967         struct action_data *data;
3968         int ret = -EINVAL;
3969         char *var_str;
3970
3971         data = kzalloc(sizeof(*data), GFP_KERNEL);
3972         if (!data)
3973                 return ERR_PTR(-ENOMEM);
3974
3975         var_str = strsep(&str, ")");
3976         if (!var_str || !str) {
3977                 ret = -EINVAL;
3978                 goto free;
3979         }
3980
3981         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3982         if (!data->track_data.var_str) {
3983                 ret = -ENOMEM;
3984                 goto free;
3985         }
3986
3987         ret = action_parse(str, data, handler);
3988         if (ret)
3989                 goto free;
3990  out:
3991         return data;
3992  free:
3993         track_data_destroy(hist_data, data);
3994         data = ERR_PTR(ret);
3995         goto out;
3996 }
3997
3998 static void onmatch_destroy(struct action_data *data)
3999 {
4000         kfree(data->match_data.event);
4001         kfree(data->match_data.event_system);
4002
4003         action_data_destroy(data);
4004 }
4005
4006 static void destroy_field_var(struct field_var *field_var)
4007 {
4008         if (!field_var)
4009                 return;
4010
4011         destroy_hist_field(field_var->var, 0);
4012         destroy_hist_field(field_var->val, 0);
4013
4014         kfree(field_var);
4015 }
4016
4017 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4018 {
4019         unsigned int i;
4020
4021         for (i = 0; i < hist_data->n_field_vars; i++)
4022                 destroy_field_var(hist_data->field_vars[i]);
4023 }
4024
4025 static void save_field_var(struct hist_trigger_data *hist_data,
4026                            struct field_var *field_var)
4027 {
4028         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4029
4030         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4031                 hist_data->n_field_var_str++;
4032 }
4033
4034
4035 static int check_synth_field(struct synth_event *event,
4036                              struct hist_field *hist_field,
4037                              unsigned int field_pos)
4038 {
4039         struct synth_field *field;
4040
4041         if (field_pos >= event->n_fields)
4042                 return -EINVAL;
4043
4044         field = event->fields[field_pos];
4045
4046         if (strcmp(field->type, hist_field->type) != 0)
4047                 return -EINVAL;
4048
4049         return 0;
4050 }
4051
4052 static struct hist_field *
4053 trace_action_find_var(struct hist_trigger_data *hist_data,
4054                       struct action_data *data,
4055                       char *system, char *event, char *var)
4056 {
4057         struct hist_field *hist_field;
4058
4059         var++; /* skip '$' */
4060
4061         hist_field = find_target_event_var(hist_data, system, event, var);
4062         if (!hist_field) {
4063                 if (!system && data->handler == HANDLER_ONMATCH) {
4064                         system = data->match_data.event_system;
4065                         event = data->match_data.event;
4066                 }
4067
4068                 hist_field = find_event_var(hist_data, system, event, var);
4069         }
4070
4071         if (!hist_field)
4072                 hist_err(HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4073
4074         return hist_field;
4075 }
4076
4077 static struct hist_field *
4078 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4079                               struct action_data *data, char *system,
4080                               char *event, char *var)
4081 {
4082         struct hist_field *hist_field = NULL;
4083         struct field_var *field_var;
4084
4085         /*
4086          * First try to create a field var on the target event (the
4087          * currently being defined).  This will create a variable for
4088          * unqualified fields on the target event, or if qualified,
4089          * target fields that have qualified names matching the target.
4090          */
4091         field_var = create_target_field_var(hist_data, system, event, var);
4092
4093         if (field_var && !IS_ERR(field_var)) {
4094                 save_field_var(hist_data, field_var);
4095                 hist_field = field_var->var;
4096         } else {
4097                 field_var = NULL;
4098                 /*
4099                  * If no explicit system.event is specfied, default to
4100                  * looking for fields on the onmatch(system.event.xxx)
4101                  * event.
4102                  */
4103                 if (!system && data->handler == HANDLER_ONMATCH) {
4104                         system = data->match_data.event_system;
4105                         event = data->match_data.event;
4106                 }
4107
4108                 /*
4109                  * At this point, we're looking at a field on another
4110                  * event.  Because we can't modify a hist trigger on
4111                  * another event to add a variable for a field, we need
4112                  * to create a new trigger on that event and create the
4113                  * variable at the same time.
4114                  */
4115                 hist_field = create_field_var_hist(hist_data, system, event, var);
4116                 if (IS_ERR(hist_field))
4117                         goto free;
4118         }
4119  out:
4120         return hist_field;
4121  free:
4122         destroy_field_var(field_var);
4123         hist_field = NULL;
4124         goto out;
4125 }
4126
4127 static int trace_action_create(struct hist_trigger_data *hist_data,
4128                                struct action_data *data)
4129 {
4130         char *event_name, *param, *system = NULL;
4131         struct hist_field *hist_field, *var_ref;
4132         unsigned int i, var_ref_idx;
4133         unsigned int field_pos = 0;
4134         struct synth_event *event;
4135         char *synth_event_name;
4136         int ret = 0;
4137
4138         lockdep_assert_held(&event_mutex);
4139
4140         if (data->use_trace_keyword)
4141                 synth_event_name = data->synth_event_name;
4142         else
4143                 synth_event_name = data->action_name;
4144
4145         event = find_synth_event(synth_event_name);
4146         if (!event) {
4147                 hist_err(HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4148                 return -EINVAL;
4149         }
4150
4151         event->ref++;
4152
4153         var_ref_idx = hist_data->n_var_refs;
4154
4155         for (i = 0; i < data->n_params; i++) {
4156                 char *p;
4157
4158                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4159                 if (!param) {
4160                         ret = -ENOMEM;
4161                         goto err;
4162                 }
4163
4164                 system = strsep(&param, ".");
4165                 if (!param) {
4166                         param = (char *)system;
4167                         system = event_name = NULL;
4168                 } else {
4169                         event_name = strsep(&param, ".");
4170                         if (!param) {
4171                                 kfree(p);
4172                                 ret = -EINVAL;
4173                                 goto err;
4174                         }
4175                 }
4176
4177                 if (param[0] == '$')
4178                         hist_field = trace_action_find_var(hist_data, data,
4179                                                            system, event_name,
4180                                                            param);
4181                 else
4182                         hist_field = trace_action_create_field_var(hist_data,
4183                                                                    data,
4184                                                                    system,
4185                                                                    event_name,
4186                                                                    param);
4187
4188                 if (!hist_field) {
4189                         kfree(p);
4190                         ret = -EINVAL;
4191                         goto err;
4192                 }
4193
4194                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4195                         var_ref = create_var_ref(hist_data, hist_field,
4196                                                  system, event_name);
4197                         if (!var_ref) {
4198                                 kfree(p);
4199                                 ret = -ENOMEM;
4200                                 goto err;
4201                         }
4202
4203                         field_pos++;
4204                         kfree(p);
4205                         continue;
4206                 }
4207
4208                 hist_err(HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4209                 kfree(p);
4210                 ret = -EINVAL;
4211                 goto err;
4212         }
4213
4214         if (field_pos != event->n_fields) {
4215                 hist_err(HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4216                 ret = -EINVAL;
4217                 goto err;
4218         }
4219
4220         data->synth_event = event;
4221         data->var_ref_idx = var_ref_idx;
4222  out:
4223         return ret;
4224  err:
4225         event->ref--;
4226
4227         goto out;
4228 }
4229
4230 static int action_create(struct hist_trigger_data *hist_data,
4231                          struct action_data *data)
4232 {
4233         struct trace_event_file *file = hist_data->event_file;
4234         struct track_data *track_data;
4235         struct field_var *field_var;
4236         unsigned int i;
4237         char *param;
4238         int ret = 0;
4239
4240         if (data->action == ACTION_TRACE)
4241                 return trace_action_create(hist_data, data);
4242
4243         if (data->action == ACTION_SNAPSHOT) {
4244                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4245                 if (IS_ERR(track_data)) {
4246                         ret = PTR_ERR(track_data);
4247                         goto out;
4248                 }
4249
4250                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4251                                                    cond_snapshot_update);
4252                 if (ret)
4253                         track_data_free(track_data);
4254
4255                 goto out;
4256         }
4257
4258         if (data->action == ACTION_SAVE) {
4259                 if (hist_data->n_save_vars) {
4260                         ret = -EEXIST;
4261                         hist_err(HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4262                         goto out;
4263                 }
4264
4265                 for (i = 0; i < data->n_params; i++) {
4266                         param = kstrdup(data->params[i], GFP_KERNEL);
4267                         if (!param) {
4268                                 ret = -ENOMEM;
4269                                 goto out;
4270                         }
4271
4272                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4273                         if (IS_ERR(field_var)) {
4274                                 hist_err(HIST_ERR_FIELD_VAR_CREATE_FAIL, errpos(param));
4275                                 ret = PTR_ERR(field_var);
4276                                 kfree(param);
4277                                 goto out;
4278                         }
4279
4280                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4281                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4282                                 hist_data->n_save_var_str++;
4283                         kfree(param);
4284                 }
4285         }
4286  out:
4287         return ret;
4288 }
4289
4290 static int onmatch_create(struct hist_trigger_data *hist_data,
4291                           struct action_data *data)
4292 {
4293         return action_create(hist_data, data);
4294 }
4295
4296 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4297 {
4298         char *match_event, *match_event_system;
4299         struct action_data *data;
4300         int ret = -EINVAL;
4301
4302         data = kzalloc(sizeof(*data), GFP_KERNEL);
4303         if (!data)
4304                 return ERR_PTR(-ENOMEM);
4305
4306         match_event = strsep(&str, ")");
4307         if (!match_event || !str) {
4308                 hist_err(HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4309                 goto free;
4310         }
4311
4312         match_event_system = strsep(&match_event, ".");
4313         if (!match_event) {
4314                 hist_err(HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4315                 goto free;
4316         }
4317
4318         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4319                 hist_err(HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4320                 goto free;
4321         }
4322
4323         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4324         if (!data->match_data.event) {
4325                 ret = -ENOMEM;
4326                 goto free;
4327         }
4328
4329         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4330         if (!data->match_data.event_system) {
4331                 ret = -ENOMEM;
4332                 goto free;
4333         }
4334
4335         ret = action_parse(str, data, HANDLER_ONMATCH);
4336         if (ret)
4337                 goto free;
4338  out:
4339         return data;
4340  free:
4341         onmatch_destroy(data);
4342         data = ERR_PTR(ret);
4343         goto out;
4344 }
4345
4346 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4347 {
4348         hist_data->fields[HITCOUNT_IDX] =
4349                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4350         if (!hist_data->fields[HITCOUNT_IDX])
4351                 return -ENOMEM;
4352
4353         hist_data->n_vals++;
4354         hist_data->n_fields++;
4355
4356         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4357                 return -EINVAL;
4358
4359         return 0;
4360 }
4361
4362 static int __create_val_field(struct hist_trigger_data *hist_data,
4363                               unsigned int val_idx,
4364                               struct trace_event_file *file,
4365                               char *var_name, char *field_str,
4366                               unsigned long flags)
4367 {
4368         struct hist_field *hist_field;
4369         int ret = 0;
4370
4371         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4372         if (IS_ERR(hist_field)) {
4373                 ret = PTR_ERR(hist_field);
4374                 goto out;
4375         }
4376
4377         hist_data->fields[val_idx] = hist_field;
4378
4379         ++hist_data->n_vals;
4380         ++hist_data->n_fields;
4381
4382         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4383                 ret = -EINVAL;
4384  out:
4385         return ret;
4386 }
4387
4388 static int create_val_field(struct hist_trigger_data *hist_data,
4389                             unsigned int val_idx,
4390                             struct trace_event_file *file,
4391                             char *field_str)
4392 {
4393         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4394                 return -EINVAL;
4395
4396         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4397 }
4398
4399 static int create_var_field(struct hist_trigger_data *hist_data,
4400                             unsigned int val_idx,
4401                             struct trace_event_file *file,
4402                             char *var_name, char *expr_str)
4403 {
4404         unsigned long flags = 0;
4405
4406         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4407                 return -EINVAL;
4408
4409         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4410                 hist_err(HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4411                 return -EINVAL;
4412         }
4413
4414         flags |= HIST_FIELD_FL_VAR;
4415         hist_data->n_vars++;
4416         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4417                 return -EINVAL;
4418
4419         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4420 }
4421
4422 static int create_val_fields(struct hist_trigger_data *hist_data,
4423                              struct trace_event_file *file)
4424 {
4425         char *fields_str, *field_str;
4426         unsigned int i, j = 1;
4427         int ret;
4428
4429         ret = create_hitcount_val(hist_data);
4430         if (ret)
4431                 goto out;
4432
4433         fields_str = hist_data->attrs->vals_str;
4434         if (!fields_str)
4435                 goto out;
4436
4437         strsep(&fields_str, "=");
4438         if (!fields_str)
4439                 goto out;
4440
4441         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4442                      j < TRACING_MAP_VALS_MAX; i++) {
4443                 field_str = strsep(&fields_str, ",");
4444                 if (!field_str)
4445                         break;
4446
4447                 if (strcmp(field_str, "hitcount") == 0)
4448                         continue;
4449
4450                 ret = create_val_field(hist_data, j++, file, field_str);
4451                 if (ret)
4452                         goto out;
4453         }
4454
4455         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4456                 ret = -EINVAL;
4457  out:
4458         return ret;
4459 }
4460
4461 static int create_key_field(struct hist_trigger_data *hist_data,
4462                             unsigned int key_idx,
4463                             unsigned int key_offset,
4464                             struct trace_event_file *file,
4465                             char *field_str)
4466 {
4467         struct hist_field *hist_field = NULL;
4468
4469         unsigned long flags = 0;
4470         unsigned int key_size;
4471         int ret = 0;
4472
4473         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4474                 return -EINVAL;
4475
4476         flags |= HIST_FIELD_FL_KEY;
4477
4478         if (strcmp(field_str, "stacktrace") == 0) {
4479                 flags |= HIST_FIELD_FL_STACKTRACE;
4480                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4481                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4482         } else {
4483                 hist_field = parse_expr(hist_data, file, field_str, flags,
4484                                         NULL, 0);
4485                 if (IS_ERR(hist_field)) {
4486                         ret = PTR_ERR(hist_field);
4487                         goto out;
4488                 }
4489
4490                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
4491                         hist_err(HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4492                         destroy_hist_field(hist_field, 0);
4493                         ret = -EINVAL;
4494                         goto out;
4495                 }
4496
4497                 key_size = hist_field->size;
4498         }
4499
4500         hist_data->fields[key_idx] = hist_field;
4501
4502         key_size = ALIGN(key_size, sizeof(u64));
4503         hist_data->fields[key_idx]->size = key_size;
4504         hist_data->fields[key_idx]->offset = key_offset;
4505
4506         hist_data->key_size += key_size;
4507
4508         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4509                 ret = -EINVAL;
4510                 goto out;
4511         }
4512
4513         hist_data->n_keys++;
4514         hist_data->n_fields++;
4515
4516         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4517                 return -EINVAL;
4518
4519         ret = key_size;
4520  out:
4521         return ret;
4522 }
4523
4524 static int create_key_fields(struct hist_trigger_data *hist_data,
4525                              struct trace_event_file *file)
4526 {
4527         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4528         char *fields_str, *field_str;
4529         int ret = -EINVAL;
4530
4531         fields_str = hist_data->attrs->keys_str;
4532         if (!fields_str)
4533                 goto out;
4534
4535         strsep(&fields_str, "=");
4536         if (!fields_str)
4537                 goto out;
4538
4539         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4540                 field_str = strsep(&fields_str, ",");
4541                 if (!field_str)
4542                         break;
4543                 ret = create_key_field(hist_data, i, key_offset,
4544                                        file, field_str);
4545                 if (ret < 0)
4546                         goto out;
4547                 key_offset += ret;
4548         }
4549         if (fields_str) {
4550                 ret = -EINVAL;
4551                 goto out;
4552         }
4553         ret = 0;
4554  out:
4555         return ret;
4556 }
4557
4558 static int create_var_fields(struct hist_trigger_data *hist_data,
4559                              struct trace_event_file *file)
4560 {
4561         unsigned int i, j = hist_data->n_vals;
4562         int ret = 0;
4563
4564         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4565
4566         for (i = 0; i < n_vars; i++) {
4567                 char *var_name = hist_data->attrs->var_defs.name[i];
4568                 char *expr = hist_data->attrs->var_defs.expr[i];
4569
4570                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4571                 if (ret)
4572                         goto out;
4573         }
4574  out:
4575         return ret;
4576 }
4577
4578 static void free_var_defs(struct hist_trigger_data *hist_data)
4579 {
4580         unsigned int i;
4581
4582         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4583                 kfree(hist_data->attrs->var_defs.name[i]);
4584                 kfree(hist_data->attrs->var_defs.expr[i]);
4585         }
4586
4587         hist_data->attrs->var_defs.n_vars = 0;
4588 }
4589
4590 static int parse_var_defs(struct hist_trigger_data *hist_data)
4591 {
4592         char *s, *str, *var_name, *field_str;
4593         unsigned int i, j, n_vars = 0;
4594         int ret = 0;
4595
4596         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4597                 str = hist_data->attrs->assignment_str[i];
4598                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4599                         field_str = strsep(&str, ",");
4600                         if (!field_str)
4601                                 break;
4602
4603                         var_name = strsep(&field_str, "=");
4604                         if (!var_name || !field_str) {
4605                                 hist_err(HIST_ERR_MALFORMED_ASSIGNMENT, errpos(var_name));
4606                                 ret = -EINVAL;
4607                                 goto free;
4608                         }
4609
4610                         if (n_vars == TRACING_MAP_VARS_MAX) {
4611                                 hist_err(HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4612                                 ret = -EINVAL;
4613                                 goto free;
4614                         }
4615
4616                         s = kstrdup(var_name, GFP_KERNEL);
4617                         if (!s) {
4618                                 ret = -ENOMEM;
4619                                 goto free;
4620                         }
4621                         hist_data->attrs->var_defs.name[n_vars] = s;
4622
4623                         s = kstrdup(field_str, GFP_KERNEL);
4624                         if (!s) {
4625                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4626                                 ret = -ENOMEM;
4627                                 goto free;
4628                         }
4629                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4630
4631                         hist_data->attrs->var_defs.n_vars = n_vars;
4632                 }
4633         }
4634
4635         return ret;
4636  free:
4637         free_var_defs(hist_data);
4638
4639         return ret;
4640 }
4641
4642 static int create_hist_fields(struct hist_trigger_data *hist_data,
4643                               struct trace_event_file *file)
4644 {
4645         int ret;
4646
4647         ret = parse_var_defs(hist_data);
4648         if (ret)
4649                 goto out;
4650
4651         ret = create_val_fields(hist_data, file);
4652         if (ret)
4653                 goto out;
4654
4655         ret = create_var_fields(hist_data, file);
4656         if (ret)
4657                 goto out;
4658
4659         ret = create_key_fields(hist_data, file);
4660         if (ret)
4661                 goto out;
4662  out:
4663         free_var_defs(hist_data);
4664
4665         return ret;
4666 }
4667
4668 static int is_descending(const char *str)
4669 {
4670         if (!str)
4671                 return 0;
4672
4673         if (strcmp(str, "descending") == 0)
4674                 return 1;
4675
4676         if (strcmp(str, "ascending") == 0)
4677                 return 0;
4678
4679         return -EINVAL;
4680 }
4681
4682 static int create_sort_keys(struct hist_trigger_data *hist_data)
4683 {
4684         char *fields_str = hist_data->attrs->sort_key_str;
4685         struct tracing_map_sort_key *sort_key;
4686         int descending, ret = 0;
4687         unsigned int i, j, k;
4688
4689         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4690
4691         if (!fields_str)
4692                 goto out;
4693
4694         strsep(&fields_str, "=");
4695         if (!fields_str) {
4696                 ret = -EINVAL;
4697                 goto out;
4698         }
4699
4700         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4701                 struct hist_field *hist_field;
4702                 char *field_str, *field_name;
4703                 const char *test_name;
4704
4705                 sort_key = &hist_data->sort_keys[i];
4706
4707                 field_str = strsep(&fields_str, ",");
4708                 if (!field_str) {
4709                         if (i == 0)
4710                                 ret = -EINVAL;
4711                         break;
4712                 }
4713
4714                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4715                         ret = -EINVAL;
4716                         break;
4717                 }
4718
4719                 field_name = strsep(&field_str, ".");
4720                 if (!field_name) {
4721                         ret = -EINVAL;
4722                         break;
4723                 }
4724
4725                 if (strcmp(field_name, "hitcount") == 0) {
4726                         descending = is_descending(field_str);
4727                         if (descending < 0) {
4728                                 ret = descending;
4729                                 break;
4730                         }
4731                         sort_key->descending = descending;
4732                         continue;
4733                 }
4734
4735                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4736                         unsigned int idx;
4737
4738                         hist_field = hist_data->fields[j];
4739                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4740                                 continue;
4741
4742                         idx = k++;
4743
4744                         test_name = hist_field_name(hist_field, 0);
4745
4746                         if (strcmp(field_name, test_name) == 0) {
4747                                 sort_key->field_idx = idx;
4748                                 descending = is_descending(field_str);
4749                                 if (descending < 0) {
4750                                         ret = descending;
4751                                         goto out;
4752                                 }
4753                                 sort_key->descending = descending;
4754                                 break;
4755                         }
4756                 }
4757                 if (j == hist_data->n_fields) {
4758                         ret = -EINVAL;
4759                         break;
4760                 }
4761         }
4762
4763         hist_data->n_sort_keys = i;
4764  out:
4765         return ret;
4766 }
4767
4768 static void destroy_actions(struct hist_trigger_data *hist_data)
4769 {
4770         unsigned int i;
4771
4772         for (i = 0; i < hist_data->n_actions; i++) {
4773                 struct action_data *data = hist_data->actions[i];
4774
4775                 if (data->handler == HANDLER_ONMATCH)
4776                         onmatch_destroy(data);
4777                 else if (data->handler == HANDLER_ONMAX ||
4778                          data->handler == HANDLER_ONCHANGE)
4779                         track_data_destroy(hist_data, data);
4780                 else
4781                         kfree(data);
4782         }
4783 }
4784
4785 static int parse_actions(struct hist_trigger_data *hist_data)
4786 {
4787         struct trace_array *tr = hist_data->event_file->tr;
4788         struct action_data *data;
4789         unsigned int i;
4790         int ret = 0;
4791         char *str;
4792         int len;
4793
4794         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4795                 str = hist_data->attrs->action_str[i];
4796
4797                 if ((len = str_has_prefix(str, "onmatch("))) {
4798                         char *action_str = str + len;
4799
4800                         data = onmatch_parse(tr, action_str);
4801                         if (IS_ERR(data)) {
4802                                 ret = PTR_ERR(data);
4803                                 break;
4804                         }
4805                 } else if ((len = str_has_prefix(str, "onmax("))) {
4806                         char *action_str = str + len;
4807
4808                         data = track_data_parse(hist_data, action_str,
4809                                                 HANDLER_ONMAX);
4810                         if (IS_ERR(data)) {
4811                                 ret = PTR_ERR(data);
4812                                 break;
4813                         }
4814                 } else if ((len = str_has_prefix(str, "onchange("))) {
4815                         char *action_str = str + len;
4816
4817                         data = track_data_parse(hist_data, action_str,
4818                                                 HANDLER_ONCHANGE);
4819                         if (IS_ERR(data)) {
4820                                 ret = PTR_ERR(data);
4821                                 break;
4822                         }
4823                 } else {
4824                         ret = -EINVAL;
4825                         break;
4826                 }
4827
4828                 hist_data->actions[hist_data->n_actions++] = data;
4829         }
4830
4831         return ret;
4832 }
4833
4834 static int create_actions(struct hist_trigger_data *hist_data)
4835 {
4836         struct action_data *data;
4837         unsigned int i;
4838         int ret = 0;
4839
4840         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4841                 data = hist_data->actions[i];
4842
4843                 if (data->handler == HANDLER_ONMATCH) {
4844                         ret = onmatch_create(hist_data, data);
4845                         if (ret)
4846                                 break;
4847                 } else if (data->handler == HANDLER_ONMAX ||
4848                            data->handler == HANDLER_ONCHANGE) {
4849                         ret = track_data_create(hist_data, data);
4850                         if (ret)
4851                                 break;
4852                 } else {
4853                         ret = -EINVAL;
4854                         break;
4855                 }
4856         }
4857
4858         return ret;
4859 }
4860
4861 static void print_actions(struct seq_file *m,
4862                           struct hist_trigger_data *hist_data,
4863                           struct tracing_map_elt *elt)
4864 {
4865         unsigned int i;
4866
4867         for (i = 0; i < hist_data->n_actions; i++) {
4868                 struct action_data *data = hist_data->actions[i];
4869
4870                 if (data->action == ACTION_SNAPSHOT)
4871                         continue;
4872
4873                 if (data->handler == HANDLER_ONMAX ||
4874                     data->handler == HANDLER_ONCHANGE)
4875                         track_data_print(m, hist_data, elt, data);
4876         }
4877 }
4878
4879 static void print_action_spec(struct seq_file *m,
4880                               struct hist_trigger_data *hist_data,
4881                               struct action_data *data)
4882 {
4883         unsigned int i;
4884
4885         if (data->action == ACTION_SAVE) {
4886                 for (i = 0; i < hist_data->n_save_vars; i++) {
4887                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4888                         if (i < hist_data->n_save_vars - 1)
4889                                 seq_puts(m, ",");
4890                 }
4891         } else if (data->action == ACTION_TRACE) {
4892                 if (data->use_trace_keyword)
4893                         seq_printf(m, "%s", data->synth_event_name);
4894                 for (i = 0; i < data->n_params; i++) {
4895                         if (i || data->use_trace_keyword)
4896                                 seq_puts(m, ",");
4897                         seq_printf(m, "%s", data->params[i]);
4898                 }
4899         }
4900 }
4901
4902 static void print_track_data_spec(struct seq_file *m,
4903                                   struct hist_trigger_data *hist_data,
4904                                   struct action_data *data)
4905 {
4906         if (data->handler == HANDLER_ONMAX)
4907                 seq_puts(m, ":onmax(");
4908         else if (data->handler == HANDLER_ONCHANGE)
4909                 seq_puts(m, ":onchange(");
4910         seq_printf(m, "%s", data->track_data.var_str);
4911         seq_printf(m, ").%s(", data->action_name);
4912
4913         print_action_spec(m, hist_data, data);
4914
4915         seq_puts(m, ")");
4916 }
4917
4918 static void print_onmatch_spec(struct seq_file *m,
4919                                struct hist_trigger_data *hist_data,
4920                                struct action_data *data)
4921 {
4922         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4923                    data->match_data.event);
4924
4925         seq_printf(m, "%s(", data->action_name);
4926
4927         print_action_spec(m, hist_data, data);
4928
4929         seq_puts(m, ")");
4930 }
4931
4932 static bool actions_match(struct hist_trigger_data *hist_data,
4933                           struct hist_trigger_data *hist_data_test)
4934 {
4935         unsigned int i, j;
4936
4937         if (hist_data->n_actions != hist_data_test->n_actions)
4938                 return false;
4939
4940         for (i = 0; i < hist_data->n_actions; i++) {
4941                 struct action_data *data = hist_data->actions[i];
4942                 struct action_data *data_test = hist_data_test->actions[i];
4943                 char *action_name, *action_name_test;
4944
4945                 if (data->handler != data_test->handler)
4946                         return false;
4947                 if (data->action != data_test->action)
4948                         return false;
4949
4950                 if (data->n_params != data_test->n_params)
4951                         return false;
4952
4953                 for (j = 0; j < data->n_params; j++) {
4954                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4955                                 return false;
4956                 }
4957
4958                 if (data->use_trace_keyword)
4959                         action_name = data->synth_event_name;
4960                 else
4961                         action_name = data->action_name;
4962
4963                 if (data_test->use_trace_keyword)
4964                         action_name_test = data_test->synth_event_name;
4965                 else
4966                         action_name_test = data_test->action_name;
4967
4968                 if (strcmp(action_name, action_name_test) != 0)
4969                         return false;
4970
4971                 if (data->handler == HANDLER_ONMATCH) {
4972                         if (strcmp(data->match_data.event_system,
4973                                    data_test->match_data.event_system) != 0)
4974                                 return false;
4975                         if (strcmp(data->match_data.event,
4976                                    data_test->match_data.event) != 0)
4977                                 return false;
4978                 } else if (data->handler == HANDLER_ONMAX ||
4979                            data->handler == HANDLER_ONCHANGE) {
4980                         if (strcmp(data->track_data.var_str,
4981                                    data_test->track_data.var_str) != 0)
4982                                 return false;
4983                 }
4984         }
4985
4986         return true;
4987 }
4988
4989
4990 static void print_actions_spec(struct seq_file *m,
4991                                struct hist_trigger_data *hist_data)
4992 {
4993         unsigned int i;
4994
4995         for (i = 0; i < hist_data->n_actions; i++) {
4996                 struct action_data *data = hist_data->actions[i];
4997
4998                 if (data->handler == HANDLER_ONMATCH)
4999                         print_onmatch_spec(m, hist_data, data);
5000                 else if (data->handler == HANDLER_ONMAX ||
5001                          data->handler == HANDLER_ONCHANGE)
5002                         print_track_data_spec(m, hist_data, data);
5003         }
5004 }
5005
5006 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5007 {
5008         unsigned int i;
5009
5010         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5011                 kfree(hist_data->field_var_hists[i]->cmd);
5012                 kfree(hist_data->field_var_hists[i]);
5013         }
5014 }
5015
5016 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5017 {
5018         if (!hist_data)
5019                 return;
5020
5021         destroy_hist_trigger_attrs(hist_data->attrs);
5022         destroy_hist_fields(hist_data);
5023         tracing_map_destroy(hist_data->map);
5024
5025         destroy_actions(hist_data);
5026         destroy_field_vars(hist_data);
5027         destroy_field_var_hists(hist_data);
5028
5029         kfree(hist_data);
5030 }
5031
5032 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5033 {
5034         struct tracing_map *map = hist_data->map;
5035         struct ftrace_event_field *field;
5036         struct hist_field *hist_field;
5037         int i, idx = 0;
5038
5039         for_each_hist_field(i, hist_data) {
5040                 hist_field = hist_data->fields[i];
5041                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5042                         tracing_map_cmp_fn_t cmp_fn;
5043
5044                         field = hist_field->field;
5045
5046                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5047                                 cmp_fn = tracing_map_cmp_none;
5048                         else if (!field)
5049                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5050                                                              hist_field->is_signed);
5051                         else if (is_string_field(field))
5052                                 cmp_fn = tracing_map_cmp_string;
5053                         else
5054                                 cmp_fn = tracing_map_cmp_num(field->size,
5055                                                              field->is_signed);
5056                         idx = tracing_map_add_key_field(map,
5057                                                         hist_field->offset,
5058                                                         cmp_fn);
5059                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5060                         idx = tracing_map_add_sum_field(map);
5061
5062                 if (idx < 0)
5063                         return idx;
5064
5065                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5066                         idx = tracing_map_add_var(map);
5067                         if (idx < 0)
5068                                 return idx;
5069                         hist_field->var.idx = idx;
5070                         hist_field->var.hist_data = hist_data;
5071                 }
5072         }
5073
5074         return 0;
5075 }
5076
5077 static struct hist_trigger_data *
5078 create_hist_data(unsigned int map_bits,
5079                  struct hist_trigger_attrs *attrs,
5080                  struct trace_event_file *file,
5081                  bool remove)
5082 {
5083         const struct tracing_map_ops *map_ops = NULL;
5084         struct hist_trigger_data *hist_data;
5085         int ret = 0;
5086
5087         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5088         if (!hist_data)
5089                 return ERR_PTR(-ENOMEM);
5090
5091         hist_data->attrs = attrs;
5092         hist_data->remove = remove;
5093         hist_data->event_file = file;
5094
5095         ret = parse_actions(hist_data);
5096         if (ret)
5097                 goto free;
5098
5099         ret = create_hist_fields(hist_data, file);
5100         if (ret)
5101                 goto free;
5102
5103         ret = create_sort_keys(hist_data);
5104         if (ret)
5105                 goto free;
5106
5107         map_ops = &hist_trigger_elt_data_ops;
5108
5109         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5110                                             map_ops, hist_data);
5111         if (IS_ERR(hist_data->map)) {
5112                 ret = PTR_ERR(hist_data->map);
5113                 hist_data->map = NULL;
5114                 goto free;
5115         }
5116
5117         ret = create_tracing_map_fields(hist_data);
5118         if (ret)
5119                 goto free;
5120  out:
5121         return hist_data;
5122  free:
5123         hist_data->attrs = NULL;
5124
5125         destroy_hist_data(hist_data);
5126
5127         hist_data = ERR_PTR(ret);
5128
5129         goto out;
5130 }
5131
5132 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5133                                     struct tracing_map_elt *elt, void *rec,
5134                                     struct ring_buffer_event *rbe,
5135                                     u64 *var_ref_vals)
5136 {
5137         struct hist_elt_data *elt_data;
5138         struct hist_field *hist_field;
5139         unsigned int i, var_idx;
5140         u64 hist_val;
5141
5142         elt_data = elt->private_data;
5143         elt_data->var_ref_vals = var_ref_vals;
5144
5145         for_each_hist_val_field(i, hist_data) {
5146                 hist_field = hist_data->fields[i];
5147                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5148                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5149                         var_idx = hist_field->var.idx;
5150                         tracing_map_set_var(elt, var_idx, hist_val);
5151                         continue;
5152                 }
5153                 tracing_map_update_sum(elt, i, hist_val);
5154         }
5155
5156         for_each_hist_key_field(i, hist_data) {
5157                 hist_field = hist_data->fields[i];
5158                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5159                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5160                         var_idx = hist_field->var.idx;
5161                         tracing_map_set_var(elt, var_idx, hist_val);
5162                 }
5163         }
5164
5165         update_field_vars(hist_data, elt, rbe, rec);
5166 }
5167
5168 static inline void add_to_key(char *compound_key, void *key,
5169                               struct hist_field *key_field, void *rec)
5170 {
5171         size_t size = key_field->size;
5172
5173         if (key_field->flags & HIST_FIELD_FL_STRING) {
5174                 struct ftrace_event_field *field;
5175
5176                 field = key_field->field;
5177                 if (field->filter_type == FILTER_DYN_STRING)
5178                         size = *(u32 *)(rec + field->offset) >> 16;
5179                 else if (field->filter_type == FILTER_PTR_STRING)
5180                         size = strlen(key);
5181                 else if (field->filter_type == FILTER_STATIC_STRING)
5182                         size = field->size;
5183
5184                 /* ensure NULL-termination */
5185                 if (size > key_field->size - 1)
5186                         size = key_field->size - 1;
5187
5188                 strncpy(compound_key + key_field->offset, (char *)key, size);
5189         } else
5190                 memcpy(compound_key + key_field->offset, key, size);
5191 }
5192
5193 static void
5194 hist_trigger_actions(struct hist_trigger_data *hist_data,
5195                      struct tracing_map_elt *elt, void *rec,
5196                      struct ring_buffer_event *rbe, void *key,
5197                      u64 *var_ref_vals)
5198 {
5199         struct action_data *data;
5200         unsigned int i;
5201
5202         for (i = 0; i < hist_data->n_actions; i++) {
5203                 data = hist_data->actions[i];
5204                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5205         }
5206 }
5207
5208 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5209                                struct ring_buffer_event *rbe)
5210 {
5211         struct hist_trigger_data *hist_data = data->private_data;
5212         bool use_compound_key = (hist_data->n_keys > 1);
5213         unsigned long entries[HIST_STACKTRACE_DEPTH];
5214         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5215         char compound_key[HIST_KEY_SIZE_MAX];
5216         struct tracing_map_elt *elt = NULL;
5217         struct stack_trace stacktrace;
5218         struct hist_field *key_field;
5219         u64 field_contents;
5220         void *key = NULL;
5221         unsigned int i;
5222
5223         memset(compound_key, 0, hist_data->key_size);
5224
5225         for_each_hist_key_field(i, hist_data) {
5226                 key_field = hist_data->fields[i];
5227
5228                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5229                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
5230                         stacktrace.entries = entries;
5231                         stacktrace.nr_entries = 0;
5232                         stacktrace.skip = HIST_STACKTRACE_SKIP;
5233
5234                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
5235                         save_stack_trace(&stacktrace);
5236
5237                         key = entries;
5238                 } else {
5239                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5240                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5241                                 key = (void *)(unsigned long)field_contents;
5242                                 use_compound_key = true;
5243                         } else
5244                                 key = (void *)&field_contents;
5245                 }
5246
5247                 if (use_compound_key)
5248                         add_to_key(compound_key, key, key_field, rec);
5249         }
5250
5251         if (use_compound_key)
5252                 key = compound_key;
5253
5254         if (hist_data->n_var_refs &&
5255             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5256                 return;
5257
5258         elt = tracing_map_insert(hist_data->map, key);
5259         if (!elt)
5260                 return;
5261
5262         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5263
5264         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5265                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5266 }
5267
5268 static void hist_trigger_stacktrace_print(struct seq_file *m,
5269                                           unsigned long *stacktrace_entries,
5270                                           unsigned int max_entries)
5271 {
5272         char str[KSYM_SYMBOL_LEN];
5273         unsigned int spaces = 8;
5274         unsigned int i;
5275
5276         for (i = 0; i < max_entries; i++) {
5277                 if (stacktrace_entries[i] == ULONG_MAX)
5278                         return;
5279
5280                 seq_printf(m, "%*c", 1 + spaces, ' ');
5281                 sprint_symbol(str, stacktrace_entries[i]);
5282                 seq_printf(m, "%s\n", str);
5283         }
5284 }
5285
5286 static void hist_trigger_print_key(struct seq_file *m,
5287                                    struct hist_trigger_data *hist_data,
5288                                    void *key,
5289                                    struct tracing_map_elt *elt)
5290 {
5291         struct hist_field *key_field;
5292         char str[KSYM_SYMBOL_LEN];
5293         bool multiline = false;
5294         const char *field_name;
5295         unsigned int i;
5296         u64 uval;
5297
5298         seq_puts(m, "{ ");
5299
5300         for_each_hist_key_field(i, hist_data) {
5301                 key_field = hist_data->fields[i];
5302
5303                 if (i > hist_data->n_vals)
5304                         seq_puts(m, ", ");
5305
5306                 field_name = hist_field_name(key_field, 0);
5307
5308                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5309                         uval = *(u64 *)(key + key_field->offset);
5310                         seq_printf(m, "%s: %llx", field_name, uval);
5311                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5312                         uval = *(u64 *)(key + key_field->offset);
5313                         sprint_symbol_no_offset(str, uval);
5314                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5315                                    uval, str);
5316                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5317                         uval = *(u64 *)(key + key_field->offset);
5318                         sprint_symbol(str, uval);
5319                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5320                                    uval, str);
5321                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5322                         struct hist_elt_data *elt_data = elt->private_data;
5323                         char *comm;
5324
5325                         if (WARN_ON_ONCE(!elt_data))
5326                                 return;
5327
5328                         comm = elt_data->comm;
5329
5330                         uval = *(u64 *)(key + key_field->offset);
5331                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5332                                    comm, uval);
5333                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5334                         const char *syscall_name;
5335
5336                         uval = *(u64 *)(key + key_field->offset);
5337                         syscall_name = get_syscall_name(uval);
5338                         if (!syscall_name)
5339                                 syscall_name = "unknown_syscall";
5340
5341                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5342                                    syscall_name, uval);
5343                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5344                         seq_puts(m, "stacktrace:\n");
5345                         hist_trigger_stacktrace_print(m,
5346                                                       key + key_field->offset,
5347                                                       HIST_STACKTRACE_DEPTH);
5348                         multiline = true;
5349                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5350                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5351                                    *(u64 *)(key + key_field->offset));
5352                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5353                         seq_printf(m, "%s: %-50s", field_name,
5354                                    (char *)(key + key_field->offset));
5355                 } else {
5356                         uval = *(u64 *)(key + key_field->offset);
5357                         seq_printf(m, "%s: %10llu", field_name, uval);
5358                 }
5359         }
5360
5361         if (!multiline)
5362                 seq_puts(m, " ");
5363
5364         seq_puts(m, "}");
5365 }
5366
5367 static void hist_trigger_entry_print(struct seq_file *m,
5368                                      struct hist_trigger_data *hist_data,
5369                                      void *key,
5370                                      struct tracing_map_elt *elt)
5371 {
5372         const char *field_name;
5373         unsigned int i;
5374
5375         hist_trigger_print_key(m, hist_data, key, elt);
5376
5377         seq_printf(m, " hitcount: %10llu",
5378                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5379
5380         for (i = 1; i < hist_data->n_vals; i++) {
5381                 field_name = hist_field_name(hist_data->fields[i], 0);
5382
5383                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5384                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5385                         continue;
5386
5387                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5388                         seq_printf(m, "  %s: %10llx", field_name,
5389                                    tracing_map_read_sum(elt, i));
5390                 } else {
5391                         seq_printf(m, "  %s: %10llu", field_name,
5392                                    tracing_map_read_sum(elt, i));
5393                 }
5394         }
5395
5396         print_actions(m, hist_data, elt);
5397
5398         seq_puts(m, "\n");
5399 }
5400
5401 static int print_entries(struct seq_file *m,
5402                          struct hist_trigger_data *hist_data)
5403 {
5404         struct tracing_map_sort_entry **sort_entries = NULL;
5405         struct tracing_map *map = hist_data->map;
5406         int i, n_entries;
5407
5408         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5409                                              hist_data->n_sort_keys,
5410                                              &sort_entries);
5411         if (n_entries < 0)
5412                 return n_entries;
5413
5414         for (i = 0; i < n_entries; i++)
5415                 hist_trigger_entry_print(m, hist_data,
5416                                          sort_entries[i]->key,
5417                                          sort_entries[i]->elt);
5418
5419         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5420
5421         return n_entries;
5422 }
5423
5424 static void hist_trigger_show(struct seq_file *m,
5425                               struct event_trigger_data *data, int n)
5426 {
5427         struct hist_trigger_data *hist_data;
5428         int n_entries;
5429
5430         if (n > 0)
5431                 seq_puts(m, "\n\n");
5432
5433         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5434         data->ops->print(m, data->ops, data);
5435         seq_puts(m, "#\n\n");
5436
5437         hist_data = data->private_data;
5438         n_entries = print_entries(m, hist_data);
5439         if (n_entries < 0)
5440                 n_entries = 0;
5441
5442         track_data_snapshot_print(m, hist_data);
5443
5444         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5445                    (u64)atomic64_read(&hist_data->map->hits),
5446                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5447 }
5448
5449 static int hist_show(struct seq_file *m, void *v)
5450 {
5451         struct event_trigger_data *data;
5452         struct trace_event_file *event_file;
5453         int n = 0, ret = 0;
5454
5455         mutex_lock(&event_mutex);
5456
5457         event_file = event_file_data(m->private);
5458         if (unlikely(!event_file)) {
5459                 ret = -ENODEV;
5460                 goto out_unlock;
5461         }
5462
5463         list_for_each_entry_rcu(data, &event_file->triggers, list) {
5464                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5465                         hist_trigger_show(m, data, n++);
5466         }
5467
5468  out_unlock:
5469         mutex_unlock(&event_mutex);
5470
5471         return ret;
5472 }
5473
5474 static int event_hist_open(struct inode *inode, struct file *file)
5475 {
5476         return single_open(file, hist_show, file);
5477 }
5478
5479 const struct file_operations event_hist_fops = {
5480         .open = event_hist_open,
5481         .read = seq_read,
5482         .llseek = seq_lseek,
5483         .release = single_release,
5484 };
5485
5486 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5487 {
5488         const char *field_name = hist_field_name(hist_field, 0);
5489
5490         if (hist_field->var.name)
5491                 seq_printf(m, "%s=", hist_field->var.name);
5492
5493         if (hist_field->flags & HIST_FIELD_FL_CPU)
5494                 seq_puts(m, "cpu");
5495         else if (field_name) {
5496                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5497                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5498                         seq_putc(m, '$');
5499                 seq_printf(m, "%s", field_name);
5500         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5501                 seq_puts(m, "common_timestamp");
5502
5503         if (hist_field->flags) {
5504                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5505                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5506                         const char *flags = get_hist_field_flags(hist_field);
5507
5508                         if (flags)
5509                                 seq_printf(m, ".%s", flags);
5510                 }
5511         }
5512 }
5513
5514 static int event_hist_trigger_print(struct seq_file *m,
5515                                     struct event_trigger_ops *ops,
5516                                     struct event_trigger_data *data)
5517 {
5518         struct hist_trigger_data *hist_data = data->private_data;
5519         struct hist_field *field;
5520         bool have_var = false;
5521         unsigned int i;
5522
5523         seq_puts(m, "hist:");
5524
5525         if (data->name)
5526                 seq_printf(m, "%s:", data->name);
5527
5528         seq_puts(m, "keys=");
5529
5530         for_each_hist_key_field(i, hist_data) {
5531                 field = hist_data->fields[i];
5532
5533                 if (i > hist_data->n_vals)
5534                         seq_puts(m, ",");
5535
5536                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5537                         seq_puts(m, "stacktrace");
5538                 else
5539                         hist_field_print(m, field);
5540         }
5541
5542         seq_puts(m, ":vals=");
5543
5544         for_each_hist_val_field(i, hist_data) {
5545                 field = hist_data->fields[i];
5546                 if (field->flags & HIST_FIELD_FL_VAR) {
5547                         have_var = true;
5548                         continue;
5549                 }
5550
5551                 if (i == HITCOUNT_IDX)
5552                         seq_puts(m, "hitcount");
5553                 else {
5554                         seq_puts(m, ",");
5555                         hist_field_print(m, field);
5556                 }
5557         }
5558
5559         if (have_var) {
5560                 unsigned int n = 0;
5561
5562                 seq_puts(m, ":");
5563
5564                 for_each_hist_val_field(i, hist_data) {
5565                         field = hist_data->fields[i];
5566
5567                         if (field->flags & HIST_FIELD_FL_VAR) {
5568                                 if (n++)
5569                                         seq_puts(m, ",");
5570                                 hist_field_print(m, field);
5571                         }
5572                 }
5573         }
5574
5575         seq_puts(m, ":sort=");
5576
5577         for (i = 0; i < hist_data->n_sort_keys; i++) {
5578                 struct tracing_map_sort_key *sort_key;
5579                 unsigned int idx, first_key_idx;
5580
5581                 /* skip VAR vals */
5582                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5583
5584                 sort_key = &hist_data->sort_keys[i];
5585                 idx = sort_key->field_idx;
5586
5587                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5588                         return -EINVAL;
5589
5590                 if (i > 0)
5591                         seq_puts(m, ",");
5592
5593                 if (idx == HITCOUNT_IDX)
5594                         seq_puts(m, "hitcount");
5595                 else {
5596                         if (idx >= first_key_idx)
5597                                 idx += hist_data->n_vars;
5598                         hist_field_print(m, hist_data->fields[idx]);
5599                 }
5600
5601                 if (sort_key->descending)
5602                         seq_puts(m, ".descending");
5603         }
5604         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5605         if (hist_data->enable_timestamps)
5606                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5607
5608         print_actions_spec(m, hist_data);
5609
5610         if (data->filter_str)
5611                 seq_printf(m, " if %s", data->filter_str);
5612
5613         if (data->paused)
5614                 seq_puts(m, " [paused]");
5615         else
5616                 seq_puts(m, " [active]");
5617
5618         seq_putc(m, '\n');
5619
5620         return 0;
5621 }
5622
5623 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5624                                    struct event_trigger_data *data)
5625 {
5626         struct hist_trigger_data *hist_data = data->private_data;
5627
5628         if (!data->ref && hist_data->attrs->name)
5629                 save_named_trigger(hist_data->attrs->name, data);
5630
5631         data->ref++;
5632
5633         return 0;
5634 }
5635
5636 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5637 {
5638         struct trace_event_file *file;
5639         unsigned int i;
5640         char *cmd;
5641         int ret;
5642
5643         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5644                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5645                 cmd = hist_data->field_var_hists[i]->cmd;
5646                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5647                                               "!hist", "hist", cmd);
5648         }
5649 }
5650
5651 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5652                                     struct event_trigger_data *data)
5653 {
5654         struct hist_trigger_data *hist_data = data->private_data;
5655
5656         if (WARN_ON_ONCE(data->ref <= 0))
5657                 return;
5658
5659         data->ref--;
5660         if (!data->ref) {
5661                 if (data->name)
5662                         del_named_trigger(data);
5663
5664                 trigger_data_free(data);
5665
5666                 remove_hist_vars(hist_data);
5667
5668                 unregister_field_var_hists(hist_data);
5669
5670                 destroy_hist_data(hist_data);
5671         }
5672 }
5673
5674 static struct event_trigger_ops event_hist_trigger_ops = {
5675         .func                   = event_hist_trigger,
5676         .print                  = event_hist_trigger_print,
5677         .init                   = event_hist_trigger_init,
5678         .free                   = event_hist_trigger_free,
5679 };
5680
5681 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5682                                          struct event_trigger_data *data)
5683 {
5684         data->ref++;
5685
5686         save_named_trigger(data->named_data->name, data);
5687
5688         event_hist_trigger_init(ops, data->named_data);
5689
5690         return 0;
5691 }
5692
5693 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5694                                           struct event_trigger_data *data)
5695 {
5696         if (WARN_ON_ONCE(data->ref <= 0))
5697                 return;
5698
5699         event_hist_trigger_free(ops, data->named_data);
5700
5701         data->ref--;
5702         if (!data->ref) {
5703                 del_named_trigger(data);
5704                 trigger_data_free(data);
5705         }
5706 }
5707
5708 static struct event_trigger_ops event_hist_trigger_named_ops = {
5709         .func                   = event_hist_trigger,
5710         .print                  = event_hist_trigger_print,
5711         .init                   = event_hist_trigger_named_init,
5712         .free                   = event_hist_trigger_named_free,
5713 };
5714
5715 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5716                                                             char *param)
5717 {
5718         return &event_hist_trigger_ops;
5719 }
5720
5721 static void hist_clear(struct event_trigger_data *data)
5722 {
5723         struct hist_trigger_data *hist_data = data->private_data;
5724
5725         if (data->name)
5726                 pause_named_trigger(data);
5727
5728         tracepoint_synchronize_unregister();
5729
5730         tracing_map_clear(hist_data->map);
5731
5732         if (data->name)
5733                 unpause_named_trigger(data);
5734 }
5735
5736 static bool compatible_field(struct ftrace_event_field *field,
5737                              struct ftrace_event_field *test_field)
5738 {
5739         if (field == test_field)
5740                 return true;
5741         if (field == NULL || test_field == NULL)
5742                 return false;
5743         if (strcmp(field->name, test_field->name) != 0)
5744                 return false;
5745         if (strcmp(field->type, test_field->type) != 0)
5746                 return false;
5747         if (field->size != test_field->size)
5748                 return false;
5749         if (field->is_signed != test_field->is_signed)
5750                 return false;
5751
5752         return true;
5753 }
5754
5755 static bool hist_trigger_match(struct event_trigger_data *data,
5756                                struct event_trigger_data *data_test,
5757                                struct event_trigger_data *named_data,
5758                                bool ignore_filter)
5759 {
5760         struct tracing_map_sort_key *sort_key, *sort_key_test;
5761         struct hist_trigger_data *hist_data, *hist_data_test;
5762         struct hist_field *key_field, *key_field_test;
5763         unsigned int i;
5764
5765         if (named_data && (named_data != data_test) &&
5766             (named_data != data_test->named_data))
5767                 return false;
5768
5769         if (!named_data && is_named_trigger(data_test))
5770                 return false;
5771
5772         hist_data = data->private_data;
5773         hist_data_test = data_test->private_data;
5774
5775         if (hist_data->n_vals != hist_data_test->n_vals ||
5776             hist_data->n_fields != hist_data_test->n_fields ||
5777             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5778                 return false;
5779
5780         if (!ignore_filter) {
5781                 if ((data->filter_str && !data_test->filter_str) ||
5782                    (!data->filter_str && data_test->filter_str))
5783                         return false;
5784         }
5785
5786         for_each_hist_field(i, hist_data) {
5787                 key_field = hist_data->fields[i];
5788                 key_field_test = hist_data_test->fields[i];
5789
5790                 if (key_field->flags != key_field_test->flags)
5791                         return false;
5792                 if (!compatible_field(key_field->field, key_field_test->field))
5793                         return false;
5794                 if (key_field->offset != key_field_test->offset)
5795                         return false;
5796                 if (key_field->size != key_field_test->size)
5797                         return false;
5798                 if (key_field->is_signed != key_field_test->is_signed)
5799                         return false;
5800                 if (!!key_field->var.name != !!key_field_test->var.name)
5801                         return false;
5802                 if (key_field->var.name &&
5803                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5804                         return false;
5805         }
5806
5807         for (i = 0; i < hist_data->n_sort_keys; i++) {
5808                 sort_key = &hist_data->sort_keys[i];
5809                 sort_key_test = &hist_data_test->sort_keys[i];
5810
5811                 if (sort_key->field_idx != sort_key_test->field_idx ||
5812                     sort_key->descending != sort_key_test->descending)
5813                         return false;
5814         }
5815
5816         if (!ignore_filter && data->filter_str &&
5817             (strcmp(data->filter_str, data_test->filter_str) != 0))
5818                 return false;
5819
5820         if (!actions_match(hist_data, hist_data_test))
5821                 return false;
5822
5823         return true;
5824 }
5825
5826 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5827                                  struct event_trigger_data *data,
5828                                  struct trace_event_file *file)
5829 {
5830         struct hist_trigger_data *hist_data = data->private_data;
5831         struct event_trigger_data *test, *named_data = NULL;
5832         int ret = 0;
5833
5834         if (hist_data->attrs->name) {
5835                 named_data = find_named_trigger(hist_data->attrs->name);
5836                 if (named_data) {
5837                         if (!hist_trigger_match(data, named_data, named_data,
5838                                                 true)) {
5839                                 hist_err(HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5840                                 ret = -EINVAL;
5841                                 goto out;
5842                         }
5843                 }
5844         }
5845
5846         if (hist_data->attrs->name && !named_data)
5847                 goto new;
5848
5849         list_for_each_entry_rcu(test, &file->triggers, list) {
5850                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5851                         if (!hist_trigger_match(data, test, named_data, false))
5852                                 continue;
5853                         if (hist_data->attrs->pause)
5854                                 test->paused = true;
5855                         else if (hist_data->attrs->cont)
5856                                 test->paused = false;
5857                         else if (hist_data->attrs->clear)
5858                                 hist_clear(test);
5859                         else {
5860                                 hist_err(HIST_ERR_TRIGGER_EEXIST, 0);
5861                                 ret = -EEXIST;
5862                         }
5863                         goto out;
5864                 }
5865         }
5866  new:
5867         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5868                 hist_err(HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5869                 ret = -ENOENT;
5870                 goto out;
5871         }
5872
5873         if (hist_data->attrs->pause)
5874                 data->paused = true;
5875
5876         if (named_data) {
5877                 data->private_data = named_data->private_data;
5878                 set_named_trigger_data(data, named_data);
5879                 data->ops = &event_hist_trigger_named_ops;
5880         }
5881
5882         if (data->ops->init) {
5883                 ret = data->ops->init(data->ops, data);
5884                 if (ret < 0)
5885                         goto out;
5886         }
5887
5888         if (hist_data->enable_timestamps) {
5889                 char *clock = hist_data->attrs->clock;
5890
5891                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5892                 if (ret) {
5893                         hist_err(HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5894                         goto out;
5895                 }
5896
5897                 tracing_set_time_stamp_abs(file->tr, true);
5898         }
5899
5900         if (named_data)
5901                 destroy_hist_data(hist_data);
5902
5903         ret++;
5904  out:
5905         return ret;
5906 }
5907
5908 static int hist_trigger_enable(struct event_trigger_data *data,
5909                                struct trace_event_file *file)
5910 {
5911         int ret = 0;
5912
5913         list_add_tail_rcu(&data->list, &file->triggers);
5914
5915         update_cond_flag(file);
5916
5917         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5918                 list_del_rcu(&data->list);
5919                 update_cond_flag(file);
5920                 ret--;
5921         }
5922
5923         return ret;
5924 }
5925
5926 static bool have_hist_trigger_match(struct event_trigger_data *data,
5927                                     struct trace_event_file *file)
5928 {
5929         struct hist_trigger_data *hist_data = data->private_data;
5930         struct event_trigger_data *test, *named_data = NULL;
5931         bool match = false;
5932
5933         if (hist_data->attrs->name)
5934                 named_data = find_named_trigger(hist_data->attrs->name);
5935
5936         list_for_each_entry_rcu(test, &file->triggers, list) {
5937                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5938                         if (hist_trigger_match(data, test, named_data, false)) {
5939                                 match = true;
5940                                 break;
5941                         }
5942                 }
5943         }
5944
5945         return match;
5946 }
5947
5948 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5949                                     struct trace_event_file *file)
5950 {
5951         struct hist_trigger_data *hist_data = data->private_data;
5952         struct event_trigger_data *test, *named_data = NULL;
5953
5954         if (hist_data->attrs->name)
5955                 named_data = find_named_trigger(hist_data->attrs->name);
5956
5957         list_for_each_entry_rcu(test, &file->triggers, list) {
5958                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5959                         if (!hist_trigger_match(data, test, named_data, false))
5960                                 continue;
5961                         hist_data = test->private_data;
5962                         if (check_var_refs(hist_data))
5963                                 return true;
5964                         break;
5965                 }
5966         }
5967
5968         return false;
5969 }
5970
5971 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5972                                     struct event_trigger_data *data,
5973                                     struct trace_event_file *file)
5974 {
5975         struct hist_trigger_data *hist_data = data->private_data;
5976         struct event_trigger_data *test, *named_data = NULL;
5977         bool unregistered = false;
5978
5979         if (hist_data->attrs->name)
5980                 named_data = find_named_trigger(hist_data->attrs->name);
5981
5982         list_for_each_entry_rcu(test, &file->triggers, list) {
5983                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5984                         if (!hist_trigger_match(data, test, named_data, false))
5985                                 continue;
5986                         unregistered = true;
5987                         list_del_rcu(&test->list);
5988                         trace_event_trigger_enable_disable(file, 0);
5989                         update_cond_flag(file);
5990                         break;
5991                 }
5992         }
5993
5994         if (unregistered && test->ops->free)
5995                 test->ops->free(test->ops, test);
5996
5997         if (hist_data->enable_timestamps) {
5998                 if (!hist_data->remove || unregistered)
5999                         tracing_set_time_stamp_abs(file->tr, false);
6000         }
6001 }
6002
6003 static bool hist_file_check_refs(struct trace_event_file *file)
6004 {
6005         struct hist_trigger_data *hist_data;
6006         struct event_trigger_data *test;
6007
6008         list_for_each_entry_rcu(test, &file->triggers, list) {
6009                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6010                         hist_data = test->private_data;
6011                         if (check_var_refs(hist_data))
6012                                 return true;
6013                 }
6014         }
6015
6016         return false;
6017 }
6018
6019 static void hist_unreg_all(struct trace_event_file *file)
6020 {
6021         struct event_trigger_data *test, *n;
6022         struct hist_trigger_data *hist_data;
6023         struct synth_event *se;
6024         const char *se_name;
6025
6026         lockdep_assert_held(&event_mutex);
6027
6028         if (hist_file_check_refs(file))
6029                 return;
6030
6031         list_for_each_entry_safe(test, n, &file->triggers, list) {
6032                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6033                         hist_data = test->private_data;
6034                         list_del_rcu(&test->list);
6035                         trace_event_trigger_enable_disable(file, 0);
6036
6037                         se_name = trace_event_name(file->event_call);
6038                         se = find_synth_event(se_name);
6039                         if (se)
6040                                 se->ref--;
6041
6042                         update_cond_flag(file);
6043                         if (hist_data->enable_timestamps)
6044                                 tracing_set_time_stamp_abs(file->tr, false);
6045                         if (test->ops->free)
6046                                 test->ops->free(test->ops, test);
6047                 }
6048         }
6049 }
6050
6051 static int event_hist_trigger_func(struct event_command *cmd_ops,
6052                                    struct trace_event_file *file,
6053                                    char *glob, char *cmd, char *param)
6054 {
6055         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6056         struct event_trigger_data *trigger_data;
6057         struct hist_trigger_attrs *attrs;
6058         struct event_trigger_ops *trigger_ops;
6059         struct hist_trigger_data *hist_data;
6060         struct synth_event *se;
6061         const char *se_name;
6062         bool remove = false;
6063         char *trigger, *p;
6064         int ret = 0;
6065
6066         lockdep_assert_held(&event_mutex);
6067
6068         if (glob && strlen(glob)) {
6069                 hist_err_clear();
6070                 last_cmd_set(file, param);
6071         }
6072
6073         if (!param)
6074                 return -EINVAL;
6075
6076         if (glob[0] == '!')
6077                 remove = true;
6078
6079         /*
6080          * separate the trigger from the filter (k:v [if filter])
6081          * allowing for whitespace in the trigger
6082          */
6083         p = trigger = param;
6084         do {
6085                 p = strstr(p, "if");
6086                 if (!p)
6087                         break;
6088                 if (p == param)
6089                         return -EINVAL;
6090                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6091                         p++;
6092                         continue;
6093                 }
6094                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6095                         return -EINVAL;
6096                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6097                         p++;
6098                         continue;
6099                 }
6100                 break;
6101         } while (p);
6102
6103         if (!p)
6104                 param = NULL;
6105         else {
6106                 *(p - 1) = '\0';
6107                 param = strstrip(p);
6108                 trigger = strstrip(trigger);
6109         }
6110
6111         attrs = parse_hist_trigger_attrs(trigger);
6112         if (IS_ERR(attrs))
6113                 return PTR_ERR(attrs);
6114
6115         if (attrs->map_bits)
6116                 hist_trigger_bits = attrs->map_bits;
6117
6118         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6119         if (IS_ERR(hist_data)) {
6120                 destroy_hist_trigger_attrs(attrs);
6121                 return PTR_ERR(hist_data);
6122         }
6123
6124         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6125
6126         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6127         if (!trigger_data) {
6128                 ret = -ENOMEM;
6129                 goto out_free;
6130         }
6131
6132         trigger_data->count = -1;
6133         trigger_data->ops = trigger_ops;
6134         trigger_data->cmd_ops = cmd_ops;
6135
6136         INIT_LIST_HEAD(&trigger_data->list);
6137         RCU_INIT_POINTER(trigger_data->filter, NULL);
6138
6139         trigger_data->private_data = hist_data;
6140
6141         /* if param is non-empty, it's supposed to be a filter */
6142         if (param && cmd_ops->set_filter) {
6143                 ret = cmd_ops->set_filter(param, trigger_data, file);
6144                 if (ret < 0)
6145                         goto out_free;
6146         }
6147
6148         if (remove) {
6149                 if (!have_hist_trigger_match(trigger_data, file))
6150                         goto out_free;
6151
6152                 if (hist_trigger_check_refs(trigger_data, file)) {
6153                         ret = -EBUSY;
6154                         goto out_free;
6155                 }
6156
6157                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6158                 se_name = trace_event_name(file->event_call);
6159                 se = find_synth_event(se_name);
6160                 if (se)
6161                         se->ref--;
6162                 ret = 0;
6163                 goto out_free;
6164         }
6165
6166         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6167         /*
6168          * The above returns on success the # of triggers registered,
6169          * but if it didn't register any it returns zero.  Consider no
6170          * triggers registered a failure too.
6171          */
6172         if (!ret) {
6173                 if (!(attrs->pause || attrs->cont || attrs->clear))
6174                         ret = -ENOENT;
6175                 goto out_free;
6176         } else if (ret < 0)
6177                 goto out_free;
6178
6179         if (get_named_trigger_data(trigger_data))
6180                 goto enable;
6181
6182         if (has_hist_vars(hist_data))
6183                 save_hist_vars(hist_data);
6184
6185         ret = create_actions(hist_data);
6186         if (ret)
6187                 goto out_unreg;
6188
6189         ret = tracing_map_init(hist_data->map);
6190         if (ret)
6191                 goto out_unreg;
6192 enable:
6193         ret = hist_trigger_enable(trigger_data, file);
6194         if (ret)
6195                 goto out_unreg;
6196
6197         se_name = trace_event_name(file->event_call);
6198         se = find_synth_event(se_name);
6199         if (se)
6200                 se->ref++;
6201         /* Just return zero, not the number of registered triggers */
6202         ret = 0;
6203  out:
6204         if (ret == 0)
6205                 hist_err_clear();
6206
6207         return ret;
6208  out_unreg:
6209         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6210  out_free:
6211         if (cmd_ops->set_filter)
6212                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6213
6214         remove_hist_vars(hist_data);
6215
6216         kfree(trigger_data);
6217
6218         destroy_hist_data(hist_data);
6219         goto out;
6220 }
6221
6222 static struct event_command trigger_hist_cmd = {
6223         .name                   = "hist",
6224         .trigger_type           = ETT_EVENT_HIST,
6225         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6226         .func                   = event_hist_trigger_func,
6227         .reg                    = hist_register_trigger,
6228         .unreg                  = hist_unregister_trigger,
6229         .unreg_all              = hist_unreg_all,
6230         .get_trigger_ops        = event_hist_get_trigger_ops,
6231         .set_filter             = set_trigger_filter,
6232 };
6233
6234 __init int register_trigger_hist_cmd(void)
6235 {
6236         int ret;
6237
6238         ret = register_event_command(&trigger_hist_cmd);
6239         WARN_ON(ret < 0);
6240
6241         return ret;
6242 }
6243
6244 static void
6245 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6246                     struct ring_buffer_event *event)
6247 {
6248         struct enable_trigger_data *enable_data = data->private_data;
6249         struct event_trigger_data *test;
6250
6251         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
6252                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6253                         if (enable_data->enable)
6254                                 test->paused = false;
6255                         else
6256                                 test->paused = true;
6257                 }
6258         }
6259 }
6260
6261 static void
6262 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6263                           struct ring_buffer_event *event)
6264 {
6265         if (!data->count)
6266                 return;
6267
6268         if (data->count != -1)
6269                 (data->count)--;
6270
6271         hist_enable_trigger(data, rec, event);
6272 }
6273
6274 static struct event_trigger_ops hist_enable_trigger_ops = {
6275         .func                   = hist_enable_trigger,
6276         .print                  = event_enable_trigger_print,
6277         .init                   = event_trigger_init,
6278         .free                   = event_enable_trigger_free,
6279 };
6280
6281 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6282         .func                   = hist_enable_count_trigger,
6283         .print                  = event_enable_trigger_print,
6284         .init                   = event_trigger_init,
6285         .free                   = event_enable_trigger_free,
6286 };
6287
6288 static struct event_trigger_ops hist_disable_trigger_ops = {
6289         .func                   = hist_enable_trigger,
6290         .print                  = event_enable_trigger_print,
6291         .init                   = event_trigger_init,
6292         .free                   = event_enable_trigger_free,
6293 };
6294
6295 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6296         .func                   = hist_enable_count_trigger,
6297         .print                  = event_enable_trigger_print,
6298         .init                   = event_trigger_init,
6299         .free                   = event_enable_trigger_free,
6300 };
6301
6302 static struct event_trigger_ops *
6303 hist_enable_get_trigger_ops(char *cmd, char *param)
6304 {
6305         struct event_trigger_ops *ops;
6306         bool enable;
6307
6308         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6309
6310         if (enable)
6311                 ops = param ? &hist_enable_count_trigger_ops :
6312                         &hist_enable_trigger_ops;
6313         else
6314                 ops = param ? &hist_disable_count_trigger_ops :
6315                         &hist_disable_trigger_ops;
6316
6317         return ops;
6318 }
6319
6320 static void hist_enable_unreg_all(struct trace_event_file *file)
6321 {
6322         struct event_trigger_data *test, *n;
6323
6324         list_for_each_entry_safe(test, n, &file->triggers, list) {
6325                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6326                         list_del_rcu(&test->list);
6327                         update_cond_flag(file);
6328                         trace_event_trigger_enable_disable(file, 0);
6329                         if (test->ops->free)
6330                                 test->ops->free(test->ops, test);
6331                 }
6332         }
6333 }
6334
6335 static struct event_command trigger_hist_enable_cmd = {
6336         .name                   = ENABLE_HIST_STR,
6337         .trigger_type           = ETT_HIST_ENABLE,
6338         .func                   = event_enable_trigger_func,
6339         .reg                    = event_enable_register_trigger,
6340         .unreg                  = event_enable_unregister_trigger,
6341         .unreg_all              = hist_enable_unreg_all,
6342         .get_trigger_ops        = hist_enable_get_trigger_ops,
6343         .set_filter             = set_trigger_filter,
6344 };
6345
6346 static struct event_command trigger_hist_disable_cmd = {
6347         .name                   = DISABLE_HIST_STR,
6348         .trigger_type           = ETT_HIST_ENABLE,
6349         .func                   = event_enable_trigger_func,
6350         .reg                    = event_enable_register_trigger,
6351         .unreg                  = event_enable_unregister_trigger,
6352         .unreg_all              = hist_enable_unreg_all,
6353         .get_trigger_ops        = hist_enable_get_trigger_ops,
6354         .set_filter             = set_trigger_filter,
6355 };
6356
6357 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6358 {
6359         unregister_event_command(&trigger_hist_enable_cmd);
6360         unregister_event_command(&trigger_hist_disable_cmd);
6361 }
6362
6363 __init int register_trigger_hist_enable_disable_cmds(void)
6364 {
6365         int ret;
6366
6367         ret = register_event_command(&trigger_hist_enable_cmd);
6368         if (WARN_ON(ret < 0))
6369                 return ret;
6370         ret = register_event_command(&trigger_hist_disable_cmd);
6371         if (WARN_ON(ret < 0))
6372                 unregister_trigger_hist_enable_disable_cmds();
6373
6374         return ret;
6375 }
6376
6377 static __init int trace_events_hist_init(void)
6378 {
6379         struct dentry *entry = NULL;
6380         struct dentry *d_tracer;
6381         int err = 0;
6382
6383         err = dyn_event_register(&synth_event_ops);
6384         if (err) {
6385                 pr_warn("Could not register synth_event_ops\n");
6386                 return err;
6387         }
6388
6389         d_tracer = tracing_init_dentry();
6390         if (IS_ERR(d_tracer)) {
6391                 err = PTR_ERR(d_tracer);
6392                 goto err;
6393         }
6394
6395         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6396                                     NULL, &synth_events_fops);
6397         if (!entry) {
6398                 err = -ENODEV;
6399                 goto err;
6400         }
6401
6402         return err;
6403  err:
6404         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6405
6406         return err;
6407 }
6408
6409 fs_initcall(trace_events_hist_init);