Merge tag 'iomap-5.14-fixes-1' of git://git.kernel.org/pub/scm/fs/xfs/xfs-linux
[linux-2.6-microblaze.git] / kernel / trace / trace_events_hist.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * trace_events_hist - trace event hist triggers
4  *
5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6  */
7
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/security.h>
11 #include <linux/mutex.h>
12 #include <linux/slab.h>
13 #include <linux/stacktrace.h>
14 #include <linux/rculist.h>
15 #include <linux/tracefs.h>
16
17 /* for gfp flag names */
18 #include <linux/trace_events.h>
19 #include <trace/events/mmflags.h>
20
21 #include "tracing_map.h"
22 #include "trace_synth.h"
23
24 #define ERRORS                                                          \
25         C(NONE,                 "No error"),                            \
26         C(DUPLICATE_VAR,        "Variable already defined"),            \
27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
53         C(INVALID_PARAM,        "Invalid action param"),                \
54         C(ACTION_NOT_FOUND,     "No action found"),                     \
55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
69
70 #undef C
71 #define C(a, b)         HIST_ERR_##a
72
73 enum { ERRORS };
74
75 #undef C
76 #define C(a, b)         b
77
78 static const char *err_text[] = { ERRORS };
79
80 struct hist_field;
81
82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
83                                 struct tracing_map_elt *elt,
84                                 struct trace_buffer *buffer,
85                                 struct ring_buffer_event *rbe,
86                                 void *event);
87
88 #define HIST_FIELD_OPERANDS_MAX 2
89 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
90 #define HIST_ACTIONS_MAX        8
91
92 enum field_op_id {
93         FIELD_OP_NONE,
94         FIELD_OP_PLUS,
95         FIELD_OP_MINUS,
96         FIELD_OP_UNARY_MINUS,
97 };
98
99 /*
100  * A hist_var (histogram variable) contains variable information for
101  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
102  * flag set.  A hist_var has a variable name e.g. ts0, and is
103  * associated with a given histogram trigger, as specified by
104  * hist_data.  The hist_var idx is the unique index assigned to the
105  * variable by the hist trigger's tracing_map.  The idx is what is
106  * used to set a variable's value and, by a variable reference, to
107  * retrieve it.
108  */
109 struct hist_var {
110         char                            *name;
111         struct hist_trigger_data        *hist_data;
112         unsigned int                    idx;
113 };
114
115 struct hist_field {
116         struct ftrace_event_field       *field;
117         unsigned long                   flags;
118         hist_field_fn_t                 fn;
119         unsigned int                    ref;
120         unsigned int                    size;
121         unsigned int                    offset;
122         unsigned int                    is_signed;
123         const char                      *type;
124         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
125         struct hist_trigger_data        *hist_data;
126
127         /*
128          * Variable fields contain variable-specific info in var.
129          */
130         struct hist_var                 var;
131         enum field_op_id                operator;
132         char                            *system;
133         char                            *event_name;
134
135         /*
136          * The name field is used for EXPR and VAR_REF fields.  VAR
137          * fields contain the variable name in var.name.
138          */
139         char                            *name;
140
141         /*
142          * When a histogram trigger is hit, if it has any references
143          * to variables, the values of those variables are collected
144          * into a var_ref_vals array by resolve_var_refs().  The
145          * current value of each variable is read from the tracing_map
146          * using the hist field's hist_var.idx and entered into the
147          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
148          */
149         unsigned int                    var_ref_idx;
150         bool                            read_once;
151
152         unsigned int                    var_str_idx;
153 };
154
155 static u64 hist_field_none(struct hist_field *field,
156                            struct tracing_map_elt *elt,
157                            struct trace_buffer *buffer,
158                            struct ring_buffer_event *rbe,
159                            void *event)
160 {
161         return 0;
162 }
163
164 static u64 hist_field_counter(struct hist_field *field,
165                               struct tracing_map_elt *elt,
166                               struct trace_buffer *buffer,
167                               struct ring_buffer_event *rbe,
168                               void *event)
169 {
170         return 1;
171 }
172
173 static u64 hist_field_string(struct hist_field *hist_field,
174                              struct tracing_map_elt *elt,
175                              struct trace_buffer *buffer,
176                              struct ring_buffer_event *rbe,
177                              void *event)
178 {
179         char *addr = (char *)(event + hist_field->field->offset);
180
181         return (u64)(unsigned long)addr;
182 }
183
184 static u64 hist_field_dynstring(struct hist_field *hist_field,
185                                 struct tracing_map_elt *elt,
186                                 struct trace_buffer *buffer,
187                                 struct ring_buffer_event *rbe,
188                                 void *event)
189 {
190         u32 str_item = *(u32 *)(event + hist_field->field->offset);
191         int str_loc = str_item & 0xffff;
192         char *addr = (char *)(event + str_loc);
193
194         return (u64)(unsigned long)addr;
195 }
196
197 static u64 hist_field_pstring(struct hist_field *hist_field,
198                               struct tracing_map_elt *elt,
199                               struct trace_buffer *buffer,
200                               struct ring_buffer_event *rbe,
201                               void *event)
202 {
203         char **addr = (char **)(event + hist_field->field->offset);
204
205         return (u64)(unsigned long)*addr;
206 }
207
208 static u64 hist_field_log2(struct hist_field *hist_field,
209                            struct tracing_map_elt *elt,
210                            struct trace_buffer *buffer,
211                            struct ring_buffer_event *rbe,
212                            void *event)
213 {
214         struct hist_field *operand = hist_field->operands[0];
215
216         u64 val = operand->fn(operand, elt, buffer, rbe, event);
217
218         return (u64) ilog2(roundup_pow_of_two(val));
219 }
220
221 static u64 hist_field_plus(struct hist_field *hist_field,
222                            struct tracing_map_elt *elt,
223                            struct trace_buffer *buffer,
224                            struct ring_buffer_event *rbe,
225                            void *event)
226 {
227         struct hist_field *operand1 = hist_field->operands[0];
228         struct hist_field *operand2 = hist_field->operands[1];
229
230         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
231         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
232
233         return val1 + val2;
234 }
235
236 static u64 hist_field_minus(struct hist_field *hist_field,
237                             struct tracing_map_elt *elt,
238                             struct trace_buffer *buffer,
239                             struct ring_buffer_event *rbe,
240                             void *event)
241 {
242         struct hist_field *operand1 = hist_field->operands[0];
243         struct hist_field *operand2 = hist_field->operands[1];
244
245         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
246         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
247
248         return val1 - val2;
249 }
250
251 static u64 hist_field_unary_minus(struct hist_field *hist_field,
252                                   struct tracing_map_elt *elt,
253                                   struct trace_buffer *buffer,
254                                   struct ring_buffer_event *rbe,
255                                   void *event)
256 {
257         struct hist_field *operand = hist_field->operands[0];
258
259         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
260         u64 val = (u64)-sval;
261
262         return val;
263 }
264
265 #define DEFINE_HIST_FIELD_FN(type)                                      \
266         static u64 hist_field_##type(struct hist_field *hist_field,     \
267                                      struct tracing_map_elt *elt,       \
268                                      struct trace_buffer *buffer,       \
269                                      struct ring_buffer_event *rbe,     \
270                                      void *event)                       \
271 {                                                                       \
272         type *addr = (type *)(event + hist_field->field->offset);       \
273                                                                         \
274         return (u64)(unsigned long)*addr;                               \
275 }
276
277 DEFINE_HIST_FIELD_FN(s64);
278 DEFINE_HIST_FIELD_FN(u64);
279 DEFINE_HIST_FIELD_FN(s32);
280 DEFINE_HIST_FIELD_FN(u32);
281 DEFINE_HIST_FIELD_FN(s16);
282 DEFINE_HIST_FIELD_FN(u16);
283 DEFINE_HIST_FIELD_FN(s8);
284 DEFINE_HIST_FIELD_FN(u8);
285
286 #define for_each_hist_field(i, hist_data)       \
287         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
288
289 #define for_each_hist_val_field(i, hist_data)   \
290         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
291
292 #define for_each_hist_key_field(i, hist_data)   \
293         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
294
295 #define HIST_STACKTRACE_DEPTH   16
296 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
297 #define HIST_STACKTRACE_SKIP    5
298
299 #define HITCOUNT_IDX            0
300 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
301
302 enum hist_field_flags {
303         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
304         HIST_FIELD_FL_KEY               = 1 << 1,
305         HIST_FIELD_FL_STRING            = 1 << 2,
306         HIST_FIELD_FL_HEX               = 1 << 3,
307         HIST_FIELD_FL_SYM               = 1 << 4,
308         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
309         HIST_FIELD_FL_EXECNAME          = 1 << 6,
310         HIST_FIELD_FL_SYSCALL           = 1 << 7,
311         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
312         HIST_FIELD_FL_LOG2              = 1 << 9,
313         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
314         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
315         HIST_FIELD_FL_VAR               = 1 << 12,
316         HIST_FIELD_FL_EXPR              = 1 << 13,
317         HIST_FIELD_FL_VAR_REF           = 1 << 14,
318         HIST_FIELD_FL_CPU               = 1 << 15,
319         HIST_FIELD_FL_ALIAS             = 1 << 16,
320 };
321
322 struct var_defs {
323         unsigned int    n_vars;
324         char            *name[TRACING_MAP_VARS_MAX];
325         char            *expr[TRACING_MAP_VARS_MAX];
326 };
327
328 struct hist_trigger_attrs {
329         char            *keys_str;
330         char            *vals_str;
331         char            *sort_key_str;
332         char            *name;
333         char            *clock;
334         bool            pause;
335         bool            cont;
336         bool            clear;
337         bool            ts_in_usecs;
338         unsigned int    map_bits;
339
340         char            *assignment_str[TRACING_MAP_VARS_MAX];
341         unsigned int    n_assignments;
342
343         char            *action_str[HIST_ACTIONS_MAX];
344         unsigned int    n_actions;
345
346         struct var_defs var_defs;
347 };
348
349 struct field_var {
350         struct hist_field       *var;
351         struct hist_field       *val;
352 };
353
354 struct field_var_hist {
355         struct hist_trigger_data        *hist_data;
356         char                            *cmd;
357 };
358
359 struct hist_trigger_data {
360         struct hist_field               *fields[HIST_FIELDS_MAX];
361         unsigned int                    n_vals;
362         unsigned int                    n_keys;
363         unsigned int                    n_fields;
364         unsigned int                    n_vars;
365         unsigned int                    n_var_str;
366         unsigned int                    key_size;
367         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
368         unsigned int                    n_sort_keys;
369         struct trace_event_file         *event_file;
370         struct hist_trigger_attrs       *attrs;
371         struct tracing_map              *map;
372         bool                            enable_timestamps;
373         bool                            remove;
374         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
375         unsigned int                    n_var_refs;
376
377         struct action_data              *actions[HIST_ACTIONS_MAX];
378         unsigned int                    n_actions;
379
380         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
381         unsigned int                    n_field_vars;
382         unsigned int                    n_field_var_str;
383         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
384         unsigned int                    n_field_var_hists;
385
386         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
387         unsigned int                    n_save_vars;
388         unsigned int                    n_save_var_str;
389 };
390
391 struct action_data;
392
393 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
394                              struct tracing_map_elt *elt,
395                              struct trace_buffer *buffer, void *rec,
396                              struct ring_buffer_event *rbe, void *key,
397                              struct action_data *data, u64 *var_ref_vals);
398
399 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
400
401 enum handler_id {
402         HANDLER_ONMATCH = 1,
403         HANDLER_ONMAX,
404         HANDLER_ONCHANGE,
405 };
406
407 enum action_id {
408         ACTION_SAVE = 1,
409         ACTION_TRACE,
410         ACTION_SNAPSHOT,
411 };
412
413 struct action_data {
414         enum handler_id         handler;
415         enum action_id          action;
416         char                    *action_name;
417         action_fn_t             fn;
418
419         unsigned int            n_params;
420         char                    *params[SYNTH_FIELDS_MAX];
421
422         /*
423          * When a histogram trigger is hit, the values of any
424          * references to variables, including variables being passed
425          * as parameters to synthetic events, are collected into a
426          * var_ref_vals array.  This var_ref_idx array is an array of
427          * indices into the var_ref_vals array, one for each synthetic
428          * event param, and is passed to the synthetic event
429          * invocation.
430          */
431         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
432         struct synth_event      *synth_event;
433         bool                    use_trace_keyword;
434         char                    *synth_event_name;
435
436         union {
437                 struct {
438                         char                    *event;
439                         char                    *event_system;
440                 } match_data;
441
442                 struct {
443                         /*
444                          * var_str contains the $-unstripped variable
445                          * name referenced by var_ref, and used when
446                          * printing the action.  Because var_ref
447                          * creation is deferred to create_actions(),
448                          * we need a per-action way to save it until
449                          * then, thus var_str.
450                          */
451                         char                    *var_str;
452
453                         /*
454                          * var_ref refers to the variable being
455                          * tracked e.g onmax($var).
456                          */
457                         struct hist_field       *var_ref;
458
459                         /*
460                          * track_var contains the 'invisible' tracking
461                          * variable created to keep the current
462                          * e.g. max value.
463                          */
464                         struct hist_field       *track_var;
465
466                         check_track_val_fn_t    check_val;
467                         action_fn_t             save_data;
468                 } track_data;
469         };
470 };
471
472 struct track_data {
473         u64                             track_val;
474         bool                            updated;
475
476         unsigned int                    key_len;
477         void                            *key;
478         struct tracing_map_elt          elt;
479
480         struct action_data              *action_data;
481         struct hist_trigger_data        *hist_data;
482 };
483
484 struct hist_elt_data {
485         char *comm;
486         u64 *var_ref_vals;
487         char *field_var_str[SYNTH_FIELDS_MAX];
488 };
489
490 struct snapshot_context {
491         struct tracing_map_elt  *elt;
492         void                    *key;
493 };
494
495 static void track_data_free(struct track_data *track_data)
496 {
497         struct hist_elt_data *elt_data;
498
499         if (!track_data)
500                 return;
501
502         kfree(track_data->key);
503
504         elt_data = track_data->elt.private_data;
505         if (elt_data) {
506                 kfree(elt_data->comm);
507                 kfree(elt_data);
508         }
509
510         kfree(track_data);
511 }
512
513 static struct track_data *track_data_alloc(unsigned int key_len,
514                                            struct action_data *action_data,
515                                            struct hist_trigger_data *hist_data)
516 {
517         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
518         struct hist_elt_data *elt_data;
519
520         if (!data)
521                 return ERR_PTR(-ENOMEM);
522
523         data->key = kzalloc(key_len, GFP_KERNEL);
524         if (!data->key) {
525                 track_data_free(data);
526                 return ERR_PTR(-ENOMEM);
527         }
528
529         data->key_len = key_len;
530         data->action_data = action_data;
531         data->hist_data = hist_data;
532
533         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
534         if (!elt_data) {
535                 track_data_free(data);
536                 return ERR_PTR(-ENOMEM);
537         }
538
539         data->elt.private_data = elt_data;
540
541         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
542         if (!elt_data->comm) {
543                 track_data_free(data);
544                 return ERR_PTR(-ENOMEM);
545         }
546
547         return data;
548 }
549
550 static char last_cmd[MAX_FILTER_STR_VAL];
551 static char last_cmd_loc[MAX_FILTER_STR_VAL];
552
553 static int errpos(char *str)
554 {
555         return err_pos(last_cmd, str);
556 }
557
558 static void last_cmd_set(struct trace_event_file *file, char *str)
559 {
560         const char *system = NULL, *name = NULL;
561         struct trace_event_call *call;
562
563         if (!str)
564                 return;
565
566         strcpy(last_cmd, "hist:");
567         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
568
569         if (file) {
570                 call = file->event_call;
571                 system = call->class->system;
572                 if (system) {
573                         name = trace_event_name(call);
574                         if (!name)
575                                 system = NULL;
576                 }
577         }
578
579         if (system)
580                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
581 }
582
583 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
584 {
585         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
586                         err_type, err_pos);
587 }
588
589 static void hist_err_clear(void)
590 {
591         last_cmd[0] = '\0';
592         last_cmd_loc[0] = '\0';
593 }
594
595 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
596                                     unsigned int *var_ref_idx);
597
598 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
599                                unsigned int *var_ref_idx)
600 {
601         struct tracepoint *tp = event->tp;
602
603         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
604                 struct tracepoint_func *probe_func_ptr;
605                 synth_probe_func_t probe_func;
606                 void *__data;
607
608                 if (!(cpu_online(raw_smp_processor_id())))
609                         return;
610
611                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
612                 if (probe_func_ptr) {
613                         do {
614                                 probe_func = probe_func_ptr->func;
615                                 __data = probe_func_ptr->data;
616                                 probe_func(__data, var_ref_vals, var_ref_idx);
617                         } while ((++probe_func_ptr)->func);
618                 }
619         }
620 }
621
622 static void action_trace(struct hist_trigger_data *hist_data,
623                          struct tracing_map_elt *elt,
624                          struct trace_buffer *buffer, void *rec,
625                          struct ring_buffer_event *rbe, void *key,
626                          struct action_data *data, u64 *var_ref_vals)
627 {
628         struct synth_event *event = data->synth_event;
629
630         trace_synth(event, var_ref_vals, data->var_ref_idx);
631 }
632
633 struct hist_var_data {
634         struct list_head list;
635         struct hist_trigger_data *hist_data;
636 };
637
638 static u64 hist_field_timestamp(struct hist_field *hist_field,
639                                 struct tracing_map_elt *elt,
640                                 struct trace_buffer *buffer,
641                                 struct ring_buffer_event *rbe,
642                                 void *event)
643 {
644         struct hist_trigger_data *hist_data = hist_field->hist_data;
645         struct trace_array *tr = hist_data->event_file->tr;
646
647         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
648
649         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
650                 ts = ns2usecs(ts);
651
652         return ts;
653 }
654
655 static u64 hist_field_cpu(struct hist_field *hist_field,
656                           struct tracing_map_elt *elt,
657                           struct trace_buffer *buffer,
658                           struct ring_buffer_event *rbe,
659                           void *event)
660 {
661         int cpu = smp_processor_id();
662
663         return cpu;
664 }
665
666 /**
667  * check_field_for_var_ref - Check if a VAR_REF field references a variable
668  * @hist_field: The VAR_REF field to check
669  * @var_data: The hist trigger that owns the variable
670  * @var_idx: The trigger variable identifier
671  *
672  * Check the given VAR_REF field to see whether or not it references
673  * the given variable associated with the given trigger.
674  *
675  * Return: The VAR_REF field if it does reference the variable, NULL if not
676  */
677 static struct hist_field *
678 check_field_for_var_ref(struct hist_field *hist_field,
679                         struct hist_trigger_data *var_data,
680                         unsigned int var_idx)
681 {
682         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
683
684         if (hist_field && hist_field->var.idx == var_idx &&
685             hist_field->var.hist_data == var_data)
686                 return hist_field;
687
688         return NULL;
689 }
690
691 /**
692  * find_var_ref - Check if a trigger has a reference to a trigger variable
693  * @hist_data: The hist trigger that might have a reference to the variable
694  * @var_data: The hist trigger that owns the variable
695  * @var_idx: The trigger variable identifier
696  *
697  * Check the list of var_refs[] on the first hist trigger to see
698  * whether any of them are references to the variable on the second
699  * trigger.
700  *
701  * Return: The VAR_REF field referencing the variable if so, NULL if not
702  */
703 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
704                                        struct hist_trigger_data *var_data,
705                                        unsigned int var_idx)
706 {
707         struct hist_field *hist_field;
708         unsigned int i;
709
710         for (i = 0; i < hist_data->n_var_refs; i++) {
711                 hist_field = hist_data->var_refs[i];
712                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
713                         return hist_field;
714         }
715
716         return NULL;
717 }
718
719 /**
720  * find_any_var_ref - Check if there is a reference to a given trigger variable
721  * @hist_data: The hist trigger
722  * @var_idx: The trigger variable identifier
723  *
724  * Check to see whether the given variable is currently referenced by
725  * any other trigger.
726  *
727  * The trigger the variable is defined on is explicitly excluded - the
728  * assumption being that a self-reference doesn't prevent a trigger
729  * from being removed.
730  *
731  * Return: The VAR_REF field referencing the variable if so, NULL if not
732  */
733 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
734                                            unsigned int var_idx)
735 {
736         struct trace_array *tr = hist_data->event_file->tr;
737         struct hist_field *found = NULL;
738         struct hist_var_data *var_data;
739
740         list_for_each_entry(var_data, &tr->hist_vars, list) {
741                 if (var_data->hist_data == hist_data)
742                         continue;
743                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
744                 if (found)
745                         break;
746         }
747
748         return found;
749 }
750
751 /**
752  * check_var_refs - Check if there is a reference to any of trigger's variables
753  * @hist_data: The hist trigger
754  *
755  * A trigger can define one or more variables.  If any one of them is
756  * currently referenced by any other trigger, this function will
757  * determine that.
758
759  * Typically used to determine whether or not a trigger can be removed
760  * - if there are any references to a trigger's variables, it cannot.
761  *
762  * Return: True if there is a reference to any of trigger's variables
763  */
764 static bool check_var_refs(struct hist_trigger_data *hist_data)
765 {
766         struct hist_field *field;
767         bool found = false;
768         int i;
769
770         for_each_hist_field(i, hist_data) {
771                 field = hist_data->fields[i];
772                 if (field && field->flags & HIST_FIELD_FL_VAR) {
773                         if (find_any_var_ref(hist_data, field->var.idx)) {
774                                 found = true;
775                                 break;
776                         }
777                 }
778         }
779
780         return found;
781 }
782
783 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
784 {
785         struct trace_array *tr = hist_data->event_file->tr;
786         struct hist_var_data *var_data, *found = NULL;
787
788         list_for_each_entry(var_data, &tr->hist_vars, list) {
789                 if (var_data->hist_data == hist_data) {
790                         found = var_data;
791                         break;
792                 }
793         }
794
795         return found;
796 }
797
798 static bool field_has_hist_vars(struct hist_field *hist_field,
799                                 unsigned int level)
800 {
801         int i;
802
803         if (level > 3)
804                 return false;
805
806         if (!hist_field)
807                 return false;
808
809         if (hist_field->flags & HIST_FIELD_FL_VAR ||
810             hist_field->flags & HIST_FIELD_FL_VAR_REF)
811                 return true;
812
813         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
814                 struct hist_field *operand;
815
816                 operand = hist_field->operands[i];
817                 if (field_has_hist_vars(operand, level + 1))
818                         return true;
819         }
820
821         return false;
822 }
823
824 static bool has_hist_vars(struct hist_trigger_data *hist_data)
825 {
826         struct hist_field *hist_field;
827         int i;
828
829         for_each_hist_field(i, hist_data) {
830                 hist_field = hist_data->fields[i];
831                 if (field_has_hist_vars(hist_field, 0))
832                         return true;
833         }
834
835         return false;
836 }
837
838 static int save_hist_vars(struct hist_trigger_data *hist_data)
839 {
840         struct trace_array *tr = hist_data->event_file->tr;
841         struct hist_var_data *var_data;
842
843         var_data = find_hist_vars(hist_data);
844         if (var_data)
845                 return 0;
846
847         if (tracing_check_open_get_tr(tr))
848                 return -ENODEV;
849
850         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
851         if (!var_data) {
852                 trace_array_put(tr);
853                 return -ENOMEM;
854         }
855
856         var_data->hist_data = hist_data;
857         list_add(&var_data->list, &tr->hist_vars);
858
859         return 0;
860 }
861
862 static void remove_hist_vars(struct hist_trigger_data *hist_data)
863 {
864         struct trace_array *tr = hist_data->event_file->tr;
865         struct hist_var_data *var_data;
866
867         var_data = find_hist_vars(hist_data);
868         if (!var_data)
869                 return;
870
871         if (WARN_ON(check_var_refs(hist_data)))
872                 return;
873
874         list_del(&var_data->list);
875
876         kfree(var_data);
877
878         trace_array_put(tr);
879 }
880
881 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
882                                          const char *var_name)
883 {
884         struct hist_field *hist_field, *found = NULL;
885         int i;
886
887         for_each_hist_field(i, hist_data) {
888                 hist_field = hist_data->fields[i];
889                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
890                     strcmp(hist_field->var.name, var_name) == 0) {
891                         found = hist_field;
892                         break;
893                 }
894         }
895
896         return found;
897 }
898
899 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
900                                    struct trace_event_file *file,
901                                    const char *var_name)
902 {
903         struct hist_trigger_data *test_data;
904         struct event_trigger_data *test;
905         struct hist_field *hist_field;
906
907         lockdep_assert_held(&event_mutex);
908
909         hist_field = find_var_field(hist_data, var_name);
910         if (hist_field)
911                 return hist_field;
912
913         list_for_each_entry(test, &file->triggers, list) {
914                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
915                         test_data = test->private_data;
916                         hist_field = find_var_field(test_data, var_name);
917                         if (hist_field)
918                                 return hist_field;
919                 }
920         }
921
922         return NULL;
923 }
924
925 static struct trace_event_file *find_var_file(struct trace_array *tr,
926                                               char *system,
927                                               char *event_name,
928                                               char *var_name)
929 {
930         struct hist_trigger_data *var_hist_data;
931         struct hist_var_data *var_data;
932         struct trace_event_file *file, *found = NULL;
933
934         if (system)
935                 return find_event_file(tr, system, event_name);
936
937         list_for_each_entry(var_data, &tr->hist_vars, list) {
938                 var_hist_data = var_data->hist_data;
939                 file = var_hist_data->event_file;
940                 if (file == found)
941                         continue;
942
943                 if (find_var_field(var_hist_data, var_name)) {
944                         if (found) {
945                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
946                                 return NULL;
947                         }
948
949                         found = file;
950                 }
951         }
952
953         return found;
954 }
955
956 static struct hist_field *find_file_var(struct trace_event_file *file,
957                                         const char *var_name)
958 {
959         struct hist_trigger_data *test_data;
960         struct event_trigger_data *test;
961         struct hist_field *hist_field;
962
963         lockdep_assert_held(&event_mutex);
964
965         list_for_each_entry(test, &file->triggers, list) {
966                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
967                         test_data = test->private_data;
968                         hist_field = find_var_field(test_data, var_name);
969                         if (hist_field)
970                                 return hist_field;
971                 }
972         }
973
974         return NULL;
975 }
976
977 static struct hist_field *
978 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
979 {
980         struct trace_array *tr = hist_data->event_file->tr;
981         struct hist_field *hist_field, *found = NULL;
982         struct trace_event_file *file;
983         unsigned int i;
984
985         for (i = 0; i < hist_data->n_actions; i++) {
986                 struct action_data *data = hist_data->actions[i];
987
988                 if (data->handler == HANDLER_ONMATCH) {
989                         char *system = data->match_data.event_system;
990                         char *event_name = data->match_data.event;
991
992                         file = find_var_file(tr, system, event_name, var_name);
993                         if (!file)
994                                 continue;
995                         hist_field = find_file_var(file, var_name);
996                         if (hist_field) {
997                                 if (found) {
998                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
999                                                  errpos(var_name));
1000                                         return ERR_PTR(-EINVAL);
1001                                 }
1002
1003                                 found = hist_field;
1004                         }
1005                 }
1006         }
1007         return found;
1008 }
1009
1010 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1011                                          char *system,
1012                                          char *event_name,
1013                                          char *var_name)
1014 {
1015         struct trace_array *tr = hist_data->event_file->tr;
1016         struct hist_field *hist_field = NULL;
1017         struct trace_event_file *file;
1018
1019         if (!system || !event_name) {
1020                 hist_field = find_match_var(hist_data, var_name);
1021                 if (IS_ERR(hist_field))
1022                         return NULL;
1023                 if (hist_field)
1024                         return hist_field;
1025         }
1026
1027         file = find_var_file(tr, system, event_name, var_name);
1028         if (!file)
1029                 return NULL;
1030
1031         hist_field = find_file_var(file, var_name);
1032
1033         return hist_field;
1034 }
1035
1036 static u64 hist_field_var_ref(struct hist_field *hist_field,
1037                               struct tracing_map_elt *elt,
1038                               struct trace_buffer *buffer,
1039                               struct ring_buffer_event *rbe,
1040                               void *event)
1041 {
1042         struct hist_elt_data *elt_data;
1043         u64 var_val = 0;
1044
1045         if (WARN_ON_ONCE(!elt))
1046                 return var_val;
1047
1048         elt_data = elt->private_data;
1049         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1050
1051         return var_val;
1052 }
1053
1054 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1055                              u64 *var_ref_vals, bool self)
1056 {
1057         struct hist_trigger_data *var_data;
1058         struct tracing_map_elt *var_elt;
1059         struct hist_field *hist_field;
1060         unsigned int i, var_idx;
1061         bool resolved = true;
1062         u64 var_val = 0;
1063
1064         for (i = 0; i < hist_data->n_var_refs; i++) {
1065                 hist_field = hist_data->var_refs[i];
1066                 var_idx = hist_field->var.idx;
1067                 var_data = hist_field->var.hist_data;
1068
1069                 if (var_data == NULL) {
1070                         resolved = false;
1071                         break;
1072                 }
1073
1074                 if ((self && var_data != hist_data) ||
1075                     (!self && var_data == hist_data))
1076                         continue;
1077
1078                 var_elt = tracing_map_lookup(var_data->map, key);
1079                 if (!var_elt) {
1080                         resolved = false;
1081                         break;
1082                 }
1083
1084                 if (!tracing_map_var_set(var_elt, var_idx)) {
1085                         resolved = false;
1086                         break;
1087                 }
1088
1089                 if (self || !hist_field->read_once)
1090                         var_val = tracing_map_read_var(var_elt, var_idx);
1091                 else
1092                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1093
1094                 var_ref_vals[i] = var_val;
1095         }
1096
1097         return resolved;
1098 }
1099
1100 static const char *hist_field_name(struct hist_field *field,
1101                                    unsigned int level)
1102 {
1103         const char *field_name = "";
1104
1105         if (level > 1)
1106                 return field_name;
1107
1108         if (field->field)
1109                 field_name = field->field->name;
1110         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1111                  field->flags & HIST_FIELD_FL_ALIAS)
1112                 field_name = hist_field_name(field->operands[0], ++level);
1113         else if (field->flags & HIST_FIELD_FL_CPU)
1114                 field_name = "cpu";
1115         else if (field->flags & HIST_FIELD_FL_EXPR ||
1116                  field->flags & HIST_FIELD_FL_VAR_REF) {
1117                 if (field->system) {
1118                         static char full_name[MAX_FILTER_STR_VAL];
1119
1120                         strcat(full_name, field->system);
1121                         strcat(full_name, ".");
1122                         strcat(full_name, field->event_name);
1123                         strcat(full_name, ".");
1124                         strcat(full_name, field->name);
1125                         field_name = full_name;
1126                 } else
1127                         field_name = field->name;
1128         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1129                 field_name = "common_timestamp";
1130
1131         if (field_name == NULL)
1132                 field_name = "";
1133
1134         return field_name;
1135 }
1136
1137 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1138 {
1139         hist_field_fn_t fn = NULL;
1140
1141         switch (field_size) {
1142         case 8:
1143                 if (field_is_signed)
1144                         fn = hist_field_s64;
1145                 else
1146                         fn = hist_field_u64;
1147                 break;
1148         case 4:
1149                 if (field_is_signed)
1150                         fn = hist_field_s32;
1151                 else
1152                         fn = hist_field_u32;
1153                 break;
1154         case 2:
1155                 if (field_is_signed)
1156                         fn = hist_field_s16;
1157                 else
1158                         fn = hist_field_u16;
1159                 break;
1160         case 1:
1161                 if (field_is_signed)
1162                         fn = hist_field_s8;
1163                 else
1164                         fn = hist_field_u8;
1165                 break;
1166         }
1167
1168         return fn;
1169 }
1170
1171 static int parse_map_size(char *str)
1172 {
1173         unsigned long size, map_bits;
1174         int ret;
1175
1176         ret = kstrtoul(str, 0, &size);
1177         if (ret)
1178                 goto out;
1179
1180         map_bits = ilog2(roundup_pow_of_two(size));
1181         if (map_bits < TRACING_MAP_BITS_MIN ||
1182             map_bits > TRACING_MAP_BITS_MAX)
1183                 ret = -EINVAL;
1184         else
1185                 ret = map_bits;
1186  out:
1187         return ret;
1188 }
1189
1190 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1191 {
1192         unsigned int i;
1193
1194         if (!attrs)
1195                 return;
1196
1197         for (i = 0; i < attrs->n_assignments; i++)
1198                 kfree(attrs->assignment_str[i]);
1199
1200         for (i = 0; i < attrs->n_actions; i++)
1201                 kfree(attrs->action_str[i]);
1202
1203         kfree(attrs->name);
1204         kfree(attrs->sort_key_str);
1205         kfree(attrs->keys_str);
1206         kfree(attrs->vals_str);
1207         kfree(attrs->clock);
1208         kfree(attrs);
1209 }
1210
1211 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1212 {
1213         int ret = -EINVAL;
1214
1215         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1216                 return ret;
1217
1218         if ((str_has_prefix(str, "onmatch(")) ||
1219             (str_has_prefix(str, "onmax(")) ||
1220             (str_has_prefix(str, "onchange("))) {
1221                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1222                 if (!attrs->action_str[attrs->n_actions]) {
1223                         ret = -ENOMEM;
1224                         return ret;
1225                 }
1226                 attrs->n_actions++;
1227                 ret = 0;
1228         }
1229         return ret;
1230 }
1231
1232 static int parse_assignment(struct trace_array *tr,
1233                             char *str, struct hist_trigger_attrs *attrs)
1234 {
1235         int len, ret = 0;
1236
1237         if ((len = str_has_prefix(str, "key=")) ||
1238             (len = str_has_prefix(str, "keys="))) {
1239                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1240                 if (!attrs->keys_str) {
1241                         ret = -ENOMEM;
1242                         goto out;
1243                 }
1244         } else if ((len = str_has_prefix(str, "val=")) ||
1245                    (len = str_has_prefix(str, "vals=")) ||
1246                    (len = str_has_prefix(str, "values="))) {
1247                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1248                 if (!attrs->vals_str) {
1249                         ret = -ENOMEM;
1250                         goto out;
1251                 }
1252         } else if ((len = str_has_prefix(str, "sort="))) {
1253                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1254                 if (!attrs->sort_key_str) {
1255                         ret = -ENOMEM;
1256                         goto out;
1257                 }
1258         } else if (str_has_prefix(str, "name=")) {
1259                 attrs->name = kstrdup(str, GFP_KERNEL);
1260                 if (!attrs->name) {
1261                         ret = -ENOMEM;
1262                         goto out;
1263                 }
1264         } else if ((len = str_has_prefix(str, "clock="))) {
1265                 str += len;
1266
1267                 str = strstrip(str);
1268                 attrs->clock = kstrdup(str, GFP_KERNEL);
1269                 if (!attrs->clock) {
1270                         ret = -ENOMEM;
1271                         goto out;
1272                 }
1273         } else if ((len = str_has_prefix(str, "size="))) {
1274                 int map_bits = parse_map_size(str + len);
1275
1276                 if (map_bits < 0) {
1277                         ret = map_bits;
1278                         goto out;
1279                 }
1280                 attrs->map_bits = map_bits;
1281         } else {
1282                 char *assignment;
1283
1284                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1285                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1286                         ret = -EINVAL;
1287                         goto out;
1288                 }
1289
1290                 assignment = kstrdup(str, GFP_KERNEL);
1291                 if (!assignment) {
1292                         ret = -ENOMEM;
1293                         goto out;
1294                 }
1295
1296                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1297         }
1298  out:
1299         return ret;
1300 }
1301
1302 static struct hist_trigger_attrs *
1303 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1304 {
1305         struct hist_trigger_attrs *attrs;
1306         int ret = 0;
1307
1308         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1309         if (!attrs)
1310                 return ERR_PTR(-ENOMEM);
1311
1312         while (trigger_str) {
1313                 char *str = strsep(&trigger_str, ":");
1314                 char *rhs;
1315
1316                 rhs = strchr(str, '=');
1317                 if (rhs) {
1318                         if (!strlen(++rhs)) {
1319                                 ret = -EINVAL;
1320                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1321                                 goto free;
1322                         }
1323                         ret = parse_assignment(tr, str, attrs);
1324                         if (ret)
1325                                 goto free;
1326                 } else if (strcmp(str, "pause") == 0)
1327                         attrs->pause = true;
1328                 else if ((strcmp(str, "cont") == 0) ||
1329                          (strcmp(str, "continue") == 0))
1330                         attrs->cont = true;
1331                 else if (strcmp(str, "clear") == 0)
1332                         attrs->clear = true;
1333                 else {
1334                         ret = parse_action(str, attrs);
1335                         if (ret)
1336                                 goto free;
1337                 }
1338         }
1339
1340         if (!attrs->keys_str) {
1341                 ret = -EINVAL;
1342                 goto free;
1343         }
1344
1345         if (!attrs->clock) {
1346                 attrs->clock = kstrdup("global", GFP_KERNEL);
1347                 if (!attrs->clock) {
1348                         ret = -ENOMEM;
1349                         goto free;
1350                 }
1351         }
1352
1353         return attrs;
1354  free:
1355         destroy_hist_trigger_attrs(attrs);
1356
1357         return ERR_PTR(ret);
1358 }
1359
1360 static inline void save_comm(char *comm, struct task_struct *task)
1361 {
1362         if (!task->pid) {
1363                 strcpy(comm, "<idle>");
1364                 return;
1365         }
1366
1367         if (WARN_ON_ONCE(task->pid < 0)) {
1368                 strcpy(comm, "<XXX>");
1369                 return;
1370         }
1371
1372         strncpy(comm, task->comm, TASK_COMM_LEN);
1373 }
1374
1375 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1376 {
1377         unsigned int i;
1378
1379         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1380                 kfree(elt_data->field_var_str[i]);
1381
1382         kfree(elt_data->comm);
1383         kfree(elt_data);
1384 }
1385
1386 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1387 {
1388         struct hist_elt_data *elt_data = elt->private_data;
1389
1390         hist_elt_data_free(elt_data);
1391 }
1392
1393 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1394 {
1395         struct hist_trigger_data *hist_data = elt->map->private_data;
1396         unsigned int size = TASK_COMM_LEN;
1397         struct hist_elt_data *elt_data;
1398         struct hist_field *key_field;
1399         unsigned int i, n_str;
1400
1401         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1402         if (!elt_data)
1403                 return -ENOMEM;
1404
1405         for_each_hist_key_field(i, hist_data) {
1406                 key_field = hist_data->fields[i];
1407
1408                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1409                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1410                         if (!elt_data->comm) {
1411                                 kfree(elt_data);
1412                                 return -ENOMEM;
1413                         }
1414                         break;
1415                 }
1416         }
1417
1418         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1419                 hist_data->n_var_str;
1420         if (n_str > SYNTH_FIELDS_MAX) {
1421                 hist_elt_data_free(elt_data);
1422                 return -EINVAL;
1423         }
1424
1425         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1426
1427         size = STR_VAR_LEN_MAX;
1428
1429         for (i = 0; i < n_str; i++) {
1430                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1431                 if (!elt_data->field_var_str[i]) {
1432                         hist_elt_data_free(elt_data);
1433                         return -ENOMEM;
1434                 }
1435         }
1436
1437         elt->private_data = elt_data;
1438
1439         return 0;
1440 }
1441
1442 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1443 {
1444         struct hist_elt_data *elt_data = elt->private_data;
1445
1446         if (elt_data->comm)
1447                 save_comm(elt_data->comm, current);
1448 }
1449
1450 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1451         .elt_alloc      = hist_trigger_elt_data_alloc,
1452         .elt_free       = hist_trigger_elt_data_free,
1453         .elt_init       = hist_trigger_elt_data_init,
1454 };
1455
1456 static const char *get_hist_field_flags(struct hist_field *hist_field)
1457 {
1458         const char *flags_str = NULL;
1459
1460         if (hist_field->flags & HIST_FIELD_FL_HEX)
1461                 flags_str = "hex";
1462         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1463                 flags_str = "sym";
1464         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1465                 flags_str = "sym-offset";
1466         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1467                 flags_str = "execname";
1468         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1469                 flags_str = "syscall";
1470         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1471                 flags_str = "log2";
1472         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1473                 flags_str = "usecs";
1474
1475         return flags_str;
1476 }
1477
1478 static void expr_field_str(struct hist_field *field, char *expr)
1479 {
1480         if (field->flags & HIST_FIELD_FL_VAR_REF)
1481                 strcat(expr, "$");
1482
1483         strcat(expr, hist_field_name(field, 0));
1484
1485         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1486                 const char *flags_str = get_hist_field_flags(field);
1487
1488                 if (flags_str) {
1489                         strcat(expr, ".");
1490                         strcat(expr, flags_str);
1491                 }
1492         }
1493 }
1494
1495 static char *expr_str(struct hist_field *field, unsigned int level)
1496 {
1497         char *expr;
1498
1499         if (level > 1)
1500                 return NULL;
1501
1502         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1503         if (!expr)
1504                 return NULL;
1505
1506         if (!field->operands[0]) {
1507                 expr_field_str(field, expr);
1508                 return expr;
1509         }
1510
1511         if (field->operator == FIELD_OP_UNARY_MINUS) {
1512                 char *subexpr;
1513
1514                 strcat(expr, "-(");
1515                 subexpr = expr_str(field->operands[0], ++level);
1516                 if (!subexpr) {
1517                         kfree(expr);
1518                         return NULL;
1519                 }
1520                 strcat(expr, subexpr);
1521                 strcat(expr, ")");
1522
1523                 kfree(subexpr);
1524
1525                 return expr;
1526         }
1527
1528         expr_field_str(field->operands[0], expr);
1529
1530         switch (field->operator) {
1531         case FIELD_OP_MINUS:
1532                 strcat(expr, "-");
1533                 break;
1534         case FIELD_OP_PLUS:
1535                 strcat(expr, "+");
1536                 break;
1537         default:
1538                 kfree(expr);
1539                 return NULL;
1540         }
1541
1542         expr_field_str(field->operands[1], expr);
1543
1544         return expr;
1545 }
1546
1547 static int contains_operator(char *str)
1548 {
1549         enum field_op_id field_op = FIELD_OP_NONE;
1550         char *op;
1551
1552         op = strpbrk(str, "+-");
1553         if (!op)
1554                 return FIELD_OP_NONE;
1555
1556         switch (*op) {
1557         case '-':
1558                 /*
1559                  * Unfortunately, the modifier ".sym-offset"
1560                  * can confuse things.
1561                  */
1562                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1563                         return FIELD_OP_NONE;
1564
1565                 if (*str == '-')
1566                         field_op = FIELD_OP_UNARY_MINUS;
1567                 else
1568                         field_op = FIELD_OP_MINUS;
1569                 break;
1570         case '+':
1571                 field_op = FIELD_OP_PLUS;
1572                 break;
1573         default:
1574                 break;
1575         }
1576
1577         return field_op;
1578 }
1579
1580 static void get_hist_field(struct hist_field *hist_field)
1581 {
1582         hist_field->ref++;
1583 }
1584
1585 static void __destroy_hist_field(struct hist_field *hist_field)
1586 {
1587         if (--hist_field->ref > 1)
1588                 return;
1589
1590         kfree(hist_field->var.name);
1591         kfree(hist_field->name);
1592         kfree(hist_field->type);
1593
1594         kfree(hist_field->system);
1595         kfree(hist_field->event_name);
1596
1597         kfree(hist_field);
1598 }
1599
1600 static void destroy_hist_field(struct hist_field *hist_field,
1601                                unsigned int level)
1602 {
1603         unsigned int i;
1604
1605         if (level > 3)
1606                 return;
1607
1608         if (!hist_field)
1609                 return;
1610
1611         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1612                 return; /* var refs will be destroyed separately */
1613
1614         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1615                 destroy_hist_field(hist_field->operands[i], level + 1);
1616
1617         __destroy_hist_field(hist_field);
1618 }
1619
1620 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1621                                             struct ftrace_event_field *field,
1622                                             unsigned long flags,
1623                                             char *var_name)
1624 {
1625         struct hist_field *hist_field;
1626
1627         if (field && is_function_field(field))
1628                 return NULL;
1629
1630         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1631         if (!hist_field)
1632                 return NULL;
1633
1634         hist_field->ref = 1;
1635
1636         hist_field->hist_data = hist_data;
1637
1638         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1639                 goto out; /* caller will populate */
1640
1641         if (flags & HIST_FIELD_FL_VAR_REF) {
1642                 hist_field->fn = hist_field_var_ref;
1643                 goto out;
1644         }
1645
1646         if (flags & HIST_FIELD_FL_HITCOUNT) {
1647                 hist_field->fn = hist_field_counter;
1648                 hist_field->size = sizeof(u64);
1649                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1650                 if (!hist_field->type)
1651                         goto free;
1652                 goto out;
1653         }
1654
1655         if (flags & HIST_FIELD_FL_STACKTRACE) {
1656                 hist_field->fn = hist_field_none;
1657                 goto out;
1658         }
1659
1660         if (flags & HIST_FIELD_FL_LOG2) {
1661                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1662                 hist_field->fn = hist_field_log2;
1663                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1664                 hist_field->size = hist_field->operands[0]->size;
1665                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1666                 if (!hist_field->type)
1667                         goto free;
1668                 goto out;
1669         }
1670
1671         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1672                 hist_field->fn = hist_field_timestamp;
1673                 hist_field->size = sizeof(u64);
1674                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1675                 if (!hist_field->type)
1676                         goto free;
1677                 goto out;
1678         }
1679
1680         if (flags & HIST_FIELD_FL_CPU) {
1681                 hist_field->fn = hist_field_cpu;
1682                 hist_field->size = sizeof(int);
1683                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1684                 if (!hist_field->type)
1685                         goto free;
1686                 goto out;
1687         }
1688
1689         if (WARN_ON_ONCE(!field))
1690                 goto out;
1691
1692         /* Pointers to strings are just pointers and dangerous to dereference */
1693         if (is_string_field(field) &&
1694             (field->filter_type != FILTER_PTR_STRING)) {
1695                 flags |= HIST_FIELD_FL_STRING;
1696
1697                 hist_field->size = MAX_FILTER_STR_VAL;
1698                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1699                 if (!hist_field->type)
1700                         goto free;
1701
1702                 if (field->filter_type == FILTER_STATIC_STRING)
1703                         hist_field->fn = hist_field_string;
1704                 else if (field->filter_type == FILTER_DYN_STRING)
1705                         hist_field->fn = hist_field_dynstring;
1706                 else
1707                         hist_field->fn = hist_field_pstring;
1708         } else {
1709                 hist_field->size = field->size;
1710                 hist_field->is_signed = field->is_signed;
1711                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1712                 if (!hist_field->type)
1713                         goto free;
1714
1715                 hist_field->fn = select_value_fn(field->size,
1716                                                  field->is_signed);
1717                 if (!hist_field->fn) {
1718                         destroy_hist_field(hist_field, 0);
1719                         return NULL;
1720                 }
1721         }
1722  out:
1723         hist_field->field = field;
1724         hist_field->flags = flags;
1725
1726         if (var_name) {
1727                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1728                 if (!hist_field->var.name)
1729                         goto free;
1730         }
1731
1732         return hist_field;
1733  free:
1734         destroy_hist_field(hist_field, 0);
1735         return NULL;
1736 }
1737
1738 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1739 {
1740         unsigned int i;
1741
1742         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1743                 if (hist_data->fields[i]) {
1744                         destroy_hist_field(hist_data->fields[i], 0);
1745                         hist_data->fields[i] = NULL;
1746                 }
1747         }
1748
1749         for (i = 0; i < hist_data->n_var_refs; i++) {
1750                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1751                 __destroy_hist_field(hist_data->var_refs[i]);
1752                 hist_data->var_refs[i] = NULL;
1753         }
1754 }
1755
1756 static int init_var_ref(struct hist_field *ref_field,
1757                         struct hist_field *var_field,
1758                         char *system, char *event_name)
1759 {
1760         int err = 0;
1761
1762         ref_field->var.idx = var_field->var.idx;
1763         ref_field->var.hist_data = var_field->hist_data;
1764         ref_field->size = var_field->size;
1765         ref_field->is_signed = var_field->is_signed;
1766         ref_field->flags |= var_field->flags &
1767                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1768
1769         if (system) {
1770                 ref_field->system = kstrdup(system, GFP_KERNEL);
1771                 if (!ref_field->system)
1772                         return -ENOMEM;
1773         }
1774
1775         if (event_name) {
1776                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1777                 if (!ref_field->event_name) {
1778                         err = -ENOMEM;
1779                         goto free;
1780                 }
1781         }
1782
1783         if (var_field->var.name) {
1784                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1785                 if (!ref_field->name) {
1786                         err = -ENOMEM;
1787                         goto free;
1788                 }
1789         } else if (var_field->name) {
1790                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1791                 if (!ref_field->name) {
1792                         err = -ENOMEM;
1793                         goto free;
1794                 }
1795         }
1796
1797         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1798         if (!ref_field->type) {
1799                 err = -ENOMEM;
1800                 goto free;
1801         }
1802  out:
1803         return err;
1804  free:
1805         kfree(ref_field->system);
1806         kfree(ref_field->event_name);
1807         kfree(ref_field->name);
1808
1809         goto out;
1810 }
1811
1812 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1813                             struct hist_field *var_field)
1814 {
1815         struct hist_field *ref_field;
1816         int i;
1817
1818         for (i = 0; i < hist_data->n_var_refs; i++) {
1819                 ref_field = hist_data->var_refs[i];
1820                 if (ref_field->var.idx == var_field->var.idx &&
1821                     ref_field->var.hist_data == var_field->hist_data)
1822                         return i;
1823         }
1824
1825         return -ENOENT;
1826 }
1827
1828 /**
1829  * create_var_ref - Create a variable reference and attach it to trigger
1830  * @hist_data: The trigger that will be referencing the variable
1831  * @var_field: The VAR field to create a reference to
1832  * @system: The optional system string
1833  * @event_name: The optional event_name string
1834  *
1835  * Given a variable hist_field, create a VAR_REF hist_field that
1836  * represents a reference to it.
1837  *
1838  * This function also adds the reference to the trigger that
1839  * now references the variable.
1840  *
1841  * Return: The VAR_REF field if successful, NULL if not
1842  */
1843 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1844                                          struct hist_field *var_field,
1845                                          char *system, char *event_name)
1846 {
1847         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1848         struct hist_field *ref_field;
1849         int i;
1850
1851         /* Check if the variable already exists */
1852         for (i = 0; i < hist_data->n_var_refs; i++) {
1853                 ref_field = hist_data->var_refs[i];
1854                 if (ref_field->var.idx == var_field->var.idx &&
1855                     ref_field->var.hist_data == var_field->hist_data) {
1856                         get_hist_field(ref_field);
1857                         return ref_field;
1858                 }
1859         }
1860
1861         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1862         if (ref_field) {
1863                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1864                         destroy_hist_field(ref_field, 0);
1865                         return NULL;
1866                 }
1867
1868                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1869                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1870         }
1871
1872         return ref_field;
1873 }
1874
1875 static bool is_var_ref(char *var_name)
1876 {
1877         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1878                 return false;
1879
1880         return true;
1881 }
1882
1883 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1884                                  char *var_name)
1885 {
1886         char *name, *field;
1887         unsigned int i;
1888
1889         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1890                 name = hist_data->attrs->var_defs.name[i];
1891
1892                 if (strcmp(var_name, name) == 0) {
1893                         field = hist_data->attrs->var_defs.expr[i];
1894                         if (contains_operator(field) || is_var_ref(field))
1895                                 continue;
1896                         return field;
1897                 }
1898         }
1899
1900         return NULL;
1901 }
1902
1903 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1904                                  char *system, char *event_name,
1905                                  char *var_name)
1906 {
1907         struct trace_event_call *call;
1908
1909         if (system && event_name) {
1910                 call = hist_data->event_file->event_call;
1911
1912                 if (strcmp(system, call->class->system) != 0)
1913                         return NULL;
1914
1915                 if (strcmp(event_name, trace_event_name(call)) != 0)
1916                         return NULL;
1917         }
1918
1919         if (!!system != !!event_name)
1920                 return NULL;
1921
1922         if (!is_var_ref(var_name))
1923                 return NULL;
1924
1925         var_name++;
1926
1927         return field_name_from_var(hist_data, var_name);
1928 }
1929
1930 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1931                                         char *system, char *event_name,
1932                                         char *var_name)
1933 {
1934         struct hist_field *var_field = NULL, *ref_field = NULL;
1935         struct trace_array *tr = hist_data->event_file->tr;
1936
1937         if (!is_var_ref(var_name))
1938                 return NULL;
1939
1940         var_name++;
1941
1942         var_field = find_event_var(hist_data, system, event_name, var_name);
1943         if (var_field)
1944                 ref_field = create_var_ref(hist_data, var_field,
1945                                            system, event_name);
1946
1947         if (!ref_field)
1948                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1949
1950         return ref_field;
1951 }
1952
1953 static struct ftrace_event_field *
1954 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1955             char *field_str, unsigned long *flags)
1956 {
1957         struct ftrace_event_field *field = NULL;
1958         char *field_name, *modifier, *str;
1959         struct trace_array *tr = file->tr;
1960
1961         modifier = str = kstrdup(field_str, GFP_KERNEL);
1962         if (!modifier)
1963                 return ERR_PTR(-ENOMEM);
1964
1965         field_name = strsep(&modifier, ".");
1966         if (modifier) {
1967                 if (strcmp(modifier, "hex") == 0)
1968                         *flags |= HIST_FIELD_FL_HEX;
1969                 else if (strcmp(modifier, "sym") == 0)
1970                         *flags |= HIST_FIELD_FL_SYM;
1971                 else if (strcmp(modifier, "sym-offset") == 0)
1972                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1973                 else if ((strcmp(modifier, "execname") == 0) &&
1974                          (strcmp(field_name, "common_pid") == 0))
1975                         *flags |= HIST_FIELD_FL_EXECNAME;
1976                 else if (strcmp(modifier, "syscall") == 0)
1977                         *flags |= HIST_FIELD_FL_SYSCALL;
1978                 else if (strcmp(modifier, "log2") == 0)
1979                         *flags |= HIST_FIELD_FL_LOG2;
1980                 else if (strcmp(modifier, "usecs") == 0)
1981                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1982                 else {
1983                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1984                         field = ERR_PTR(-EINVAL);
1985                         goto out;
1986                 }
1987         }
1988
1989         if (strcmp(field_name, "common_timestamp") == 0) {
1990                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1991                 hist_data->enable_timestamps = true;
1992                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1993                         hist_data->attrs->ts_in_usecs = true;
1994         } else if (strcmp(field_name, "cpu") == 0)
1995                 *flags |= HIST_FIELD_FL_CPU;
1996         else {
1997                 field = trace_find_event_field(file->event_call, field_name);
1998                 if (!field || !field->size) {
1999                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2000                         field = ERR_PTR(-EINVAL);
2001                         goto out;
2002                 }
2003         }
2004  out:
2005         kfree(str);
2006
2007         return field;
2008 }
2009
2010 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2011                                        struct hist_field *var_ref,
2012                                        char *var_name)
2013 {
2014         struct hist_field *alias = NULL;
2015         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2016
2017         alias = create_hist_field(hist_data, NULL, flags, var_name);
2018         if (!alias)
2019                 return NULL;
2020
2021         alias->fn = var_ref->fn;
2022         alias->operands[0] = var_ref;
2023
2024         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2025                 destroy_hist_field(alias, 0);
2026                 return NULL;
2027         }
2028
2029         alias->var_ref_idx = var_ref->var_ref_idx;
2030
2031         return alias;
2032 }
2033
2034 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2035                                      struct trace_event_file *file, char *str,
2036                                      unsigned long *flags, char *var_name)
2037 {
2038         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2039         struct ftrace_event_field *field = NULL;
2040         struct hist_field *hist_field = NULL;
2041         int ret = 0;
2042
2043         s = strchr(str, '.');
2044         if (s) {
2045                 s = strchr(++s, '.');
2046                 if (s) {
2047                         ref_system = strsep(&str, ".");
2048                         if (!str) {
2049                                 ret = -EINVAL;
2050                                 goto out;
2051                         }
2052                         ref_event = strsep(&str, ".");
2053                         if (!str) {
2054                                 ret = -EINVAL;
2055                                 goto out;
2056                         }
2057                         ref_var = str;
2058                 }
2059         }
2060
2061         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2062         if (!s) {
2063                 hist_field = parse_var_ref(hist_data, ref_system,
2064                                            ref_event, ref_var);
2065                 if (hist_field) {
2066                         if (var_name) {
2067                                 hist_field = create_alias(hist_data, hist_field, var_name);
2068                                 if (!hist_field) {
2069                                         ret = -ENOMEM;
2070                                         goto out;
2071                                 }
2072                         }
2073                         return hist_field;
2074                 }
2075         } else
2076                 str = s;
2077
2078         field = parse_field(hist_data, file, str, flags);
2079         if (IS_ERR(field)) {
2080                 ret = PTR_ERR(field);
2081                 goto out;
2082         }
2083
2084         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2085         if (!hist_field) {
2086                 ret = -ENOMEM;
2087                 goto out;
2088         }
2089
2090         return hist_field;
2091  out:
2092         return ERR_PTR(ret);
2093 }
2094
2095 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2096                                      struct trace_event_file *file,
2097                                      char *str, unsigned long flags,
2098                                      char *var_name, unsigned int level);
2099
2100 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2101                                       struct trace_event_file *file,
2102                                       char *str, unsigned long flags,
2103                                       char *var_name, unsigned int level)
2104 {
2105         struct hist_field *operand1, *expr = NULL;
2106         unsigned long operand_flags;
2107         int ret = 0;
2108         char *s;
2109
2110         /* we support only -(xxx) i.e. explicit parens required */
2111
2112         if (level > 3) {
2113                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2114                 ret = -EINVAL;
2115                 goto free;
2116         }
2117
2118         str++; /* skip leading '-' */
2119
2120         s = strchr(str, '(');
2121         if (s)
2122                 str++;
2123         else {
2124                 ret = -EINVAL;
2125                 goto free;
2126         }
2127
2128         s = strrchr(str, ')');
2129         if (s)
2130                 *s = '\0';
2131         else {
2132                 ret = -EINVAL; /* no closing ')' */
2133                 goto free;
2134         }
2135
2136         flags |= HIST_FIELD_FL_EXPR;
2137         expr = create_hist_field(hist_data, NULL, flags, var_name);
2138         if (!expr) {
2139                 ret = -ENOMEM;
2140                 goto free;
2141         }
2142
2143         operand_flags = 0;
2144         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2145         if (IS_ERR(operand1)) {
2146                 ret = PTR_ERR(operand1);
2147                 goto free;
2148         }
2149
2150         expr->flags |= operand1->flags &
2151                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2152         expr->fn = hist_field_unary_minus;
2153         expr->operands[0] = operand1;
2154         expr->operator = FIELD_OP_UNARY_MINUS;
2155         expr->name = expr_str(expr, 0);
2156         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2157         if (!expr->type) {
2158                 ret = -ENOMEM;
2159                 goto free;
2160         }
2161
2162         return expr;
2163  free:
2164         destroy_hist_field(expr, 0);
2165         return ERR_PTR(ret);
2166 }
2167
2168 static int check_expr_operands(struct trace_array *tr,
2169                                struct hist_field *operand1,
2170                                struct hist_field *operand2)
2171 {
2172         unsigned long operand1_flags = operand1->flags;
2173         unsigned long operand2_flags = operand2->flags;
2174
2175         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2176             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2177                 struct hist_field *var;
2178
2179                 var = find_var_field(operand1->var.hist_data, operand1->name);
2180                 if (!var)
2181                         return -EINVAL;
2182                 operand1_flags = var->flags;
2183         }
2184
2185         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2186             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2187                 struct hist_field *var;
2188
2189                 var = find_var_field(operand2->var.hist_data, operand2->name);
2190                 if (!var)
2191                         return -EINVAL;
2192                 operand2_flags = var->flags;
2193         }
2194
2195         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2196             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2197                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2198                 return -EINVAL;
2199         }
2200
2201         return 0;
2202 }
2203
2204 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2205                                      struct trace_event_file *file,
2206                                      char *str, unsigned long flags,
2207                                      char *var_name, unsigned int level)
2208 {
2209         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2210         unsigned long operand_flags;
2211         int field_op, ret = -EINVAL;
2212         char *sep, *operand1_str;
2213
2214         if (level > 3) {
2215                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2216                 return ERR_PTR(-EINVAL);
2217         }
2218
2219         field_op = contains_operator(str);
2220
2221         if (field_op == FIELD_OP_NONE)
2222                 return parse_atom(hist_data, file, str, &flags, var_name);
2223
2224         if (field_op == FIELD_OP_UNARY_MINUS)
2225                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2226
2227         switch (field_op) {
2228         case FIELD_OP_MINUS:
2229                 sep = "-";
2230                 break;
2231         case FIELD_OP_PLUS:
2232                 sep = "+";
2233                 break;
2234         default:
2235                 goto free;
2236         }
2237
2238         operand1_str = strsep(&str, sep);
2239         if (!operand1_str || !str)
2240                 goto free;
2241
2242         operand_flags = 0;
2243         operand1 = parse_atom(hist_data, file, operand1_str,
2244                               &operand_flags, NULL);
2245         if (IS_ERR(operand1)) {
2246                 ret = PTR_ERR(operand1);
2247                 operand1 = NULL;
2248                 goto free;
2249         }
2250
2251         /* rest of string could be another expression e.g. b+c in a+b+c */
2252         operand_flags = 0;
2253         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2254         if (IS_ERR(operand2)) {
2255                 ret = PTR_ERR(operand2);
2256                 operand2 = NULL;
2257                 goto free;
2258         }
2259
2260         ret = check_expr_operands(file->tr, operand1, operand2);
2261         if (ret)
2262                 goto free;
2263
2264         flags |= HIST_FIELD_FL_EXPR;
2265
2266         flags |= operand1->flags &
2267                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2268
2269         expr = create_hist_field(hist_data, NULL, flags, var_name);
2270         if (!expr) {
2271                 ret = -ENOMEM;
2272                 goto free;
2273         }
2274
2275         operand1->read_once = true;
2276         operand2->read_once = true;
2277
2278         expr->operands[0] = operand1;
2279         expr->operands[1] = operand2;
2280         expr->operator = field_op;
2281         expr->name = expr_str(expr, 0);
2282         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2283         if (!expr->type) {
2284                 ret = -ENOMEM;
2285                 goto free;
2286         }
2287
2288         switch (field_op) {
2289         case FIELD_OP_MINUS:
2290                 expr->fn = hist_field_minus;
2291                 break;
2292         case FIELD_OP_PLUS:
2293                 expr->fn = hist_field_plus;
2294                 break;
2295         default:
2296                 ret = -EINVAL;
2297                 goto free;
2298         }
2299
2300         return expr;
2301  free:
2302         destroy_hist_field(operand1, 0);
2303         destroy_hist_field(operand2, 0);
2304         destroy_hist_field(expr, 0);
2305
2306         return ERR_PTR(ret);
2307 }
2308
2309 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2310                                  struct trace_event_file *file)
2311 {
2312         struct event_trigger_data *test;
2313
2314         lockdep_assert_held(&event_mutex);
2315
2316         list_for_each_entry(test, &file->triggers, list) {
2317                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2318                         if (test->private_data == hist_data)
2319                                 return test->filter_str;
2320                 }
2321         }
2322
2323         return NULL;
2324 }
2325
2326 static struct event_command trigger_hist_cmd;
2327 static int event_hist_trigger_func(struct event_command *cmd_ops,
2328                                    struct trace_event_file *file,
2329                                    char *glob, char *cmd, char *param);
2330
2331 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2332                             struct hist_trigger_data *hist_data,
2333                             unsigned int n_keys)
2334 {
2335         struct hist_field *target_hist_field, *hist_field;
2336         unsigned int n, i, j;
2337
2338         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2339                 return false;
2340
2341         i = hist_data->n_vals;
2342         j = target_hist_data->n_vals;
2343
2344         for (n = 0; n < n_keys; n++) {
2345                 hist_field = hist_data->fields[i + n];
2346                 target_hist_field = target_hist_data->fields[j + n];
2347
2348                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2349                         return false;
2350                 if (hist_field->size != target_hist_field->size)
2351                         return false;
2352                 if (hist_field->is_signed != target_hist_field->is_signed)
2353                         return false;
2354         }
2355
2356         return true;
2357 }
2358
2359 static struct hist_trigger_data *
2360 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2361                      struct trace_event_file *file)
2362 {
2363         struct hist_trigger_data *hist_data;
2364         struct event_trigger_data *test;
2365         unsigned int n_keys;
2366
2367         lockdep_assert_held(&event_mutex);
2368
2369         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2370
2371         list_for_each_entry(test, &file->triggers, list) {
2372                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2373                         hist_data = test->private_data;
2374
2375                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2376                                 return hist_data;
2377                 }
2378         }
2379
2380         return NULL;
2381 }
2382
2383 static struct trace_event_file *event_file(struct trace_array *tr,
2384                                            char *system, char *event_name)
2385 {
2386         struct trace_event_file *file;
2387
2388         file = __find_event_file(tr, system, event_name);
2389         if (!file)
2390                 return ERR_PTR(-EINVAL);
2391
2392         return file;
2393 }
2394
2395 static struct hist_field *
2396 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2397                          char *system, char *event_name, char *field_name)
2398 {
2399         struct hist_field *event_var;
2400         char *synthetic_name;
2401
2402         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2403         if (!synthetic_name)
2404                 return ERR_PTR(-ENOMEM);
2405
2406         strcpy(synthetic_name, "synthetic_");
2407         strcat(synthetic_name, field_name);
2408
2409         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2410
2411         kfree(synthetic_name);
2412
2413         return event_var;
2414 }
2415
2416 /**
2417  * create_field_var_hist - Automatically create a histogram and var for a field
2418  * @target_hist_data: The target hist trigger
2419  * @subsys_name: Optional subsystem name
2420  * @event_name: Optional event name
2421  * @field_name: The name of the field (and the resulting variable)
2422  *
2423  * Hist trigger actions fetch data from variables, not directly from
2424  * events.  However, for convenience, users are allowed to directly
2425  * specify an event field in an action, which will be automatically
2426  * converted into a variable on their behalf.
2427
2428  * If a user specifies a field on an event that isn't the event the
2429  * histogram currently being defined (the target event histogram), the
2430  * only way that can be accomplished is if a new hist trigger is
2431  * created and the field variable defined on that.
2432  *
2433  * This function creates a new histogram compatible with the target
2434  * event (meaning a histogram with the same key as the target
2435  * histogram), and creates a variable for the specified field, but
2436  * with 'synthetic_' prepended to the variable name in order to avoid
2437  * collision with normal field variables.
2438  *
2439  * Return: The variable created for the field.
2440  */
2441 static struct hist_field *
2442 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2443                       char *subsys_name, char *event_name, char *field_name)
2444 {
2445         struct trace_array *tr = target_hist_data->event_file->tr;
2446         struct hist_trigger_data *hist_data;
2447         unsigned int i, n, first = true;
2448         struct field_var_hist *var_hist;
2449         struct trace_event_file *file;
2450         struct hist_field *key_field;
2451         struct hist_field *event_var;
2452         char *saved_filter;
2453         char *cmd;
2454         int ret;
2455
2456         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2457                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2458                 return ERR_PTR(-EINVAL);
2459         }
2460
2461         file = event_file(tr, subsys_name, event_name);
2462
2463         if (IS_ERR(file)) {
2464                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2465                 ret = PTR_ERR(file);
2466                 return ERR_PTR(ret);
2467         }
2468
2469         /*
2470          * Look for a histogram compatible with target.  We'll use the
2471          * found histogram specification to create a new matching
2472          * histogram with our variable on it.  target_hist_data is not
2473          * yet a registered histogram so we can't use that.
2474          */
2475         hist_data = find_compatible_hist(target_hist_data, file);
2476         if (!hist_data) {
2477                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2478                 return ERR_PTR(-EINVAL);
2479         }
2480
2481         /* See if a synthetic field variable has already been created */
2482         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2483                                              event_name, field_name);
2484         if (!IS_ERR_OR_NULL(event_var))
2485                 return event_var;
2486
2487         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2488         if (!var_hist)
2489                 return ERR_PTR(-ENOMEM);
2490
2491         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2492         if (!cmd) {
2493                 kfree(var_hist);
2494                 return ERR_PTR(-ENOMEM);
2495         }
2496
2497         /* Use the same keys as the compatible histogram */
2498         strcat(cmd, "keys=");
2499
2500         for_each_hist_key_field(i, hist_data) {
2501                 key_field = hist_data->fields[i];
2502                 if (!first)
2503                         strcat(cmd, ",");
2504                 strcat(cmd, key_field->field->name);
2505                 first = false;
2506         }
2507
2508         /* Create the synthetic field variable specification */
2509         strcat(cmd, ":synthetic_");
2510         strcat(cmd, field_name);
2511         strcat(cmd, "=");
2512         strcat(cmd, field_name);
2513
2514         /* Use the same filter as the compatible histogram */
2515         saved_filter = find_trigger_filter(hist_data, file);
2516         if (saved_filter) {
2517                 strcat(cmd, " if ");
2518                 strcat(cmd, saved_filter);
2519         }
2520
2521         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2522         if (!var_hist->cmd) {
2523                 kfree(cmd);
2524                 kfree(var_hist);
2525                 return ERR_PTR(-ENOMEM);
2526         }
2527
2528         /* Save the compatible histogram information */
2529         var_hist->hist_data = hist_data;
2530
2531         /* Create the new histogram with our variable */
2532         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2533                                       "", "hist", cmd);
2534         if (ret) {
2535                 kfree(cmd);
2536                 kfree(var_hist->cmd);
2537                 kfree(var_hist);
2538                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2539                 return ERR_PTR(ret);
2540         }
2541
2542         kfree(cmd);
2543
2544         /* If we can't find the variable, something went wrong */
2545         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2546                                              event_name, field_name);
2547         if (IS_ERR_OR_NULL(event_var)) {
2548                 kfree(var_hist->cmd);
2549                 kfree(var_hist);
2550                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2551                 return ERR_PTR(-EINVAL);
2552         }
2553
2554         n = target_hist_data->n_field_var_hists;
2555         target_hist_data->field_var_hists[n] = var_hist;
2556         target_hist_data->n_field_var_hists++;
2557
2558         return event_var;
2559 }
2560
2561 static struct hist_field *
2562 find_target_event_var(struct hist_trigger_data *hist_data,
2563                       char *subsys_name, char *event_name, char *var_name)
2564 {
2565         struct trace_event_file *file = hist_data->event_file;
2566         struct hist_field *hist_field = NULL;
2567
2568         if (subsys_name) {
2569                 struct trace_event_call *call;
2570
2571                 if (!event_name)
2572                         return NULL;
2573
2574                 call = file->event_call;
2575
2576                 if (strcmp(subsys_name, call->class->system) != 0)
2577                         return NULL;
2578
2579                 if (strcmp(event_name, trace_event_name(call)) != 0)
2580                         return NULL;
2581         }
2582
2583         hist_field = find_var_field(hist_data, var_name);
2584
2585         return hist_field;
2586 }
2587
2588 static inline void __update_field_vars(struct tracing_map_elt *elt,
2589                                        struct trace_buffer *buffer,
2590                                        struct ring_buffer_event *rbe,
2591                                        void *rec,
2592                                        struct field_var **field_vars,
2593                                        unsigned int n_field_vars,
2594                                        unsigned int field_var_str_start)
2595 {
2596         struct hist_elt_data *elt_data = elt->private_data;
2597         unsigned int i, j, var_idx;
2598         u64 var_val;
2599
2600         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2601                 struct field_var *field_var = field_vars[i];
2602                 struct hist_field *var = field_var->var;
2603                 struct hist_field *val = field_var->val;
2604
2605                 var_val = val->fn(val, elt, buffer, rbe, rec);
2606                 var_idx = var->var.idx;
2607
2608                 if (val->flags & HIST_FIELD_FL_STRING) {
2609                         char *str = elt_data->field_var_str[j++];
2610                         char *val_str = (char *)(uintptr_t)var_val;
2611
2612                         strscpy(str, val_str, STR_VAR_LEN_MAX);
2613                         var_val = (u64)(uintptr_t)str;
2614                 }
2615                 tracing_map_set_var(elt, var_idx, var_val);
2616         }
2617 }
2618
2619 static void update_field_vars(struct hist_trigger_data *hist_data,
2620                               struct tracing_map_elt *elt,
2621                               struct trace_buffer *buffer,
2622                               struct ring_buffer_event *rbe,
2623                               void *rec)
2624 {
2625         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2626                             hist_data->n_field_vars, 0);
2627 }
2628
2629 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2630                                  struct tracing_map_elt *elt,
2631                                  struct trace_buffer *buffer,  void *rec,
2632                                  struct ring_buffer_event *rbe, void *key,
2633                                  struct action_data *data, u64 *var_ref_vals)
2634 {
2635         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2636                             hist_data->n_save_vars, hist_data->n_field_var_str);
2637 }
2638
2639 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2640                                      struct trace_event_file *file,
2641                                      char *name, int size, const char *type)
2642 {
2643         struct hist_field *var;
2644         int idx;
2645
2646         if (find_var(hist_data, file, name) && !hist_data->remove) {
2647                 var = ERR_PTR(-EINVAL);
2648                 goto out;
2649         }
2650
2651         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2652         if (!var) {
2653                 var = ERR_PTR(-ENOMEM);
2654                 goto out;
2655         }
2656
2657         idx = tracing_map_add_var(hist_data->map);
2658         if (idx < 0) {
2659                 kfree(var);
2660                 var = ERR_PTR(-EINVAL);
2661                 goto out;
2662         }
2663
2664         var->ref = 1;
2665         var->flags = HIST_FIELD_FL_VAR;
2666         var->var.idx = idx;
2667         var->var.hist_data = var->hist_data = hist_data;
2668         var->size = size;
2669         var->var.name = kstrdup(name, GFP_KERNEL);
2670         var->type = kstrdup(type, GFP_KERNEL);
2671         if (!var->var.name || !var->type) {
2672                 kfree(var->var.name);
2673                 kfree(var->type);
2674                 kfree(var);
2675                 var = ERR_PTR(-ENOMEM);
2676         }
2677  out:
2678         return var;
2679 }
2680
2681 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2682                                           struct trace_event_file *file,
2683                                           char *field_name)
2684 {
2685         struct hist_field *val = NULL, *var = NULL;
2686         unsigned long flags = HIST_FIELD_FL_VAR;
2687         struct trace_array *tr = file->tr;
2688         struct field_var *field_var;
2689         int ret = 0;
2690
2691         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2692                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2693                 ret = -EINVAL;
2694                 goto err;
2695         }
2696
2697         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2698         if (IS_ERR(val)) {
2699                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2700                 ret = PTR_ERR(val);
2701                 goto err;
2702         }
2703
2704         var = create_var(hist_data, file, field_name, val->size, val->type);
2705         if (IS_ERR(var)) {
2706                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2707                 kfree(val);
2708                 ret = PTR_ERR(var);
2709                 goto err;
2710         }
2711
2712         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2713         if (!field_var) {
2714                 kfree(val);
2715                 kfree(var);
2716                 ret =  -ENOMEM;
2717                 goto err;
2718         }
2719
2720         field_var->var = var;
2721         field_var->val = val;
2722  out:
2723         return field_var;
2724  err:
2725         field_var = ERR_PTR(ret);
2726         goto out;
2727 }
2728
2729 /**
2730  * create_target_field_var - Automatically create a variable for a field
2731  * @target_hist_data: The target hist trigger
2732  * @subsys_name: Optional subsystem name
2733  * @event_name: Optional event name
2734  * @var_name: The name of the field (and the resulting variable)
2735  *
2736  * Hist trigger actions fetch data from variables, not directly from
2737  * events.  However, for convenience, users are allowed to directly
2738  * specify an event field in an action, which will be automatically
2739  * converted into a variable on their behalf.
2740
2741  * This function creates a field variable with the name var_name on
2742  * the hist trigger currently being defined on the target event.  If
2743  * subsys_name and event_name are specified, this function simply
2744  * verifies that they do in fact match the target event subsystem and
2745  * event name.
2746  *
2747  * Return: The variable created for the field.
2748  */
2749 static struct field_var *
2750 create_target_field_var(struct hist_trigger_data *target_hist_data,
2751                         char *subsys_name, char *event_name, char *var_name)
2752 {
2753         struct trace_event_file *file = target_hist_data->event_file;
2754
2755         if (subsys_name) {
2756                 struct trace_event_call *call;
2757
2758                 if (!event_name)
2759                         return NULL;
2760
2761                 call = file->event_call;
2762
2763                 if (strcmp(subsys_name, call->class->system) != 0)
2764                         return NULL;
2765
2766                 if (strcmp(event_name, trace_event_name(call)) != 0)
2767                         return NULL;
2768         }
2769
2770         return create_field_var(target_hist_data, file, var_name);
2771 }
2772
2773 static bool check_track_val_max(u64 track_val, u64 var_val)
2774 {
2775         if (var_val <= track_val)
2776                 return false;
2777
2778         return true;
2779 }
2780
2781 static bool check_track_val_changed(u64 track_val, u64 var_val)
2782 {
2783         if (var_val == track_val)
2784                 return false;
2785
2786         return true;
2787 }
2788
2789 static u64 get_track_val(struct hist_trigger_data *hist_data,
2790                          struct tracing_map_elt *elt,
2791                          struct action_data *data)
2792 {
2793         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2794         u64 track_val;
2795
2796         track_val = tracing_map_read_var(elt, track_var_idx);
2797
2798         return track_val;
2799 }
2800
2801 static void save_track_val(struct hist_trigger_data *hist_data,
2802                            struct tracing_map_elt *elt,
2803                            struct action_data *data, u64 var_val)
2804 {
2805         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2806
2807         tracing_map_set_var(elt, track_var_idx, var_val);
2808 }
2809
2810 static void save_track_data(struct hist_trigger_data *hist_data,
2811                             struct tracing_map_elt *elt,
2812                             struct trace_buffer *buffer, void *rec,
2813                             struct ring_buffer_event *rbe, void *key,
2814                             struct action_data *data, u64 *var_ref_vals)
2815 {
2816         if (data->track_data.save_data)
2817                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2818                                            key, data, var_ref_vals);
2819 }
2820
2821 static bool check_track_val(struct tracing_map_elt *elt,
2822                             struct action_data *data,
2823                             u64 var_val)
2824 {
2825         struct hist_trigger_data *hist_data;
2826         u64 track_val;
2827
2828         hist_data = data->track_data.track_var->hist_data;
2829         track_val = get_track_val(hist_data, elt, data);
2830
2831         return data->track_data.check_val(track_val, var_val);
2832 }
2833
2834 #ifdef CONFIG_TRACER_SNAPSHOT
2835 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2836 {
2837         /* called with tr->max_lock held */
2838         struct track_data *track_data = tr->cond_snapshot->cond_data;
2839         struct hist_elt_data *elt_data, *track_elt_data;
2840         struct snapshot_context *context = cond_data;
2841         struct action_data *action;
2842         u64 track_val;
2843
2844         if (!track_data)
2845                 return false;
2846
2847         action = track_data->action_data;
2848
2849         track_val = get_track_val(track_data->hist_data, context->elt,
2850                                   track_data->action_data);
2851
2852         if (!action->track_data.check_val(track_data->track_val, track_val))
2853                 return false;
2854
2855         track_data->track_val = track_val;
2856         memcpy(track_data->key, context->key, track_data->key_len);
2857
2858         elt_data = context->elt->private_data;
2859         track_elt_data = track_data->elt.private_data;
2860         if (elt_data->comm)
2861                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2862
2863         track_data->updated = true;
2864
2865         return true;
2866 }
2867
2868 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2869                                      struct tracing_map_elt *elt,
2870                                      struct trace_buffer *buffer, void *rec,
2871                                      struct ring_buffer_event *rbe, void *key,
2872                                      struct action_data *data,
2873                                      u64 *var_ref_vals)
2874 {
2875         struct trace_event_file *file = hist_data->event_file;
2876         struct snapshot_context context;
2877
2878         context.elt = elt;
2879         context.key = key;
2880
2881         tracing_snapshot_cond(file->tr, &context);
2882 }
2883
2884 static void hist_trigger_print_key(struct seq_file *m,
2885                                    struct hist_trigger_data *hist_data,
2886                                    void *key,
2887                                    struct tracing_map_elt *elt);
2888
2889 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2890 {
2891         unsigned int i;
2892
2893         if (!hist_data->n_actions)
2894                 return NULL;
2895
2896         for (i = 0; i < hist_data->n_actions; i++) {
2897                 struct action_data *data = hist_data->actions[i];
2898
2899                 if (data->action == ACTION_SNAPSHOT)
2900                         return data;
2901         }
2902
2903         return NULL;
2904 }
2905
2906 static void track_data_snapshot_print(struct seq_file *m,
2907                                       struct hist_trigger_data *hist_data)
2908 {
2909         struct trace_event_file *file = hist_data->event_file;
2910         struct track_data *track_data;
2911         struct action_data *action;
2912
2913         track_data = tracing_cond_snapshot_data(file->tr);
2914         if (!track_data)
2915                 return;
2916
2917         if (!track_data->updated)
2918                 return;
2919
2920         action = snapshot_action(hist_data);
2921         if (!action)
2922                 return;
2923
2924         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2925         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2926                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2927                    action->track_data.var_str, track_data->track_val);
2928
2929         seq_puts(m, "\ttriggered by event with key: ");
2930         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2931         seq_putc(m, '\n');
2932 }
2933 #else
2934 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2935 {
2936         return false;
2937 }
2938 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2939                                      struct tracing_map_elt *elt,
2940                                      struct trace_buffer *buffer, void *rec,
2941                                      struct ring_buffer_event *rbe, void *key,
2942                                      struct action_data *data,
2943                                      u64 *var_ref_vals) {}
2944 static void track_data_snapshot_print(struct seq_file *m,
2945                                       struct hist_trigger_data *hist_data) {}
2946 #endif /* CONFIG_TRACER_SNAPSHOT */
2947
2948 static void track_data_print(struct seq_file *m,
2949                              struct hist_trigger_data *hist_data,
2950                              struct tracing_map_elt *elt,
2951                              struct action_data *data)
2952 {
2953         u64 track_val = get_track_val(hist_data, elt, data);
2954         unsigned int i, save_var_idx;
2955
2956         if (data->handler == HANDLER_ONMAX)
2957                 seq_printf(m, "\n\tmax: %10llu", track_val);
2958         else if (data->handler == HANDLER_ONCHANGE)
2959                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2960
2961         if (data->action == ACTION_SNAPSHOT)
2962                 return;
2963
2964         for (i = 0; i < hist_data->n_save_vars; i++) {
2965                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2966                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2967                 u64 val;
2968
2969                 save_var_idx = save_var->var.idx;
2970
2971                 val = tracing_map_read_var(elt, save_var_idx);
2972
2973                 if (save_val->flags & HIST_FIELD_FL_STRING) {
2974                         seq_printf(m, "  %s: %-32s", save_var->var.name,
2975                                    (char *)(uintptr_t)(val));
2976                 } else
2977                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2978         }
2979 }
2980
2981 static void ontrack_action(struct hist_trigger_data *hist_data,
2982                            struct tracing_map_elt *elt,
2983                            struct trace_buffer *buffer, void *rec,
2984                            struct ring_buffer_event *rbe, void *key,
2985                            struct action_data *data, u64 *var_ref_vals)
2986 {
2987         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2988
2989         if (check_track_val(elt, data, var_val)) {
2990                 save_track_val(hist_data, elt, data, var_val);
2991                 save_track_data(hist_data, elt, buffer, rec, rbe,
2992                                 key, data, var_ref_vals);
2993         }
2994 }
2995
2996 static void action_data_destroy(struct action_data *data)
2997 {
2998         unsigned int i;
2999
3000         lockdep_assert_held(&event_mutex);
3001
3002         kfree(data->action_name);
3003
3004         for (i = 0; i < data->n_params; i++)
3005                 kfree(data->params[i]);
3006
3007         if (data->synth_event)
3008                 data->synth_event->ref--;
3009
3010         kfree(data->synth_event_name);
3011
3012         kfree(data);
3013 }
3014
3015 static void track_data_destroy(struct hist_trigger_data *hist_data,
3016                                struct action_data *data)
3017 {
3018         struct trace_event_file *file = hist_data->event_file;
3019
3020         destroy_hist_field(data->track_data.track_var, 0);
3021
3022         if (data->action == ACTION_SNAPSHOT) {
3023                 struct track_data *track_data;
3024
3025                 track_data = tracing_cond_snapshot_data(file->tr);
3026                 if (track_data && track_data->hist_data == hist_data) {
3027                         tracing_snapshot_cond_disable(file->tr);
3028                         track_data_free(track_data);
3029                 }
3030         }
3031
3032         kfree(data->track_data.var_str);
3033
3034         action_data_destroy(data);
3035 }
3036
3037 static int action_create(struct hist_trigger_data *hist_data,
3038                          struct action_data *data);
3039
3040 static int track_data_create(struct hist_trigger_data *hist_data,
3041                              struct action_data *data)
3042 {
3043         struct hist_field *var_field, *ref_field, *track_var = NULL;
3044         struct trace_event_file *file = hist_data->event_file;
3045         struct trace_array *tr = file->tr;
3046         char *track_data_var_str;
3047         int ret = 0;
3048
3049         track_data_var_str = data->track_data.var_str;
3050         if (track_data_var_str[0] != '$') {
3051                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3052                 return -EINVAL;
3053         }
3054         track_data_var_str++;
3055
3056         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3057         if (!var_field) {
3058                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3059                 return -EINVAL;
3060         }
3061
3062         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3063         if (!ref_field)
3064                 return -ENOMEM;
3065
3066         data->track_data.var_ref = ref_field;
3067
3068         if (data->handler == HANDLER_ONMAX)
3069                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3070         if (IS_ERR(track_var)) {
3071                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3072                 ret = PTR_ERR(track_var);
3073                 goto out;
3074         }
3075
3076         if (data->handler == HANDLER_ONCHANGE)
3077                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3078         if (IS_ERR(track_var)) {
3079                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3080                 ret = PTR_ERR(track_var);
3081                 goto out;
3082         }
3083         data->track_data.track_var = track_var;
3084
3085         ret = action_create(hist_data, data);
3086  out:
3087         return ret;
3088 }
3089
3090 static int parse_action_params(struct trace_array *tr, char *params,
3091                                struct action_data *data)
3092 {
3093         char *param, *saved_param;
3094         bool first_param = true;
3095         int ret = 0;
3096
3097         while (params) {
3098                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3099                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3100                         goto out;
3101                 }
3102
3103                 param = strsep(&params, ",");
3104                 if (!param) {
3105                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3106                         ret = -EINVAL;
3107                         goto out;
3108                 }
3109
3110                 param = strstrip(param);
3111                 if (strlen(param) < 2) {
3112                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3113                         ret = -EINVAL;
3114                         goto out;
3115                 }
3116
3117                 saved_param = kstrdup(param, GFP_KERNEL);
3118                 if (!saved_param) {
3119                         ret = -ENOMEM;
3120                         goto out;
3121                 }
3122
3123                 if (first_param && data->use_trace_keyword) {
3124                         data->synth_event_name = saved_param;
3125                         first_param = false;
3126                         continue;
3127                 }
3128                 first_param = false;
3129
3130                 data->params[data->n_params++] = saved_param;
3131         }
3132  out:
3133         return ret;
3134 }
3135
3136 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3137                         enum handler_id handler)
3138 {
3139         char *action_name;
3140         int ret = 0;
3141
3142         strsep(&str, ".");
3143         if (!str) {
3144                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3145                 ret = -EINVAL;
3146                 goto out;
3147         }
3148
3149         action_name = strsep(&str, "(");
3150         if (!action_name || !str) {
3151                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3152                 ret = -EINVAL;
3153                 goto out;
3154         }
3155
3156         if (str_has_prefix(action_name, "save")) {
3157                 char *params = strsep(&str, ")");
3158
3159                 if (!params) {
3160                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3161                         ret = -EINVAL;
3162                         goto out;
3163                 }
3164
3165                 ret = parse_action_params(tr, params, data);
3166                 if (ret)
3167                         goto out;
3168
3169                 if (handler == HANDLER_ONMAX)
3170                         data->track_data.check_val = check_track_val_max;
3171                 else if (handler == HANDLER_ONCHANGE)
3172                         data->track_data.check_val = check_track_val_changed;
3173                 else {
3174                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3175                         ret = -EINVAL;
3176                         goto out;
3177                 }
3178
3179                 data->track_data.save_data = save_track_data_vars;
3180                 data->fn = ontrack_action;
3181                 data->action = ACTION_SAVE;
3182         } else if (str_has_prefix(action_name, "snapshot")) {
3183                 char *params = strsep(&str, ")");
3184
3185                 if (!str) {
3186                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3187                         ret = -EINVAL;
3188                         goto out;
3189                 }
3190
3191                 if (handler == HANDLER_ONMAX)
3192                         data->track_data.check_val = check_track_val_max;
3193                 else if (handler == HANDLER_ONCHANGE)
3194                         data->track_data.check_val = check_track_val_changed;
3195                 else {
3196                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3197                         ret = -EINVAL;
3198                         goto out;
3199                 }
3200
3201                 data->track_data.save_data = save_track_data_snapshot;
3202                 data->fn = ontrack_action;
3203                 data->action = ACTION_SNAPSHOT;
3204         } else {
3205                 char *params = strsep(&str, ")");
3206
3207                 if (str_has_prefix(action_name, "trace"))
3208                         data->use_trace_keyword = true;
3209
3210                 if (params) {
3211                         ret = parse_action_params(tr, params, data);
3212                         if (ret)
3213                                 goto out;
3214                 }
3215
3216                 if (handler == HANDLER_ONMAX)
3217                         data->track_data.check_val = check_track_val_max;
3218                 else if (handler == HANDLER_ONCHANGE)
3219                         data->track_data.check_val = check_track_val_changed;
3220
3221                 if (handler != HANDLER_ONMATCH) {
3222                         data->track_data.save_data = action_trace;
3223                         data->fn = ontrack_action;
3224                 } else
3225                         data->fn = action_trace;
3226
3227                 data->action = ACTION_TRACE;
3228         }
3229
3230         data->action_name = kstrdup(action_name, GFP_KERNEL);
3231         if (!data->action_name) {
3232                 ret = -ENOMEM;
3233                 goto out;
3234         }
3235
3236         data->handler = handler;
3237  out:
3238         return ret;
3239 }
3240
3241 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3242                                             char *str, enum handler_id handler)
3243 {
3244         struct action_data *data;
3245         int ret = -EINVAL;
3246         char *var_str;
3247
3248         data = kzalloc(sizeof(*data), GFP_KERNEL);
3249         if (!data)
3250                 return ERR_PTR(-ENOMEM);
3251
3252         var_str = strsep(&str, ")");
3253         if (!var_str || !str) {
3254                 ret = -EINVAL;
3255                 goto free;
3256         }
3257
3258         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3259         if (!data->track_data.var_str) {
3260                 ret = -ENOMEM;
3261                 goto free;
3262         }
3263
3264         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3265         if (ret)
3266                 goto free;
3267  out:
3268         return data;
3269  free:
3270         track_data_destroy(hist_data, data);
3271         data = ERR_PTR(ret);
3272         goto out;
3273 }
3274
3275 static void onmatch_destroy(struct action_data *data)
3276 {
3277         kfree(data->match_data.event);
3278         kfree(data->match_data.event_system);
3279
3280         action_data_destroy(data);
3281 }
3282
3283 static void destroy_field_var(struct field_var *field_var)
3284 {
3285         if (!field_var)
3286                 return;
3287
3288         destroy_hist_field(field_var->var, 0);
3289         destroy_hist_field(field_var->val, 0);
3290
3291         kfree(field_var);
3292 }
3293
3294 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3295 {
3296         unsigned int i;
3297
3298         for (i = 0; i < hist_data->n_field_vars; i++)
3299                 destroy_field_var(hist_data->field_vars[i]);
3300
3301         for (i = 0; i < hist_data->n_save_vars; i++)
3302                 destroy_field_var(hist_data->save_vars[i]);
3303 }
3304
3305 static void save_field_var(struct hist_trigger_data *hist_data,
3306                            struct field_var *field_var)
3307 {
3308         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3309
3310         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3311                 hist_data->n_field_var_str++;
3312 }
3313
3314
3315 static int check_synth_field(struct synth_event *event,
3316                              struct hist_field *hist_field,
3317                              unsigned int field_pos)
3318 {
3319         struct synth_field *field;
3320
3321         if (field_pos >= event->n_fields)
3322                 return -EINVAL;
3323
3324         field = event->fields[field_pos];
3325
3326         /*
3327          * A dynamic string synth field can accept static or
3328          * dynamic. A static string synth field can only accept a
3329          * same-sized static string, which is checked for later.
3330          */
3331         if (strstr(hist_field->type, "char[") && field->is_string
3332             && field->is_dynamic)
3333                 return 0;
3334
3335         if (strcmp(field->type, hist_field->type) != 0) {
3336                 if (field->size != hist_field->size ||
3337                     field->is_signed != hist_field->is_signed)
3338                         return -EINVAL;
3339         }
3340
3341         return 0;
3342 }
3343
3344 static struct hist_field *
3345 trace_action_find_var(struct hist_trigger_data *hist_data,
3346                       struct action_data *data,
3347                       char *system, char *event, char *var)
3348 {
3349         struct trace_array *tr = hist_data->event_file->tr;
3350         struct hist_field *hist_field;
3351
3352         var++; /* skip '$' */
3353
3354         hist_field = find_target_event_var(hist_data, system, event, var);
3355         if (!hist_field) {
3356                 if (!system && data->handler == HANDLER_ONMATCH) {
3357                         system = data->match_data.event_system;
3358                         event = data->match_data.event;
3359                 }
3360
3361                 hist_field = find_event_var(hist_data, system, event, var);
3362         }
3363
3364         if (!hist_field)
3365                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3366
3367         return hist_field;
3368 }
3369
3370 static struct hist_field *
3371 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3372                               struct action_data *data, char *system,
3373                               char *event, char *var)
3374 {
3375         struct hist_field *hist_field = NULL;
3376         struct field_var *field_var;
3377
3378         /*
3379          * First try to create a field var on the target event (the
3380          * currently being defined).  This will create a variable for
3381          * unqualified fields on the target event, or if qualified,
3382          * target fields that have qualified names matching the target.
3383          */
3384         field_var = create_target_field_var(hist_data, system, event, var);
3385
3386         if (field_var && !IS_ERR(field_var)) {
3387                 save_field_var(hist_data, field_var);
3388                 hist_field = field_var->var;
3389         } else {
3390                 field_var = NULL;
3391                 /*
3392                  * If no explicit system.event is specified, default to
3393                  * looking for fields on the onmatch(system.event.xxx)
3394                  * event.
3395                  */
3396                 if (!system && data->handler == HANDLER_ONMATCH) {
3397                         system = data->match_data.event_system;
3398                         event = data->match_data.event;
3399                 }
3400
3401                 /*
3402                  * At this point, we're looking at a field on another
3403                  * event.  Because we can't modify a hist trigger on
3404                  * another event to add a variable for a field, we need
3405                  * to create a new trigger on that event and create the
3406                  * variable at the same time.
3407                  */
3408                 hist_field = create_field_var_hist(hist_data, system, event, var);
3409                 if (IS_ERR(hist_field))
3410                         goto free;
3411         }
3412  out:
3413         return hist_field;
3414  free:
3415         destroy_field_var(field_var);
3416         hist_field = NULL;
3417         goto out;
3418 }
3419
3420 static int trace_action_create(struct hist_trigger_data *hist_data,
3421                                struct action_data *data)
3422 {
3423         struct trace_array *tr = hist_data->event_file->tr;
3424         char *event_name, *param, *system = NULL;
3425         struct hist_field *hist_field, *var_ref;
3426         unsigned int i;
3427         unsigned int field_pos = 0;
3428         struct synth_event *event;
3429         char *synth_event_name;
3430         int var_ref_idx, ret = 0;
3431
3432         lockdep_assert_held(&event_mutex);
3433
3434         if (data->use_trace_keyword)
3435                 synth_event_name = data->synth_event_name;
3436         else
3437                 synth_event_name = data->action_name;
3438
3439         event = find_synth_event(synth_event_name);
3440         if (!event) {
3441                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3442                 return -EINVAL;
3443         }
3444
3445         event->ref++;
3446
3447         for (i = 0; i < data->n_params; i++) {
3448                 char *p;
3449
3450                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3451                 if (!param) {
3452                         ret = -ENOMEM;
3453                         goto err;
3454                 }
3455
3456                 system = strsep(&param, ".");
3457                 if (!param) {
3458                         param = (char *)system;
3459                         system = event_name = NULL;
3460                 } else {
3461                         event_name = strsep(&param, ".");
3462                         if (!param) {
3463                                 kfree(p);
3464                                 ret = -EINVAL;
3465                                 goto err;
3466                         }
3467                 }
3468
3469                 if (param[0] == '$')
3470                         hist_field = trace_action_find_var(hist_data, data,
3471                                                            system, event_name,
3472                                                            param);
3473                 else
3474                         hist_field = trace_action_create_field_var(hist_data,
3475                                                                    data,
3476                                                                    system,
3477                                                                    event_name,
3478                                                                    param);
3479
3480                 if (!hist_field) {
3481                         kfree(p);
3482                         ret = -EINVAL;
3483                         goto err;
3484                 }
3485
3486                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3487                         var_ref = create_var_ref(hist_data, hist_field,
3488                                                  system, event_name);
3489                         if (!var_ref) {
3490                                 kfree(p);
3491                                 ret = -ENOMEM;
3492                                 goto err;
3493                         }
3494
3495                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3496                         if (WARN_ON(var_ref_idx < 0)) {
3497                                 ret = var_ref_idx;
3498                                 goto err;
3499                         }
3500
3501                         data->var_ref_idx[i] = var_ref_idx;
3502
3503                         field_pos++;
3504                         kfree(p);
3505                         continue;
3506                 }
3507
3508                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3509                 kfree(p);
3510                 ret = -EINVAL;
3511                 goto err;
3512         }
3513
3514         if (field_pos != event->n_fields) {
3515                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3516                 ret = -EINVAL;
3517                 goto err;
3518         }
3519
3520         data->synth_event = event;
3521  out:
3522         return ret;
3523  err:
3524         event->ref--;
3525
3526         goto out;
3527 }
3528
3529 static int action_create(struct hist_trigger_data *hist_data,
3530                          struct action_data *data)
3531 {
3532         struct trace_event_file *file = hist_data->event_file;
3533         struct trace_array *tr = file->tr;
3534         struct track_data *track_data;
3535         struct field_var *field_var;
3536         unsigned int i;
3537         char *param;
3538         int ret = 0;
3539
3540         if (data->action == ACTION_TRACE)
3541                 return trace_action_create(hist_data, data);
3542
3543         if (data->action == ACTION_SNAPSHOT) {
3544                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3545                 if (IS_ERR(track_data)) {
3546                         ret = PTR_ERR(track_data);
3547                         goto out;
3548                 }
3549
3550                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3551                                                    cond_snapshot_update);
3552                 if (ret)
3553                         track_data_free(track_data);
3554
3555                 goto out;
3556         }
3557
3558         if (data->action == ACTION_SAVE) {
3559                 if (hist_data->n_save_vars) {
3560                         ret = -EEXIST;
3561                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3562                         goto out;
3563                 }
3564
3565                 for (i = 0; i < data->n_params; i++) {
3566                         param = kstrdup(data->params[i], GFP_KERNEL);
3567                         if (!param) {
3568                                 ret = -ENOMEM;
3569                                 goto out;
3570                         }
3571
3572                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3573                         if (IS_ERR(field_var)) {
3574                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3575                                          errpos(param));
3576                                 ret = PTR_ERR(field_var);
3577                                 kfree(param);
3578                                 goto out;
3579                         }
3580
3581                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3582                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3583                                 hist_data->n_save_var_str++;
3584                         kfree(param);
3585                 }
3586         }
3587  out:
3588         return ret;
3589 }
3590
3591 static int onmatch_create(struct hist_trigger_data *hist_data,
3592                           struct action_data *data)
3593 {
3594         return action_create(hist_data, data);
3595 }
3596
3597 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3598 {
3599         char *match_event, *match_event_system;
3600         struct action_data *data;
3601         int ret = -EINVAL;
3602
3603         data = kzalloc(sizeof(*data), GFP_KERNEL);
3604         if (!data)
3605                 return ERR_PTR(-ENOMEM);
3606
3607         match_event = strsep(&str, ")");
3608         if (!match_event || !str) {
3609                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3610                 goto free;
3611         }
3612
3613         match_event_system = strsep(&match_event, ".");
3614         if (!match_event) {
3615                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3616                 goto free;
3617         }
3618
3619         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3620                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3621                 goto free;
3622         }
3623
3624         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3625         if (!data->match_data.event) {
3626                 ret = -ENOMEM;
3627                 goto free;
3628         }
3629
3630         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3631         if (!data->match_data.event_system) {
3632                 ret = -ENOMEM;
3633                 goto free;
3634         }
3635
3636         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3637         if (ret)
3638                 goto free;
3639  out:
3640         return data;
3641  free:
3642         onmatch_destroy(data);
3643         data = ERR_PTR(ret);
3644         goto out;
3645 }
3646
3647 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3648 {
3649         hist_data->fields[HITCOUNT_IDX] =
3650                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3651         if (!hist_data->fields[HITCOUNT_IDX])
3652                 return -ENOMEM;
3653
3654         hist_data->n_vals++;
3655         hist_data->n_fields++;
3656
3657         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3658                 return -EINVAL;
3659
3660         return 0;
3661 }
3662
3663 static int __create_val_field(struct hist_trigger_data *hist_data,
3664                               unsigned int val_idx,
3665                               struct trace_event_file *file,
3666                               char *var_name, char *field_str,
3667                               unsigned long flags)
3668 {
3669         struct hist_field *hist_field;
3670         int ret = 0;
3671
3672         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3673         if (IS_ERR(hist_field)) {
3674                 ret = PTR_ERR(hist_field);
3675                 goto out;
3676         }
3677
3678         hist_data->fields[val_idx] = hist_field;
3679
3680         ++hist_data->n_vals;
3681         ++hist_data->n_fields;
3682
3683         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3684                 ret = -EINVAL;
3685  out:
3686         return ret;
3687 }
3688
3689 static int create_val_field(struct hist_trigger_data *hist_data,
3690                             unsigned int val_idx,
3691                             struct trace_event_file *file,
3692                             char *field_str)
3693 {
3694         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3695                 return -EINVAL;
3696
3697         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3698 }
3699
3700 static int create_var_field(struct hist_trigger_data *hist_data,
3701                             unsigned int val_idx,
3702                             struct trace_event_file *file,
3703                             char *var_name, char *expr_str)
3704 {
3705         struct trace_array *tr = hist_data->event_file->tr;
3706         unsigned long flags = 0;
3707         int ret;
3708
3709         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3710                 return -EINVAL;
3711
3712         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3713                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3714                 return -EINVAL;
3715         }
3716
3717         flags |= HIST_FIELD_FL_VAR;
3718         hist_data->n_vars++;
3719         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3720                 return -EINVAL;
3721
3722         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3723
3724         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3725                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3726
3727         return ret;
3728 }
3729
3730 static int create_val_fields(struct hist_trigger_data *hist_data,
3731                              struct trace_event_file *file)
3732 {
3733         char *fields_str, *field_str;
3734         unsigned int i, j = 1;
3735         int ret;
3736
3737         ret = create_hitcount_val(hist_data);
3738         if (ret)
3739                 goto out;
3740
3741         fields_str = hist_data->attrs->vals_str;
3742         if (!fields_str)
3743                 goto out;
3744
3745         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3746                      j < TRACING_MAP_VALS_MAX; i++) {
3747                 field_str = strsep(&fields_str, ",");
3748                 if (!field_str)
3749                         break;
3750
3751                 if (strcmp(field_str, "hitcount") == 0)
3752                         continue;
3753
3754                 ret = create_val_field(hist_data, j++, file, field_str);
3755                 if (ret)
3756                         goto out;
3757         }
3758
3759         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3760                 ret = -EINVAL;
3761  out:
3762         return ret;
3763 }
3764
3765 static int create_key_field(struct hist_trigger_data *hist_data,
3766                             unsigned int key_idx,
3767                             unsigned int key_offset,
3768                             struct trace_event_file *file,
3769                             char *field_str)
3770 {
3771         struct trace_array *tr = hist_data->event_file->tr;
3772         struct hist_field *hist_field = NULL;
3773         unsigned long flags = 0;
3774         unsigned int key_size;
3775         int ret = 0;
3776
3777         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3778                 return -EINVAL;
3779
3780         flags |= HIST_FIELD_FL_KEY;
3781
3782         if (strcmp(field_str, "stacktrace") == 0) {
3783                 flags |= HIST_FIELD_FL_STACKTRACE;
3784                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3785                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3786         } else {
3787                 hist_field = parse_expr(hist_data, file, field_str, flags,
3788                                         NULL, 0);
3789                 if (IS_ERR(hist_field)) {
3790                         ret = PTR_ERR(hist_field);
3791                         goto out;
3792                 }
3793
3794                 if (field_has_hist_vars(hist_field, 0)) {
3795                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3796                         destroy_hist_field(hist_field, 0);
3797                         ret = -EINVAL;
3798                         goto out;
3799                 }
3800
3801                 key_size = hist_field->size;
3802         }
3803
3804         hist_data->fields[key_idx] = hist_field;
3805
3806         key_size = ALIGN(key_size, sizeof(u64));
3807         hist_data->fields[key_idx]->size = key_size;
3808         hist_data->fields[key_idx]->offset = key_offset;
3809
3810         hist_data->key_size += key_size;
3811
3812         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3813                 ret = -EINVAL;
3814                 goto out;
3815         }
3816
3817         hist_data->n_keys++;
3818         hist_data->n_fields++;
3819
3820         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3821                 return -EINVAL;
3822
3823         ret = key_size;
3824  out:
3825         return ret;
3826 }
3827
3828 static int create_key_fields(struct hist_trigger_data *hist_data,
3829                              struct trace_event_file *file)
3830 {
3831         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3832         char *fields_str, *field_str;
3833         int ret = -EINVAL;
3834
3835         fields_str = hist_data->attrs->keys_str;
3836         if (!fields_str)
3837                 goto out;
3838
3839         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3840                 field_str = strsep(&fields_str, ",");
3841                 if (!field_str)
3842                         break;
3843                 ret = create_key_field(hist_data, i, key_offset,
3844                                        file, field_str);
3845                 if (ret < 0)
3846                         goto out;
3847                 key_offset += ret;
3848         }
3849         if (fields_str) {
3850                 ret = -EINVAL;
3851                 goto out;
3852         }
3853         ret = 0;
3854  out:
3855         return ret;
3856 }
3857
3858 static int create_var_fields(struct hist_trigger_data *hist_data,
3859                              struct trace_event_file *file)
3860 {
3861         unsigned int i, j = hist_data->n_vals;
3862         int ret = 0;
3863
3864         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3865
3866         for (i = 0; i < n_vars; i++) {
3867                 char *var_name = hist_data->attrs->var_defs.name[i];
3868                 char *expr = hist_data->attrs->var_defs.expr[i];
3869
3870                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3871                 if (ret)
3872                         goto out;
3873         }
3874  out:
3875         return ret;
3876 }
3877
3878 static void free_var_defs(struct hist_trigger_data *hist_data)
3879 {
3880         unsigned int i;
3881
3882         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3883                 kfree(hist_data->attrs->var_defs.name[i]);
3884                 kfree(hist_data->attrs->var_defs.expr[i]);
3885         }
3886
3887         hist_data->attrs->var_defs.n_vars = 0;
3888 }
3889
3890 static int parse_var_defs(struct hist_trigger_data *hist_data)
3891 {
3892         struct trace_array *tr = hist_data->event_file->tr;
3893         char *s, *str, *var_name, *field_str;
3894         unsigned int i, j, n_vars = 0;
3895         int ret = 0;
3896
3897         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3898                 str = hist_data->attrs->assignment_str[i];
3899                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3900                         field_str = strsep(&str, ",");
3901                         if (!field_str)
3902                                 break;
3903
3904                         var_name = strsep(&field_str, "=");
3905                         if (!var_name || !field_str) {
3906                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3907                                          errpos(var_name));
3908                                 ret = -EINVAL;
3909                                 goto free;
3910                         }
3911
3912                         if (n_vars == TRACING_MAP_VARS_MAX) {
3913                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3914                                 ret = -EINVAL;
3915                                 goto free;
3916                         }
3917
3918                         s = kstrdup(var_name, GFP_KERNEL);
3919                         if (!s) {
3920                                 ret = -ENOMEM;
3921                                 goto free;
3922                         }
3923                         hist_data->attrs->var_defs.name[n_vars] = s;
3924
3925                         s = kstrdup(field_str, GFP_KERNEL);
3926                         if (!s) {
3927                                 ret = -ENOMEM;
3928                                 goto free;
3929                         }
3930                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3931
3932                         hist_data->attrs->var_defs.n_vars = n_vars;
3933                 }
3934         }
3935
3936         return ret;
3937  free:
3938         free_var_defs(hist_data);
3939
3940         return ret;
3941 }
3942
3943 static int create_hist_fields(struct hist_trigger_data *hist_data,
3944                               struct trace_event_file *file)
3945 {
3946         int ret;
3947
3948         ret = parse_var_defs(hist_data);
3949         if (ret)
3950                 goto out;
3951
3952         ret = create_val_fields(hist_data, file);
3953         if (ret)
3954                 goto out;
3955
3956         ret = create_var_fields(hist_data, file);
3957         if (ret)
3958                 goto out;
3959
3960         ret = create_key_fields(hist_data, file);
3961         if (ret)
3962                 goto out;
3963  out:
3964         free_var_defs(hist_data);
3965
3966         return ret;
3967 }
3968
3969 static int is_descending(struct trace_array *tr, const char *str)
3970 {
3971         if (!str)
3972                 return 0;
3973
3974         if (strcmp(str, "descending") == 0)
3975                 return 1;
3976
3977         if (strcmp(str, "ascending") == 0)
3978                 return 0;
3979
3980         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3981
3982         return -EINVAL;
3983 }
3984
3985 static int create_sort_keys(struct hist_trigger_data *hist_data)
3986 {
3987         struct trace_array *tr = hist_data->event_file->tr;
3988         char *fields_str = hist_data->attrs->sort_key_str;
3989         struct tracing_map_sort_key *sort_key;
3990         int descending, ret = 0;
3991         unsigned int i, j, k;
3992
3993         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3994
3995         if (!fields_str)
3996                 goto out;
3997
3998         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3999                 struct hist_field *hist_field;
4000                 char *field_str, *field_name;
4001                 const char *test_name;
4002
4003                 sort_key = &hist_data->sort_keys[i];
4004
4005                 field_str = strsep(&fields_str, ",");
4006                 if (!field_str)
4007                         break;
4008
4009                 if (!*field_str) {
4010                         ret = -EINVAL;
4011                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4012                         break;
4013                 }
4014
4015                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4016                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4017                         ret = -EINVAL;
4018                         break;
4019                 }
4020
4021                 field_name = strsep(&field_str, ".");
4022                 if (!field_name || !*field_name) {
4023                         ret = -EINVAL;
4024                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4025                         break;
4026                 }
4027
4028                 if (strcmp(field_name, "hitcount") == 0) {
4029                         descending = is_descending(tr, field_str);
4030                         if (descending < 0) {
4031                                 ret = descending;
4032                                 break;
4033                         }
4034                         sort_key->descending = descending;
4035                         continue;
4036                 }
4037
4038                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4039                         unsigned int idx;
4040
4041                         hist_field = hist_data->fields[j];
4042                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4043                                 continue;
4044
4045                         idx = k++;
4046
4047                         test_name = hist_field_name(hist_field, 0);
4048
4049                         if (strcmp(field_name, test_name) == 0) {
4050                                 sort_key->field_idx = idx;
4051                                 descending = is_descending(tr, field_str);
4052                                 if (descending < 0) {
4053                                         ret = descending;
4054                                         goto out;
4055                                 }
4056                                 sort_key->descending = descending;
4057                                 break;
4058                         }
4059                 }
4060                 if (j == hist_data->n_fields) {
4061                         ret = -EINVAL;
4062                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4063                         break;
4064                 }
4065         }
4066
4067         hist_data->n_sort_keys = i;
4068  out:
4069         return ret;
4070 }
4071
4072 static void destroy_actions(struct hist_trigger_data *hist_data)
4073 {
4074         unsigned int i;
4075
4076         for (i = 0; i < hist_data->n_actions; i++) {
4077                 struct action_data *data = hist_data->actions[i];
4078
4079                 if (data->handler == HANDLER_ONMATCH)
4080                         onmatch_destroy(data);
4081                 else if (data->handler == HANDLER_ONMAX ||
4082                          data->handler == HANDLER_ONCHANGE)
4083                         track_data_destroy(hist_data, data);
4084                 else
4085                         kfree(data);
4086         }
4087 }
4088
4089 static int parse_actions(struct hist_trigger_data *hist_data)
4090 {
4091         struct trace_array *tr = hist_data->event_file->tr;
4092         struct action_data *data;
4093         unsigned int i;
4094         int ret = 0;
4095         char *str;
4096         int len;
4097
4098         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4099                 str = hist_data->attrs->action_str[i];
4100
4101                 if ((len = str_has_prefix(str, "onmatch("))) {
4102                         char *action_str = str + len;
4103
4104                         data = onmatch_parse(tr, action_str);
4105                         if (IS_ERR(data)) {
4106                                 ret = PTR_ERR(data);
4107                                 break;
4108                         }
4109                 } else if ((len = str_has_prefix(str, "onmax("))) {
4110                         char *action_str = str + len;
4111
4112                         data = track_data_parse(hist_data, action_str,
4113                                                 HANDLER_ONMAX);
4114                         if (IS_ERR(data)) {
4115                                 ret = PTR_ERR(data);
4116                                 break;
4117                         }
4118                 } else if ((len = str_has_prefix(str, "onchange("))) {
4119                         char *action_str = str + len;
4120
4121                         data = track_data_parse(hist_data, action_str,
4122                                                 HANDLER_ONCHANGE);
4123                         if (IS_ERR(data)) {
4124                                 ret = PTR_ERR(data);
4125                                 break;
4126                         }
4127                 } else {
4128                         ret = -EINVAL;
4129                         break;
4130                 }
4131
4132                 hist_data->actions[hist_data->n_actions++] = data;
4133         }
4134
4135         return ret;
4136 }
4137
4138 static int create_actions(struct hist_trigger_data *hist_data)
4139 {
4140         struct action_data *data;
4141         unsigned int i;
4142         int ret = 0;
4143
4144         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4145                 data = hist_data->actions[i];
4146
4147                 if (data->handler == HANDLER_ONMATCH) {
4148                         ret = onmatch_create(hist_data, data);
4149                         if (ret)
4150                                 break;
4151                 } else if (data->handler == HANDLER_ONMAX ||
4152                            data->handler == HANDLER_ONCHANGE) {
4153                         ret = track_data_create(hist_data, data);
4154                         if (ret)
4155                                 break;
4156                 } else {
4157                         ret = -EINVAL;
4158                         break;
4159                 }
4160         }
4161
4162         return ret;
4163 }
4164
4165 static void print_actions(struct seq_file *m,
4166                           struct hist_trigger_data *hist_data,
4167                           struct tracing_map_elt *elt)
4168 {
4169         unsigned int i;
4170
4171         for (i = 0; i < hist_data->n_actions; i++) {
4172                 struct action_data *data = hist_data->actions[i];
4173
4174                 if (data->action == ACTION_SNAPSHOT)
4175                         continue;
4176
4177                 if (data->handler == HANDLER_ONMAX ||
4178                     data->handler == HANDLER_ONCHANGE)
4179                         track_data_print(m, hist_data, elt, data);
4180         }
4181 }
4182
4183 static void print_action_spec(struct seq_file *m,
4184                               struct hist_trigger_data *hist_data,
4185                               struct action_data *data)
4186 {
4187         unsigned int i;
4188
4189         if (data->action == ACTION_SAVE) {
4190                 for (i = 0; i < hist_data->n_save_vars; i++) {
4191                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4192                         if (i < hist_data->n_save_vars - 1)
4193                                 seq_puts(m, ",");
4194                 }
4195         } else if (data->action == ACTION_TRACE) {
4196                 if (data->use_trace_keyword)
4197                         seq_printf(m, "%s", data->synth_event_name);
4198                 for (i = 0; i < data->n_params; i++) {
4199                         if (i || data->use_trace_keyword)
4200                                 seq_puts(m, ",");
4201                         seq_printf(m, "%s", data->params[i]);
4202                 }
4203         }
4204 }
4205
4206 static void print_track_data_spec(struct seq_file *m,
4207                                   struct hist_trigger_data *hist_data,
4208                                   struct action_data *data)
4209 {
4210         if (data->handler == HANDLER_ONMAX)
4211                 seq_puts(m, ":onmax(");
4212         else if (data->handler == HANDLER_ONCHANGE)
4213                 seq_puts(m, ":onchange(");
4214         seq_printf(m, "%s", data->track_data.var_str);
4215         seq_printf(m, ").%s(", data->action_name);
4216
4217         print_action_spec(m, hist_data, data);
4218
4219         seq_puts(m, ")");
4220 }
4221
4222 static void print_onmatch_spec(struct seq_file *m,
4223                                struct hist_trigger_data *hist_data,
4224                                struct action_data *data)
4225 {
4226         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4227                    data->match_data.event);
4228
4229         seq_printf(m, "%s(", data->action_name);
4230
4231         print_action_spec(m, hist_data, data);
4232
4233         seq_puts(m, ")");
4234 }
4235
4236 static bool actions_match(struct hist_trigger_data *hist_data,
4237                           struct hist_trigger_data *hist_data_test)
4238 {
4239         unsigned int i, j;
4240
4241         if (hist_data->n_actions != hist_data_test->n_actions)
4242                 return false;
4243
4244         for (i = 0; i < hist_data->n_actions; i++) {
4245                 struct action_data *data = hist_data->actions[i];
4246                 struct action_data *data_test = hist_data_test->actions[i];
4247                 char *action_name, *action_name_test;
4248
4249                 if (data->handler != data_test->handler)
4250                         return false;
4251                 if (data->action != data_test->action)
4252                         return false;
4253
4254                 if (data->n_params != data_test->n_params)
4255                         return false;
4256
4257                 for (j = 0; j < data->n_params; j++) {
4258                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4259                                 return false;
4260                 }
4261
4262                 if (data->use_trace_keyword)
4263                         action_name = data->synth_event_name;
4264                 else
4265                         action_name = data->action_name;
4266
4267                 if (data_test->use_trace_keyword)
4268                         action_name_test = data_test->synth_event_name;
4269                 else
4270                         action_name_test = data_test->action_name;
4271
4272                 if (strcmp(action_name, action_name_test) != 0)
4273                         return false;
4274
4275                 if (data->handler == HANDLER_ONMATCH) {
4276                         if (strcmp(data->match_data.event_system,
4277                                    data_test->match_data.event_system) != 0)
4278                                 return false;
4279                         if (strcmp(data->match_data.event,
4280                                    data_test->match_data.event) != 0)
4281                                 return false;
4282                 } else if (data->handler == HANDLER_ONMAX ||
4283                            data->handler == HANDLER_ONCHANGE) {
4284                         if (strcmp(data->track_data.var_str,
4285                                    data_test->track_data.var_str) != 0)
4286                                 return false;
4287                 }
4288         }
4289
4290         return true;
4291 }
4292
4293
4294 static void print_actions_spec(struct seq_file *m,
4295                                struct hist_trigger_data *hist_data)
4296 {
4297         unsigned int i;
4298
4299         for (i = 0; i < hist_data->n_actions; i++) {
4300                 struct action_data *data = hist_data->actions[i];
4301
4302                 if (data->handler == HANDLER_ONMATCH)
4303                         print_onmatch_spec(m, hist_data, data);
4304                 else if (data->handler == HANDLER_ONMAX ||
4305                          data->handler == HANDLER_ONCHANGE)
4306                         print_track_data_spec(m, hist_data, data);
4307         }
4308 }
4309
4310 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4311 {
4312         unsigned int i;
4313
4314         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4315                 kfree(hist_data->field_var_hists[i]->cmd);
4316                 kfree(hist_data->field_var_hists[i]);
4317         }
4318 }
4319
4320 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4321 {
4322         if (!hist_data)
4323                 return;
4324
4325         destroy_hist_trigger_attrs(hist_data->attrs);
4326         destroy_hist_fields(hist_data);
4327         tracing_map_destroy(hist_data->map);
4328
4329         destroy_actions(hist_data);
4330         destroy_field_vars(hist_data);
4331         destroy_field_var_hists(hist_data);
4332
4333         kfree(hist_data);
4334 }
4335
4336 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4337 {
4338         struct tracing_map *map = hist_data->map;
4339         struct ftrace_event_field *field;
4340         struct hist_field *hist_field;
4341         int i, idx = 0;
4342
4343         for_each_hist_field(i, hist_data) {
4344                 hist_field = hist_data->fields[i];
4345                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4346                         tracing_map_cmp_fn_t cmp_fn;
4347
4348                         field = hist_field->field;
4349
4350                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4351                                 cmp_fn = tracing_map_cmp_none;
4352                         else if (!field)
4353                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4354                                                              hist_field->is_signed);
4355                         else if (is_string_field(field))
4356                                 cmp_fn = tracing_map_cmp_string;
4357                         else
4358                                 cmp_fn = tracing_map_cmp_num(field->size,
4359                                                              field->is_signed);
4360                         idx = tracing_map_add_key_field(map,
4361                                                         hist_field->offset,
4362                                                         cmp_fn);
4363                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4364                         idx = tracing_map_add_sum_field(map);
4365
4366                 if (idx < 0)
4367                         return idx;
4368
4369                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4370                         idx = tracing_map_add_var(map);
4371                         if (idx < 0)
4372                                 return idx;
4373                         hist_field->var.idx = idx;
4374                         hist_field->var.hist_data = hist_data;
4375                 }
4376         }
4377
4378         return 0;
4379 }
4380
4381 static struct hist_trigger_data *
4382 create_hist_data(unsigned int map_bits,
4383                  struct hist_trigger_attrs *attrs,
4384                  struct trace_event_file *file,
4385                  bool remove)
4386 {
4387         const struct tracing_map_ops *map_ops = NULL;
4388         struct hist_trigger_data *hist_data;
4389         int ret = 0;
4390
4391         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4392         if (!hist_data)
4393                 return ERR_PTR(-ENOMEM);
4394
4395         hist_data->attrs = attrs;
4396         hist_data->remove = remove;
4397         hist_data->event_file = file;
4398
4399         ret = parse_actions(hist_data);
4400         if (ret)
4401                 goto free;
4402
4403         ret = create_hist_fields(hist_data, file);
4404         if (ret)
4405                 goto free;
4406
4407         ret = create_sort_keys(hist_data);
4408         if (ret)
4409                 goto free;
4410
4411         map_ops = &hist_trigger_elt_data_ops;
4412
4413         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4414                                             map_ops, hist_data);
4415         if (IS_ERR(hist_data->map)) {
4416                 ret = PTR_ERR(hist_data->map);
4417                 hist_data->map = NULL;
4418                 goto free;
4419         }
4420
4421         ret = create_tracing_map_fields(hist_data);
4422         if (ret)
4423                 goto free;
4424  out:
4425         return hist_data;
4426  free:
4427         hist_data->attrs = NULL;
4428
4429         destroy_hist_data(hist_data);
4430
4431         hist_data = ERR_PTR(ret);
4432
4433         goto out;
4434 }
4435
4436 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4437                                     struct tracing_map_elt *elt,
4438                                     struct trace_buffer *buffer, void *rec,
4439                                     struct ring_buffer_event *rbe,
4440                                     u64 *var_ref_vals)
4441 {
4442         struct hist_elt_data *elt_data;
4443         struct hist_field *hist_field;
4444         unsigned int i, var_idx;
4445         u64 hist_val;
4446
4447         elt_data = elt->private_data;
4448         elt_data->var_ref_vals = var_ref_vals;
4449
4450         for_each_hist_val_field(i, hist_data) {
4451                 hist_field = hist_data->fields[i];
4452                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4453                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4454                         var_idx = hist_field->var.idx;
4455
4456                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4457                                 unsigned int str_start, var_str_idx, idx;
4458                                 char *str, *val_str;
4459
4460                                 str_start = hist_data->n_field_var_str +
4461                                         hist_data->n_save_var_str;
4462                                 var_str_idx = hist_field->var_str_idx;
4463                                 idx = str_start + var_str_idx;
4464
4465                                 str = elt_data->field_var_str[idx];
4466                                 val_str = (char *)(uintptr_t)hist_val;
4467                                 strscpy(str, val_str, STR_VAR_LEN_MAX);
4468
4469                                 hist_val = (u64)(uintptr_t)str;
4470                         }
4471                         tracing_map_set_var(elt, var_idx, hist_val);
4472                         continue;
4473                 }
4474                 tracing_map_update_sum(elt, i, hist_val);
4475         }
4476
4477         for_each_hist_key_field(i, hist_data) {
4478                 hist_field = hist_data->fields[i];
4479                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4480                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4481                         var_idx = hist_field->var.idx;
4482                         tracing_map_set_var(elt, var_idx, hist_val);
4483                 }
4484         }
4485
4486         update_field_vars(hist_data, elt, buffer, rbe, rec);
4487 }
4488
4489 static inline void add_to_key(char *compound_key, void *key,
4490                               struct hist_field *key_field, void *rec)
4491 {
4492         size_t size = key_field->size;
4493
4494         if (key_field->flags & HIST_FIELD_FL_STRING) {
4495                 struct ftrace_event_field *field;
4496
4497                 field = key_field->field;
4498                 if (field->filter_type == FILTER_DYN_STRING)
4499                         size = *(u32 *)(rec + field->offset) >> 16;
4500                 else if (field->filter_type == FILTER_STATIC_STRING)
4501                         size = field->size;
4502
4503                 /* ensure NULL-termination */
4504                 if (size > key_field->size - 1)
4505                         size = key_field->size - 1;
4506
4507                 strncpy(compound_key + key_field->offset, (char *)key, size);
4508         } else
4509                 memcpy(compound_key + key_field->offset, key, size);
4510 }
4511
4512 static void
4513 hist_trigger_actions(struct hist_trigger_data *hist_data,
4514                      struct tracing_map_elt *elt,
4515                      struct trace_buffer *buffer, void *rec,
4516                      struct ring_buffer_event *rbe, void *key,
4517                      u64 *var_ref_vals)
4518 {
4519         struct action_data *data;
4520         unsigned int i;
4521
4522         for (i = 0; i < hist_data->n_actions; i++) {
4523                 data = hist_data->actions[i];
4524                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4525         }
4526 }
4527
4528 static void event_hist_trigger(struct event_trigger_data *data,
4529                                struct trace_buffer *buffer, void *rec,
4530                                struct ring_buffer_event *rbe)
4531 {
4532         struct hist_trigger_data *hist_data = data->private_data;
4533         bool use_compound_key = (hist_data->n_keys > 1);
4534         unsigned long entries[HIST_STACKTRACE_DEPTH];
4535         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4536         char compound_key[HIST_KEY_SIZE_MAX];
4537         struct tracing_map_elt *elt = NULL;
4538         struct hist_field *key_field;
4539         u64 field_contents;
4540         void *key = NULL;
4541         unsigned int i;
4542
4543         memset(compound_key, 0, hist_data->key_size);
4544
4545         for_each_hist_key_field(i, hist_data) {
4546                 key_field = hist_data->fields[i];
4547
4548                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4549                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4550                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4551                                          HIST_STACKTRACE_SKIP);
4552                         key = entries;
4553                 } else {
4554                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4555                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4556                                 key = (void *)(unsigned long)field_contents;
4557                                 use_compound_key = true;
4558                         } else
4559                                 key = (void *)&field_contents;
4560                 }
4561
4562                 if (use_compound_key)
4563                         add_to_key(compound_key, key, key_field, rec);
4564         }
4565
4566         if (use_compound_key)
4567                 key = compound_key;
4568
4569         if (hist_data->n_var_refs &&
4570             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4571                 return;
4572
4573         elt = tracing_map_insert(hist_data->map, key);
4574         if (!elt)
4575                 return;
4576
4577         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4578
4579         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4580                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4581 }
4582
4583 static void hist_trigger_stacktrace_print(struct seq_file *m,
4584                                           unsigned long *stacktrace_entries,
4585                                           unsigned int max_entries)
4586 {
4587         char str[KSYM_SYMBOL_LEN];
4588         unsigned int spaces = 8;
4589         unsigned int i;
4590
4591         for (i = 0; i < max_entries; i++) {
4592                 if (!stacktrace_entries[i])
4593                         return;
4594
4595                 seq_printf(m, "%*c", 1 + spaces, ' ');
4596                 sprint_symbol(str, stacktrace_entries[i]);
4597                 seq_printf(m, "%s\n", str);
4598         }
4599 }
4600
4601 static void hist_trigger_print_key(struct seq_file *m,
4602                                    struct hist_trigger_data *hist_data,
4603                                    void *key,
4604                                    struct tracing_map_elt *elt)
4605 {
4606         struct hist_field *key_field;
4607         char str[KSYM_SYMBOL_LEN];
4608         bool multiline = false;
4609         const char *field_name;
4610         unsigned int i;
4611         u64 uval;
4612
4613         seq_puts(m, "{ ");
4614
4615         for_each_hist_key_field(i, hist_data) {
4616                 key_field = hist_data->fields[i];
4617
4618                 if (i > hist_data->n_vals)
4619                         seq_puts(m, ", ");
4620
4621                 field_name = hist_field_name(key_field, 0);
4622
4623                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4624                         uval = *(u64 *)(key + key_field->offset);
4625                         seq_printf(m, "%s: %llx", field_name, uval);
4626                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4627                         uval = *(u64 *)(key + key_field->offset);
4628                         sprint_symbol_no_offset(str, uval);
4629                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4630                                    uval, str);
4631                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4632                         uval = *(u64 *)(key + key_field->offset);
4633                         sprint_symbol(str, uval);
4634                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4635                                    uval, str);
4636                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4637                         struct hist_elt_data *elt_data = elt->private_data;
4638                         char *comm;
4639
4640                         if (WARN_ON_ONCE(!elt_data))
4641                                 return;
4642
4643                         comm = elt_data->comm;
4644
4645                         uval = *(u64 *)(key + key_field->offset);
4646                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4647                                    comm, uval);
4648                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4649                         const char *syscall_name;
4650
4651                         uval = *(u64 *)(key + key_field->offset);
4652                         syscall_name = get_syscall_name(uval);
4653                         if (!syscall_name)
4654                                 syscall_name = "unknown_syscall";
4655
4656                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4657                                    syscall_name, uval);
4658                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4659                         seq_puts(m, "stacktrace:\n");
4660                         hist_trigger_stacktrace_print(m,
4661                                                       key + key_field->offset,
4662                                                       HIST_STACKTRACE_DEPTH);
4663                         multiline = true;
4664                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4665                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4666                                    *(u64 *)(key + key_field->offset));
4667                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4668                         seq_printf(m, "%s: %-50s", field_name,
4669                                    (char *)(key + key_field->offset));
4670                 } else {
4671                         uval = *(u64 *)(key + key_field->offset);
4672                         seq_printf(m, "%s: %10llu", field_name, uval);
4673                 }
4674         }
4675
4676         if (!multiline)
4677                 seq_puts(m, " ");
4678
4679         seq_puts(m, "}");
4680 }
4681
4682 static void hist_trigger_entry_print(struct seq_file *m,
4683                                      struct hist_trigger_data *hist_data,
4684                                      void *key,
4685                                      struct tracing_map_elt *elt)
4686 {
4687         const char *field_name;
4688         unsigned int i;
4689
4690         hist_trigger_print_key(m, hist_data, key, elt);
4691
4692         seq_printf(m, " hitcount: %10llu",
4693                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4694
4695         for (i = 1; i < hist_data->n_vals; i++) {
4696                 field_name = hist_field_name(hist_data->fields[i], 0);
4697
4698                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4699                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4700                         continue;
4701
4702                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4703                         seq_printf(m, "  %s: %10llx", field_name,
4704                                    tracing_map_read_sum(elt, i));
4705                 } else {
4706                         seq_printf(m, "  %s: %10llu", field_name,
4707                                    tracing_map_read_sum(elt, i));
4708                 }
4709         }
4710
4711         print_actions(m, hist_data, elt);
4712
4713         seq_puts(m, "\n");
4714 }
4715
4716 static int print_entries(struct seq_file *m,
4717                          struct hist_trigger_data *hist_data)
4718 {
4719         struct tracing_map_sort_entry **sort_entries = NULL;
4720         struct tracing_map *map = hist_data->map;
4721         int i, n_entries;
4722
4723         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4724                                              hist_data->n_sort_keys,
4725                                              &sort_entries);
4726         if (n_entries < 0)
4727                 return n_entries;
4728
4729         for (i = 0; i < n_entries; i++)
4730                 hist_trigger_entry_print(m, hist_data,
4731                                          sort_entries[i]->key,
4732                                          sort_entries[i]->elt);
4733
4734         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4735
4736         return n_entries;
4737 }
4738
4739 static void hist_trigger_show(struct seq_file *m,
4740                               struct event_trigger_data *data, int n)
4741 {
4742         struct hist_trigger_data *hist_data;
4743         int n_entries;
4744
4745         if (n > 0)
4746                 seq_puts(m, "\n\n");
4747
4748         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4749         data->ops->print(m, data->ops, data);
4750         seq_puts(m, "#\n\n");
4751
4752         hist_data = data->private_data;
4753         n_entries = print_entries(m, hist_data);
4754         if (n_entries < 0)
4755                 n_entries = 0;
4756
4757         track_data_snapshot_print(m, hist_data);
4758
4759         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4760                    (u64)atomic64_read(&hist_data->map->hits),
4761                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4762 }
4763
4764 static int hist_show(struct seq_file *m, void *v)
4765 {
4766         struct event_trigger_data *data;
4767         struct trace_event_file *event_file;
4768         int n = 0, ret = 0;
4769
4770         mutex_lock(&event_mutex);
4771
4772         event_file = event_file_data(m->private);
4773         if (unlikely(!event_file)) {
4774                 ret = -ENODEV;
4775                 goto out_unlock;
4776         }
4777
4778         list_for_each_entry(data, &event_file->triggers, list) {
4779                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4780                         hist_trigger_show(m, data, n++);
4781         }
4782
4783  out_unlock:
4784         mutex_unlock(&event_mutex);
4785
4786         return ret;
4787 }
4788
4789 static int event_hist_open(struct inode *inode, struct file *file)
4790 {
4791         int ret;
4792
4793         ret = security_locked_down(LOCKDOWN_TRACEFS);
4794         if (ret)
4795                 return ret;
4796
4797         return single_open(file, hist_show, file);
4798 }
4799
4800 const struct file_operations event_hist_fops = {
4801         .open = event_hist_open,
4802         .read = seq_read,
4803         .llseek = seq_lseek,
4804         .release = single_release,
4805 };
4806
4807 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4808 static void hist_field_debug_show_flags(struct seq_file *m,
4809                                         unsigned long flags)
4810 {
4811         seq_puts(m, "      flags:\n");
4812
4813         if (flags & HIST_FIELD_FL_KEY)
4814                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4815         else if (flags & HIST_FIELD_FL_HITCOUNT)
4816                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4817         else if (flags & HIST_FIELD_FL_VAR)
4818                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4819         else if (flags & HIST_FIELD_FL_VAR_REF)
4820                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4821         else
4822                 seq_puts(m, "        VAL: normal u64 value\n");
4823
4824         if (flags & HIST_FIELD_FL_ALIAS)
4825                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4826 }
4827
4828 static int hist_field_debug_show(struct seq_file *m,
4829                                  struct hist_field *field, unsigned long flags)
4830 {
4831         if ((field->flags & flags) != flags) {
4832                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4833                 return -EINVAL;
4834         }
4835
4836         hist_field_debug_show_flags(m, field->flags);
4837         if (field->field)
4838                 seq_printf(m, "      ftrace_event_field name: %s\n",
4839                            field->field->name);
4840
4841         if (field->flags & HIST_FIELD_FL_VAR) {
4842                 seq_printf(m, "      var.name: %s\n", field->var.name);
4843                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4844                            field->var.idx);
4845         }
4846
4847         if (field->flags & HIST_FIELD_FL_ALIAS)
4848                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4849                            field->var_ref_idx);
4850
4851         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4852                 seq_printf(m, "      name: %s\n", field->name);
4853                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4854                            field->var.idx);
4855                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4856                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4857                            field->var_ref_idx);
4858                 if (field->system)
4859                         seq_printf(m, "      system: %s\n", field->system);
4860                 if (field->event_name)
4861                         seq_printf(m, "      event_name: %s\n", field->event_name);
4862         }
4863
4864         seq_printf(m, "      type: %s\n", field->type);
4865         seq_printf(m, "      size: %u\n", field->size);
4866         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4867
4868         return 0;
4869 }
4870
4871 static int field_var_debug_show(struct seq_file *m,
4872                                 struct field_var *field_var, unsigned int i,
4873                                 bool save_vars)
4874 {
4875         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4876         struct hist_field *field;
4877         int ret = 0;
4878
4879         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4880
4881         field = field_var->var;
4882
4883         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4884
4885         hist_field_debug_show_flags(m, field->flags);
4886         seq_printf(m, "      var.name: %s\n", field->var.name);
4887         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4888                    field->var.idx);
4889
4890         field = field_var->val;
4891
4892         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4893         if (field->field)
4894                 seq_printf(m, "      ftrace_event_field name: %s\n",
4895                            field->field->name);
4896         else {
4897                 ret = -EINVAL;
4898                 goto out;
4899         }
4900
4901         seq_printf(m, "      type: %s\n", field->type);
4902         seq_printf(m, "      size: %u\n", field->size);
4903         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4904 out:
4905         return ret;
4906 }
4907
4908 static int hist_action_debug_show(struct seq_file *m,
4909                                   struct action_data *data, int i)
4910 {
4911         int ret = 0;
4912
4913         if (data->handler == HANDLER_ONMAX ||
4914             data->handler == HANDLER_ONCHANGE) {
4915                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4916                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4917                                             HIST_FIELD_FL_VAR_REF);
4918                 if (ret)
4919                         goto out;
4920
4921                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4922                 ret = hist_field_debug_show(m, data->track_data.track_var,
4923                                             HIST_FIELD_FL_VAR);
4924                 if (ret)
4925                         goto out;
4926         }
4927
4928         if (data->handler == HANDLER_ONMATCH) {
4929                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4930                            i, data->match_data.event_system);
4931                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4932                            i, data->match_data.event);
4933         }
4934 out:
4935         return ret;
4936 }
4937
4938 static int hist_actions_debug_show(struct seq_file *m,
4939                                    struct hist_trigger_data *hist_data)
4940 {
4941         int i, ret = 0;
4942
4943         if (hist_data->n_actions)
4944                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4945
4946         for (i = 0; i < hist_data->n_actions; i++) {
4947                 struct action_data *action = hist_data->actions[i];
4948
4949                 ret = hist_action_debug_show(m, action, i);
4950                 if (ret)
4951                         goto out;
4952         }
4953
4954         if (hist_data->n_save_vars)
4955                 seq_puts(m, "\n  save action variables (save() params):\n");
4956
4957         for (i = 0; i < hist_data->n_save_vars; i++) {
4958                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4959                 if (ret)
4960                         goto out;
4961         }
4962 out:
4963         return ret;
4964 }
4965
4966 static void hist_trigger_debug_show(struct seq_file *m,
4967                                     struct event_trigger_data *data, int n)
4968 {
4969         struct hist_trigger_data *hist_data;
4970         int i, ret;
4971
4972         if (n > 0)
4973                 seq_puts(m, "\n\n");
4974
4975         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4976         data->ops->print(m, data->ops, data);
4977         seq_puts(m, "#\n\n");
4978
4979         hist_data = data->private_data;
4980
4981         seq_printf(m, "hist_data: %p\n\n", hist_data);
4982         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4983         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4984         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4985
4986         seq_puts(m, "\n  val fields:\n\n");
4987
4988         seq_puts(m, "    hist_data->fields[0]:\n");
4989         ret = hist_field_debug_show(m, hist_data->fields[0],
4990                                     HIST_FIELD_FL_HITCOUNT);
4991         if (ret)
4992                 return;
4993
4994         for (i = 1; i < hist_data->n_vals; i++) {
4995                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4996                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4997                 if (ret)
4998                         return;
4999         }
5000
5001         seq_puts(m, "\n  key fields:\n");
5002
5003         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5004                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5005                 ret = hist_field_debug_show(m, hist_data->fields[i],
5006                                             HIST_FIELD_FL_KEY);
5007                 if (ret)
5008                         return;
5009         }
5010
5011         if (hist_data->n_var_refs)
5012                 seq_puts(m, "\n  variable reference fields:\n");
5013
5014         for (i = 0; i < hist_data->n_var_refs; i++) {
5015                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5016                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5017                                             HIST_FIELD_FL_VAR_REF);
5018                 if (ret)
5019                         return;
5020         }
5021
5022         if (hist_data->n_field_vars)
5023                 seq_puts(m, "\n  field variables:\n");
5024
5025         for (i = 0; i < hist_data->n_field_vars; i++) {
5026                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5027                 if (ret)
5028                         return;
5029         }
5030
5031         ret = hist_actions_debug_show(m, hist_data);
5032         if (ret)
5033                 return;
5034 }
5035
5036 static int hist_debug_show(struct seq_file *m, void *v)
5037 {
5038         struct event_trigger_data *data;
5039         struct trace_event_file *event_file;
5040         int n = 0, ret = 0;
5041
5042         mutex_lock(&event_mutex);
5043
5044         event_file = event_file_data(m->private);
5045         if (unlikely(!event_file)) {
5046                 ret = -ENODEV;
5047                 goto out_unlock;
5048         }
5049
5050         list_for_each_entry(data, &event_file->triggers, list) {
5051                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5052                         hist_trigger_debug_show(m, data, n++);
5053         }
5054
5055  out_unlock:
5056         mutex_unlock(&event_mutex);
5057
5058         return ret;
5059 }
5060
5061 static int event_hist_debug_open(struct inode *inode, struct file *file)
5062 {
5063         int ret;
5064
5065         ret = security_locked_down(LOCKDOWN_TRACEFS);
5066         if (ret)
5067                 return ret;
5068
5069         return single_open(file, hist_debug_show, file);
5070 }
5071
5072 const struct file_operations event_hist_debug_fops = {
5073         .open = event_hist_debug_open,
5074         .read = seq_read,
5075         .llseek = seq_lseek,
5076         .release = single_release,
5077 };
5078 #endif
5079
5080 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5081 {
5082         const char *field_name = hist_field_name(hist_field, 0);
5083
5084         if (hist_field->var.name)
5085                 seq_printf(m, "%s=", hist_field->var.name);
5086
5087         if (hist_field->flags & HIST_FIELD_FL_CPU)
5088                 seq_puts(m, "cpu");
5089         else if (field_name) {
5090                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5091                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5092                         seq_putc(m, '$');
5093                 seq_printf(m, "%s", field_name);
5094         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5095                 seq_puts(m, "common_timestamp");
5096
5097         if (hist_field->flags) {
5098                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5099                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5100                         const char *flags = get_hist_field_flags(hist_field);
5101
5102                         if (flags)
5103                                 seq_printf(m, ".%s", flags);
5104                 }
5105         }
5106 }
5107
5108 static int event_hist_trigger_print(struct seq_file *m,
5109                                     struct event_trigger_ops *ops,
5110                                     struct event_trigger_data *data)
5111 {
5112         struct hist_trigger_data *hist_data = data->private_data;
5113         struct hist_field *field;
5114         bool have_var = false;
5115         unsigned int i;
5116
5117         seq_puts(m, "hist:");
5118
5119         if (data->name)
5120                 seq_printf(m, "%s:", data->name);
5121
5122         seq_puts(m, "keys=");
5123
5124         for_each_hist_key_field(i, hist_data) {
5125                 field = hist_data->fields[i];
5126
5127                 if (i > hist_data->n_vals)
5128                         seq_puts(m, ",");
5129
5130                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5131                         seq_puts(m, "stacktrace");
5132                 else
5133                         hist_field_print(m, field);
5134         }
5135
5136         seq_puts(m, ":vals=");
5137
5138         for_each_hist_val_field(i, hist_data) {
5139                 field = hist_data->fields[i];
5140                 if (field->flags & HIST_FIELD_FL_VAR) {
5141                         have_var = true;
5142                         continue;
5143                 }
5144
5145                 if (i == HITCOUNT_IDX)
5146                         seq_puts(m, "hitcount");
5147                 else {
5148                         seq_puts(m, ",");
5149                         hist_field_print(m, field);
5150                 }
5151         }
5152
5153         if (have_var) {
5154                 unsigned int n = 0;
5155
5156                 seq_puts(m, ":");
5157
5158                 for_each_hist_val_field(i, hist_data) {
5159                         field = hist_data->fields[i];
5160
5161                         if (field->flags & HIST_FIELD_FL_VAR) {
5162                                 if (n++)
5163                                         seq_puts(m, ",");
5164                                 hist_field_print(m, field);
5165                         }
5166                 }
5167         }
5168
5169         seq_puts(m, ":sort=");
5170
5171         for (i = 0; i < hist_data->n_sort_keys; i++) {
5172                 struct tracing_map_sort_key *sort_key;
5173                 unsigned int idx, first_key_idx;
5174
5175                 /* skip VAR vals */
5176                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5177
5178                 sort_key = &hist_data->sort_keys[i];
5179                 idx = sort_key->field_idx;
5180
5181                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5182                         return -EINVAL;
5183
5184                 if (i > 0)
5185                         seq_puts(m, ",");
5186
5187                 if (idx == HITCOUNT_IDX)
5188                         seq_puts(m, "hitcount");
5189                 else {
5190                         if (idx >= first_key_idx)
5191                                 idx += hist_data->n_vars;
5192                         hist_field_print(m, hist_data->fields[idx]);
5193                 }
5194
5195                 if (sort_key->descending)
5196                         seq_puts(m, ".descending");
5197         }
5198         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5199         if (hist_data->enable_timestamps)
5200                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5201
5202         print_actions_spec(m, hist_data);
5203
5204         if (data->filter_str)
5205                 seq_printf(m, " if %s", data->filter_str);
5206
5207         if (data->paused)
5208                 seq_puts(m, " [paused]");
5209         else
5210                 seq_puts(m, " [active]");
5211
5212         seq_putc(m, '\n');
5213
5214         return 0;
5215 }
5216
5217 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5218                                    struct event_trigger_data *data)
5219 {
5220         struct hist_trigger_data *hist_data = data->private_data;
5221
5222         if (!data->ref && hist_data->attrs->name)
5223                 save_named_trigger(hist_data->attrs->name, data);
5224
5225         data->ref++;
5226
5227         return 0;
5228 }
5229
5230 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5231 {
5232         struct trace_event_file *file;
5233         unsigned int i;
5234         char *cmd;
5235         int ret;
5236
5237         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5238                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5239                 cmd = hist_data->field_var_hists[i]->cmd;
5240                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5241                                               "!hist", "hist", cmd);
5242                 WARN_ON_ONCE(ret < 0);
5243         }
5244 }
5245
5246 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5247                                     struct event_trigger_data *data)
5248 {
5249         struct hist_trigger_data *hist_data = data->private_data;
5250
5251         if (WARN_ON_ONCE(data->ref <= 0))
5252                 return;
5253
5254         data->ref--;
5255         if (!data->ref) {
5256                 if (data->name)
5257                         del_named_trigger(data);
5258
5259                 trigger_data_free(data);
5260
5261                 remove_hist_vars(hist_data);
5262
5263                 unregister_field_var_hists(hist_data);
5264
5265                 destroy_hist_data(hist_data);
5266         }
5267 }
5268
5269 static struct event_trigger_ops event_hist_trigger_ops = {
5270         .func                   = event_hist_trigger,
5271         .print                  = event_hist_trigger_print,
5272         .init                   = event_hist_trigger_init,
5273         .free                   = event_hist_trigger_free,
5274 };
5275
5276 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5277                                          struct event_trigger_data *data)
5278 {
5279         data->ref++;
5280
5281         save_named_trigger(data->named_data->name, data);
5282
5283         event_hist_trigger_init(ops, data->named_data);
5284
5285         return 0;
5286 }
5287
5288 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5289                                           struct event_trigger_data *data)
5290 {
5291         if (WARN_ON_ONCE(data->ref <= 0))
5292                 return;
5293
5294         event_hist_trigger_free(ops, data->named_data);
5295
5296         data->ref--;
5297         if (!data->ref) {
5298                 del_named_trigger(data);
5299                 trigger_data_free(data);
5300         }
5301 }
5302
5303 static struct event_trigger_ops event_hist_trigger_named_ops = {
5304         .func                   = event_hist_trigger,
5305         .print                  = event_hist_trigger_print,
5306         .init                   = event_hist_trigger_named_init,
5307         .free                   = event_hist_trigger_named_free,
5308 };
5309
5310 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5311                                                             char *param)
5312 {
5313         return &event_hist_trigger_ops;
5314 }
5315
5316 static void hist_clear(struct event_trigger_data *data)
5317 {
5318         struct hist_trigger_data *hist_data = data->private_data;
5319
5320         if (data->name)
5321                 pause_named_trigger(data);
5322
5323         tracepoint_synchronize_unregister();
5324
5325         tracing_map_clear(hist_data->map);
5326
5327         if (data->name)
5328                 unpause_named_trigger(data);
5329 }
5330
5331 static bool compatible_field(struct ftrace_event_field *field,
5332                              struct ftrace_event_field *test_field)
5333 {
5334         if (field == test_field)
5335                 return true;
5336         if (field == NULL || test_field == NULL)
5337                 return false;
5338         if (strcmp(field->name, test_field->name) != 0)
5339                 return false;
5340         if (strcmp(field->type, test_field->type) != 0)
5341                 return false;
5342         if (field->size != test_field->size)
5343                 return false;
5344         if (field->is_signed != test_field->is_signed)
5345                 return false;
5346
5347         return true;
5348 }
5349
5350 static bool hist_trigger_match(struct event_trigger_data *data,
5351                                struct event_trigger_data *data_test,
5352                                struct event_trigger_data *named_data,
5353                                bool ignore_filter)
5354 {
5355         struct tracing_map_sort_key *sort_key, *sort_key_test;
5356         struct hist_trigger_data *hist_data, *hist_data_test;
5357         struct hist_field *key_field, *key_field_test;
5358         unsigned int i;
5359
5360         if (named_data && (named_data != data_test) &&
5361             (named_data != data_test->named_data))
5362                 return false;
5363
5364         if (!named_data && is_named_trigger(data_test))
5365                 return false;
5366
5367         hist_data = data->private_data;
5368         hist_data_test = data_test->private_data;
5369
5370         if (hist_data->n_vals != hist_data_test->n_vals ||
5371             hist_data->n_fields != hist_data_test->n_fields ||
5372             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5373                 return false;
5374
5375         if (!ignore_filter) {
5376                 if ((data->filter_str && !data_test->filter_str) ||
5377                    (!data->filter_str && data_test->filter_str))
5378                         return false;
5379         }
5380
5381         for_each_hist_field(i, hist_data) {
5382                 key_field = hist_data->fields[i];
5383                 key_field_test = hist_data_test->fields[i];
5384
5385                 if (key_field->flags != key_field_test->flags)
5386                         return false;
5387                 if (!compatible_field(key_field->field, key_field_test->field))
5388                         return false;
5389                 if (key_field->offset != key_field_test->offset)
5390                         return false;
5391                 if (key_field->size != key_field_test->size)
5392                         return false;
5393                 if (key_field->is_signed != key_field_test->is_signed)
5394                         return false;
5395                 if (!!key_field->var.name != !!key_field_test->var.name)
5396                         return false;
5397                 if (key_field->var.name &&
5398                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5399                         return false;
5400         }
5401
5402         for (i = 0; i < hist_data->n_sort_keys; i++) {
5403                 sort_key = &hist_data->sort_keys[i];
5404                 sort_key_test = &hist_data_test->sort_keys[i];
5405
5406                 if (sort_key->field_idx != sort_key_test->field_idx ||
5407                     sort_key->descending != sort_key_test->descending)
5408                         return false;
5409         }
5410
5411         if (!ignore_filter && data->filter_str &&
5412             (strcmp(data->filter_str, data_test->filter_str) != 0))
5413                 return false;
5414
5415         if (!actions_match(hist_data, hist_data_test))
5416                 return false;
5417
5418         return true;
5419 }
5420
5421 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5422                                  struct event_trigger_data *data,
5423                                  struct trace_event_file *file)
5424 {
5425         struct hist_trigger_data *hist_data = data->private_data;
5426         struct event_trigger_data *test, *named_data = NULL;
5427         struct trace_array *tr = file->tr;
5428         int ret = 0;
5429
5430         if (hist_data->attrs->name) {
5431                 named_data = find_named_trigger(hist_data->attrs->name);
5432                 if (named_data) {
5433                         if (!hist_trigger_match(data, named_data, named_data,
5434                                                 true)) {
5435                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5436                                 ret = -EINVAL;
5437                                 goto out;
5438                         }
5439                 }
5440         }
5441
5442         if (hist_data->attrs->name && !named_data)
5443                 goto new;
5444
5445         lockdep_assert_held(&event_mutex);
5446
5447         list_for_each_entry(test, &file->triggers, list) {
5448                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5449                         if (!hist_trigger_match(data, test, named_data, false))
5450                                 continue;
5451                         if (hist_data->attrs->pause)
5452                                 test->paused = true;
5453                         else if (hist_data->attrs->cont)
5454                                 test->paused = false;
5455                         else if (hist_data->attrs->clear)
5456                                 hist_clear(test);
5457                         else {
5458                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5459                                 ret = -EEXIST;
5460                         }
5461                         goto out;
5462                 }
5463         }
5464  new:
5465         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5466                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5467                 ret = -ENOENT;
5468                 goto out;
5469         }
5470
5471         if (hist_data->attrs->pause)
5472                 data->paused = true;
5473
5474         if (named_data) {
5475                 data->private_data = named_data->private_data;
5476                 set_named_trigger_data(data, named_data);
5477                 data->ops = &event_hist_trigger_named_ops;
5478         }
5479
5480         if (data->ops->init) {
5481                 ret = data->ops->init(data->ops, data);
5482                 if (ret < 0)
5483                         goto out;
5484         }
5485
5486         if (hist_data->enable_timestamps) {
5487                 char *clock = hist_data->attrs->clock;
5488
5489                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5490                 if (ret) {
5491                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5492                         goto out;
5493                 }
5494
5495                 tracing_set_filter_buffering(file->tr, true);
5496         }
5497
5498         if (named_data)
5499                 destroy_hist_data(hist_data);
5500
5501         ret++;
5502  out:
5503         return ret;
5504 }
5505
5506 static int hist_trigger_enable(struct event_trigger_data *data,
5507                                struct trace_event_file *file)
5508 {
5509         int ret = 0;
5510
5511         list_add_tail_rcu(&data->list, &file->triggers);
5512
5513         update_cond_flag(file);
5514
5515         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5516                 list_del_rcu(&data->list);
5517                 update_cond_flag(file);
5518                 ret--;
5519         }
5520
5521         return ret;
5522 }
5523
5524 static bool have_hist_trigger_match(struct event_trigger_data *data,
5525                                     struct trace_event_file *file)
5526 {
5527         struct hist_trigger_data *hist_data = data->private_data;
5528         struct event_trigger_data *test, *named_data = NULL;
5529         bool match = false;
5530
5531         lockdep_assert_held(&event_mutex);
5532
5533         if (hist_data->attrs->name)
5534                 named_data = find_named_trigger(hist_data->attrs->name);
5535
5536         list_for_each_entry(test, &file->triggers, list) {
5537                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5538                         if (hist_trigger_match(data, test, named_data, false)) {
5539                                 match = true;
5540                                 break;
5541                         }
5542                 }
5543         }
5544
5545         return match;
5546 }
5547
5548 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5549                                     struct trace_event_file *file)
5550 {
5551         struct hist_trigger_data *hist_data = data->private_data;
5552         struct event_trigger_data *test, *named_data = NULL;
5553
5554         lockdep_assert_held(&event_mutex);
5555
5556         if (hist_data->attrs->name)
5557                 named_data = find_named_trigger(hist_data->attrs->name);
5558
5559         list_for_each_entry(test, &file->triggers, list) {
5560                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5561                         if (!hist_trigger_match(data, test, named_data, false))
5562                                 continue;
5563                         hist_data = test->private_data;
5564                         if (check_var_refs(hist_data))
5565                                 return true;
5566                         break;
5567                 }
5568         }
5569
5570         return false;
5571 }
5572
5573 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5574                                     struct event_trigger_data *data,
5575                                     struct trace_event_file *file)
5576 {
5577         struct hist_trigger_data *hist_data = data->private_data;
5578         struct event_trigger_data *test, *named_data = NULL;
5579         bool unregistered = false;
5580
5581         lockdep_assert_held(&event_mutex);
5582
5583         if (hist_data->attrs->name)
5584                 named_data = find_named_trigger(hist_data->attrs->name);
5585
5586         list_for_each_entry(test, &file->triggers, list) {
5587                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5588                         if (!hist_trigger_match(data, test, named_data, false))
5589                                 continue;
5590                         unregistered = true;
5591                         list_del_rcu(&test->list);
5592                         trace_event_trigger_enable_disable(file, 0);
5593                         update_cond_flag(file);
5594                         break;
5595                 }
5596         }
5597
5598         if (unregistered && test->ops->free)
5599                 test->ops->free(test->ops, test);
5600
5601         if (hist_data->enable_timestamps) {
5602                 if (!hist_data->remove || unregistered)
5603                         tracing_set_filter_buffering(file->tr, false);
5604         }
5605 }
5606
5607 static bool hist_file_check_refs(struct trace_event_file *file)
5608 {
5609         struct hist_trigger_data *hist_data;
5610         struct event_trigger_data *test;
5611
5612         lockdep_assert_held(&event_mutex);
5613
5614         list_for_each_entry(test, &file->triggers, list) {
5615                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5616                         hist_data = test->private_data;
5617                         if (check_var_refs(hist_data))
5618                                 return true;
5619                 }
5620         }
5621
5622         return false;
5623 }
5624
5625 static void hist_unreg_all(struct trace_event_file *file)
5626 {
5627         struct event_trigger_data *test, *n;
5628         struct hist_trigger_data *hist_data;
5629         struct synth_event *se;
5630         const char *se_name;
5631
5632         lockdep_assert_held(&event_mutex);
5633
5634         if (hist_file_check_refs(file))
5635                 return;
5636
5637         list_for_each_entry_safe(test, n, &file->triggers, list) {
5638                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5639                         hist_data = test->private_data;
5640                         list_del_rcu(&test->list);
5641                         trace_event_trigger_enable_disable(file, 0);
5642
5643                         se_name = trace_event_name(file->event_call);
5644                         se = find_synth_event(se_name);
5645                         if (se)
5646                                 se->ref--;
5647
5648                         update_cond_flag(file);
5649                         if (hist_data->enable_timestamps)
5650                                 tracing_set_filter_buffering(file->tr, false);
5651                         if (test->ops->free)
5652                                 test->ops->free(test->ops, test);
5653                 }
5654         }
5655 }
5656
5657 static int event_hist_trigger_func(struct event_command *cmd_ops,
5658                                    struct trace_event_file *file,
5659                                    char *glob, char *cmd, char *param)
5660 {
5661         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5662         struct event_trigger_data *trigger_data;
5663         struct hist_trigger_attrs *attrs;
5664         struct event_trigger_ops *trigger_ops;
5665         struct hist_trigger_data *hist_data;
5666         struct synth_event *se;
5667         const char *se_name;
5668         bool remove = false;
5669         char *trigger, *p;
5670         int ret = 0;
5671
5672         lockdep_assert_held(&event_mutex);
5673
5674         if (glob && strlen(glob)) {
5675                 hist_err_clear();
5676                 last_cmd_set(file, param);
5677         }
5678
5679         if (!param)
5680                 return -EINVAL;
5681
5682         if (glob[0] == '!')
5683                 remove = true;
5684
5685         /*
5686          * separate the trigger from the filter (k:v [if filter])
5687          * allowing for whitespace in the trigger
5688          */
5689         p = trigger = param;
5690         do {
5691                 p = strstr(p, "if");
5692                 if (!p)
5693                         break;
5694                 if (p == param)
5695                         return -EINVAL;
5696                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5697                         p++;
5698                         continue;
5699                 }
5700                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5701                         return -EINVAL;
5702                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5703                         p++;
5704                         continue;
5705                 }
5706                 break;
5707         } while (p);
5708
5709         if (!p)
5710                 param = NULL;
5711         else {
5712                 *(p - 1) = '\0';
5713                 param = strstrip(p);
5714                 trigger = strstrip(trigger);
5715         }
5716
5717         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5718         if (IS_ERR(attrs))
5719                 return PTR_ERR(attrs);
5720
5721         if (attrs->map_bits)
5722                 hist_trigger_bits = attrs->map_bits;
5723
5724         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5725         if (IS_ERR(hist_data)) {
5726                 destroy_hist_trigger_attrs(attrs);
5727                 return PTR_ERR(hist_data);
5728         }
5729
5730         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5731
5732         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5733         if (!trigger_data) {
5734                 ret = -ENOMEM;
5735                 goto out_free;
5736         }
5737
5738         trigger_data->count = -1;
5739         trigger_data->ops = trigger_ops;
5740         trigger_data->cmd_ops = cmd_ops;
5741
5742         INIT_LIST_HEAD(&trigger_data->list);
5743         RCU_INIT_POINTER(trigger_data->filter, NULL);
5744
5745         trigger_data->private_data = hist_data;
5746
5747         /* if param is non-empty, it's supposed to be a filter */
5748         if (param && cmd_ops->set_filter) {
5749                 ret = cmd_ops->set_filter(param, trigger_data, file);
5750                 if (ret < 0)
5751                         goto out_free;
5752         }
5753
5754         if (remove) {
5755                 if (!have_hist_trigger_match(trigger_data, file))
5756                         goto out_free;
5757
5758                 if (hist_trigger_check_refs(trigger_data, file)) {
5759                         ret = -EBUSY;
5760                         goto out_free;
5761                 }
5762
5763                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5764                 se_name = trace_event_name(file->event_call);
5765                 se = find_synth_event(se_name);
5766                 if (se)
5767                         se->ref--;
5768                 ret = 0;
5769                 goto out_free;
5770         }
5771
5772         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5773         /*
5774          * The above returns on success the # of triggers registered,
5775          * but if it didn't register any it returns zero.  Consider no
5776          * triggers registered a failure too.
5777          */
5778         if (!ret) {
5779                 if (!(attrs->pause || attrs->cont || attrs->clear))
5780                         ret = -ENOENT;
5781                 goto out_free;
5782         } else if (ret < 0)
5783                 goto out_free;
5784
5785         if (get_named_trigger_data(trigger_data))
5786                 goto enable;
5787
5788         if (has_hist_vars(hist_data))
5789                 save_hist_vars(hist_data);
5790
5791         ret = create_actions(hist_data);
5792         if (ret)
5793                 goto out_unreg;
5794
5795         ret = tracing_map_init(hist_data->map);
5796         if (ret)
5797                 goto out_unreg;
5798 enable:
5799         ret = hist_trigger_enable(trigger_data, file);
5800         if (ret)
5801                 goto out_unreg;
5802
5803         se_name = trace_event_name(file->event_call);
5804         se = find_synth_event(se_name);
5805         if (se)
5806                 se->ref++;
5807         /* Just return zero, not the number of registered triggers */
5808         ret = 0;
5809  out:
5810         if (ret == 0)
5811                 hist_err_clear();
5812
5813         return ret;
5814  out_unreg:
5815         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5816  out_free:
5817         if (cmd_ops->set_filter)
5818                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5819
5820         remove_hist_vars(hist_data);
5821
5822         kfree(trigger_data);
5823
5824         destroy_hist_data(hist_data);
5825         goto out;
5826 }
5827
5828 static struct event_command trigger_hist_cmd = {
5829         .name                   = "hist",
5830         .trigger_type           = ETT_EVENT_HIST,
5831         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5832         .func                   = event_hist_trigger_func,
5833         .reg                    = hist_register_trigger,
5834         .unreg                  = hist_unregister_trigger,
5835         .unreg_all              = hist_unreg_all,
5836         .get_trigger_ops        = event_hist_get_trigger_ops,
5837         .set_filter             = set_trigger_filter,
5838 };
5839
5840 __init int register_trigger_hist_cmd(void)
5841 {
5842         int ret;
5843
5844         ret = register_event_command(&trigger_hist_cmd);
5845         WARN_ON(ret < 0);
5846
5847         return ret;
5848 }
5849
5850 static void
5851 hist_enable_trigger(struct event_trigger_data *data,
5852                     struct trace_buffer *buffer,  void *rec,
5853                     struct ring_buffer_event *event)
5854 {
5855         struct enable_trigger_data *enable_data = data->private_data;
5856         struct event_trigger_data *test;
5857
5858         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5859                                 lockdep_is_held(&event_mutex)) {
5860                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5861                         if (enable_data->enable)
5862                                 test->paused = false;
5863                         else
5864                                 test->paused = true;
5865                 }
5866         }
5867 }
5868
5869 static void
5870 hist_enable_count_trigger(struct event_trigger_data *data,
5871                           struct trace_buffer *buffer,  void *rec,
5872                           struct ring_buffer_event *event)
5873 {
5874         if (!data->count)
5875                 return;
5876
5877         if (data->count != -1)
5878                 (data->count)--;
5879
5880         hist_enable_trigger(data, buffer, rec, event);
5881 }
5882
5883 static struct event_trigger_ops hist_enable_trigger_ops = {
5884         .func                   = hist_enable_trigger,
5885         .print                  = event_enable_trigger_print,
5886         .init                   = event_trigger_init,
5887         .free                   = event_enable_trigger_free,
5888 };
5889
5890 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5891         .func                   = hist_enable_count_trigger,
5892         .print                  = event_enable_trigger_print,
5893         .init                   = event_trigger_init,
5894         .free                   = event_enable_trigger_free,
5895 };
5896
5897 static struct event_trigger_ops hist_disable_trigger_ops = {
5898         .func                   = hist_enable_trigger,
5899         .print                  = event_enable_trigger_print,
5900         .init                   = event_trigger_init,
5901         .free                   = event_enable_trigger_free,
5902 };
5903
5904 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5905         .func                   = hist_enable_count_trigger,
5906         .print                  = event_enable_trigger_print,
5907         .init                   = event_trigger_init,
5908         .free                   = event_enable_trigger_free,
5909 };
5910
5911 static struct event_trigger_ops *
5912 hist_enable_get_trigger_ops(char *cmd, char *param)
5913 {
5914         struct event_trigger_ops *ops;
5915         bool enable;
5916
5917         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5918
5919         if (enable)
5920                 ops = param ? &hist_enable_count_trigger_ops :
5921                         &hist_enable_trigger_ops;
5922         else
5923                 ops = param ? &hist_disable_count_trigger_ops :
5924                         &hist_disable_trigger_ops;
5925
5926         return ops;
5927 }
5928
5929 static void hist_enable_unreg_all(struct trace_event_file *file)
5930 {
5931         struct event_trigger_data *test, *n;
5932
5933         list_for_each_entry_safe(test, n, &file->triggers, list) {
5934                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5935                         list_del_rcu(&test->list);
5936                         update_cond_flag(file);
5937                         trace_event_trigger_enable_disable(file, 0);
5938                         if (test->ops->free)
5939                                 test->ops->free(test->ops, test);
5940                 }
5941         }
5942 }
5943
5944 static struct event_command trigger_hist_enable_cmd = {
5945         .name                   = ENABLE_HIST_STR,
5946         .trigger_type           = ETT_HIST_ENABLE,
5947         .func                   = event_enable_trigger_func,
5948         .reg                    = event_enable_register_trigger,
5949         .unreg                  = event_enable_unregister_trigger,
5950         .unreg_all              = hist_enable_unreg_all,
5951         .get_trigger_ops        = hist_enable_get_trigger_ops,
5952         .set_filter             = set_trigger_filter,
5953 };
5954
5955 static struct event_command trigger_hist_disable_cmd = {
5956         .name                   = DISABLE_HIST_STR,
5957         .trigger_type           = ETT_HIST_ENABLE,
5958         .func                   = event_enable_trigger_func,
5959         .reg                    = event_enable_register_trigger,
5960         .unreg                  = event_enable_unregister_trigger,
5961         .unreg_all              = hist_enable_unreg_all,
5962         .get_trigger_ops        = hist_enable_get_trigger_ops,
5963         .set_filter             = set_trigger_filter,
5964 };
5965
5966 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5967 {
5968         unregister_event_command(&trigger_hist_enable_cmd);
5969         unregister_event_command(&trigger_hist_disable_cmd);
5970 }
5971
5972 __init int register_trigger_hist_enable_disable_cmds(void)
5973 {
5974         int ret;
5975
5976         ret = register_event_command(&trigger_hist_enable_cmd);
5977         if (WARN_ON(ret < 0))
5978                 return ret;
5979         ret = register_event_command(&trigger_hist_disable_cmd);
5980         if (WARN_ON(ret < 0))
5981                 unregister_trigger_hist_enable_disable_cmds();
5982
5983         return ret;
5984 }