1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer based function tracer
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/security.h>
21 #include <linux/seq_file.h>
22 #include <linux/notifier.h>
23 #include <linux/irqflags.h>
24 #include <linux/debugfs.h>
25 #include <linux/tracefs.h>
26 #include <linux/pagemap.h>
27 #include <linux/hardirq.h>
28 #include <linux/linkage.h>
29 #include <linux/uaccess.h>
30 #include <linux/vmalloc.h>
31 #include <linux/ftrace.h>
32 #include <linux/module.h>
33 #include <linux/percpu.h>
34 #include <linux/splice.h>
35 #include <linux/kdebug.h>
36 #include <linux/string.h>
37 #include <linux/mount.h>
38 #include <linux/rwsem.h>
39 #include <linux/slab.h>
40 #include <linux/ctype.h>
41 #include <linux/init.h>
42 #include <linux/panic_notifier.h>
43 #include <linux/poll.h>
44 #include <linux/nmi.h>
46 #include <linux/trace.h>
47 #include <linux/sched/clock.h>
48 #include <linux/sched/rt.h>
49 #include <linux/fsnotify.h>
50 #include <linux/irq_work.h>
51 #include <linux/workqueue.h>
54 #include "trace_output.h"
57 * On boot up, the ring buffer is set to the minimum size, so that
58 * we do not waste memory on systems that are not using tracing.
60 bool ring_buffer_expanded;
63 * We need to change this state when a selftest is running.
64 * A selftest will lurk into the ring-buffer to count the
65 * entries inserted during the selftest although some concurrent
66 * insertions into the ring-buffer such as trace_printk could occurred
67 * at the same time, giving false positive or negative results.
69 static bool __read_mostly tracing_selftest_running;
72 * If boot-time tracing including tracers/events via kernel cmdline
73 * is running, we do not want to run SELFTEST.
75 bool __read_mostly tracing_selftest_disabled;
77 #ifdef CONFIG_FTRACE_STARTUP_TEST
78 void __init disable_tracing_selftest(const char *reason)
80 if (!tracing_selftest_disabled) {
81 tracing_selftest_disabled = true;
82 pr_info("Ftrace startup test is disabled due to %s\n", reason);
87 /* Pipe tracepoints to printk */
88 struct trace_iterator *tracepoint_print_iter;
89 int tracepoint_printk;
90 static bool tracepoint_printk_stop_on_boot __initdata;
91 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
93 /* For tracers that don't implement custom flags */
94 static struct tracer_opt dummy_tracer_opt[] = {
99 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
105 * To prevent the comm cache from being overwritten when no
106 * tracing is active, only save the comm when a trace event
109 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
112 * Kill all tracing for good (never come back).
113 * It is initialized to 1 but will turn to zero if the initialization
114 * of the tracer is successful. But that is the only place that sets
117 static int tracing_disabled = 1;
119 cpumask_var_t __read_mostly tracing_buffer_mask;
122 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
124 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
125 * is set, then ftrace_dump is called. This will output the contents
126 * of the ftrace buffers to the console. This is very useful for
127 * capturing traces that lead to crashes and outputing it to a
130 * It is default off, but you can enable it with either specifying
131 * "ftrace_dump_on_oops" in the kernel command line, or setting
132 * /proc/sys/kernel/ftrace_dump_on_oops
133 * Set 1 if you want to dump buffers of all CPUs
134 * Set 2 if you want to dump the buffer of the CPU that triggered oops
137 enum ftrace_dump_mode ftrace_dump_on_oops;
139 /* When set, tracing will stop when a WARN*() is hit */
140 int __disable_trace_on_warning;
142 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
143 /* Map of enums to their values, for "eval_map" file */
144 struct trace_eval_map_head {
146 unsigned long length;
149 union trace_eval_map_item;
151 struct trace_eval_map_tail {
153 * "end" is first and points to NULL as it must be different
154 * than "mod" or "eval_string"
156 union trace_eval_map_item *next;
157 const char *end; /* points to NULL */
160 static DEFINE_MUTEX(trace_eval_mutex);
163 * The trace_eval_maps are saved in an array with two extra elements,
164 * one at the beginning, and one at the end. The beginning item contains
165 * the count of the saved maps (head.length), and the module they
166 * belong to if not built in (head.mod). The ending item contains a
167 * pointer to the next array of saved eval_map items.
169 union trace_eval_map_item {
170 struct trace_eval_map map;
171 struct trace_eval_map_head head;
172 struct trace_eval_map_tail tail;
175 static union trace_eval_map_item *trace_eval_maps;
176 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
178 int tracing_set_tracer(struct trace_array *tr, const char *buf);
179 static void ftrace_trace_userstack(struct trace_array *tr,
180 struct trace_buffer *buffer,
181 unsigned int trace_ctx);
183 #define MAX_TRACER_SIZE 100
184 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
185 static char *default_bootup_tracer;
187 static bool allocate_snapshot;
189 static int __init set_cmdline_ftrace(char *str)
191 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
192 default_bootup_tracer = bootup_tracer_buf;
193 /* We are using ftrace early, expand it */
194 ring_buffer_expanded = true;
197 __setup("ftrace=", set_cmdline_ftrace);
199 static int __init set_ftrace_dump_on_oops(char *str)
201 if (*str++ != '=' || !*str || !strcmp("1", str)) {
202 ftrace_dump_on_oops = DUMP_ALL;
206 if (!strcmp("orig_cpu", str) || !strcmp("2", str)) {
207 ftrace_dump_on_oops = DUMP_ORIG;
213 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
215 static int __init stop_trace_on_warning(char *str)
217 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
218 __disable_trace_on_warning = 1;
221 __setup("traceoff_on_warning", stop_trace_on_warning);
223 static int __init boot_alloc_snapshot(char *str)
225 allocate_snapshot = true;
226 /* We also need the main ring buffer expanded */
227 ring_buffer_expanded = true;
230 __setup("alloc_snapshot", boot_alloc_snapshot);
233 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
235 static int __init set_trace_boot_options(char *str)
237 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
240 __setup("trace_options=", set_trace_boot_options);
242 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
243 static char *trace_boot_clock __initdata;
245 static int __init set_trace_boot_clock(char *str)
247 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
248 trace_boot_clock = trace_boot_clock_buf;
251 __setup("trace_clock=", set_trace_boot_clock);
253 static int __init set_tracepoint_printk(char *str)
255 /* Ignore the "tp_printk_stop_on_boot" param */
259 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
260 tracepoint_printk = 1;
263 __setup("tp_printk", set_tracepoint_printk);
265 static int __init set_tracepoint_printk_stop(char *str)
267 tracepoint_printk_stop_on_boot = true;
270 __setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
272 unsigned long long ns2usecs(u64 nsec)
280 trace_process_export(struct trace_export *export,
281 struct ring_buffer_event *event, int flag)
283 struct trace_entry *entry;
284 unsigned int size = 0;
286 if (export->flags & flag) {
287 entry = ring_buffer_event_data(event);
288 size = ring_buffer_event_length(event);
289 export->write(export, entry, size);
293 static DEFINE_MUTEX(ftrace_export_lock);
295 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
297 static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
298 static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
299 static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
301 static inline void ftrace_exports_enable(struct trace_export *export)
303 if (export->flags & TRACE_EXPORT_FUNCTION)
304 static_branch_inc(&trace_function_exports_enabled);
306 if (export->flags & TRACE_EXPORT_EVENT)
307 static_branch_inc(&trace_event_exports_enabled);
309 if (export->flags & TRACE_EXPORT_MARKER)
310 static_branch_inc(&trace_marker_exports_enabled);
313 static inline void ftrace_exports_disable(struct trace_export *export)
315 if (export->flags & TRACE_EXPORT_FUNCTION)
316 static_branch_dec(&trace_function_exports_enabled);
318 if (export->flags & TRACE_EXPORT_EVENT)
319 static_branch_dec(&trace_event_exports_enabled);
321 if (export->flags & TRACE_EXPORT_MARKER)
322 static_branch_dec(&trace_marker_exports_enabled);
325 static void ftrace_exports(struct ring_buffer_event *event, int flag)
327 struct trace_export *export;
329 preempt_disable_notrace();
331 export = rcu_dereference_raw_check(ftrace_exports_list);
333 trace_process_export(export, event, flag);
334 export = rcu_dereference_raw_check(export->next);
337 preempt_enable_notrace();
341 add_trace_export(struct trace_export **list, struct trace_export *export)
343 rcu_assign_pointer(export->next, *list);
345 * We are entering export into the list but another
346 * CPU might be walking that list. We need to make sure
347 * the export->next pointer is valid before another CPU sees
348 * the export pointer included into the list.
350 rcu_assign_pointer(*list, export);
354 rm_trace_export(struct trace_export **list, struct trace_export *export)
356 struct trace_export **p;
358 for (p = list; *p != NULL; p = &(*p)->next)
365 rcu_assign_pointer(*p, (*p)->next);
371 add_ftrace_export(struct trace_export **list, struct trace_export *export)
373 ftrace_exports_enable(export);
375 add_trace_export(list, export);
379 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
383 ret = rm_trace_export(list, export);
384 ftrace_exports_disable(export);
389 int register_ftrace_export(struct trace_export *export)
391 if (WARN_ON_ONCE(!export->write))
394 mutex_lock(&ftrace_export_lock);
396 add_ftrace_export(&ftrace_exports_list, export);
398 mutex_unlock(&ftrace_export_lock);
402 EXPORT_SYMBOL_GPL(register_ftrace_export);
404 int unregister_ftrace_export(struct trace_export *export)
408 mutex_lock(&ftrace_export_lock);
410 ret = rm_ftrace_export(&ftrace_exports_list, export);
412 mutex_unlock(&ftrace_export_lock);
416 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
418 /* trace_flags holds trace_options default values */
419 #define TRACE_DEFAULT_FLAGS \
420 (FUNCTION_DEFAULT_FLAGS | \
421 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
422 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
423 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
424 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | \
427 /* trace_options that are only supported by global_trace */
428 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
429 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
431 /* trace_flags that are default zero for instances */
432 #define ZEROED_TRACE_FLAGS \
433 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
436 * The global_trace is the descriptor that holds the top-level tracing
437 * buffers for the live tracing.
439 static struct trace_array global_trace = {
440 .trace_flags = TRACE_DEFAULT_FLAGS,
443 LIST_HEAD(ftrace_trace_arrays);
445 int trace_array_get(struct trace_array *this_tr)
447 struct trace_array *tr;
450 mutex_lock(&trace_types_lock);
451 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
458 mutex_unlock(&trace_types_lock);
463 static void __trace_array_put(struct trace_array *this_tr)
465 WARN_ON(!this_tr->ref);
470 * trace_array_put - Decrement the reference counter for this trace array.
471 * @this_tr : pointer to the trace array
473 * NOTE: Use this when we no longer need the trace array returned by
474 * trace_array_get_by_name(). This ensures the trace array can be later
478 void trace_array_put(struct trace_array *this_tr)
483 mutex_lock(&trace_types_lock);
484 __trace_array_put(this_tr);
485 mutex_unlock(&trace_types_lock);
487 EXPORT_SYMBOL_GPL(trace_array_put);
489 int tracing_check_open_get_tr(struct trace_array *tr)
493 ret = security_locked_down(LOCKDOWN_TRACEFS);
497 if (tracing_disabled)
500 if (tr && trace_array_get(tr) < 0)
506 int call_filter_check_discard(struct trace_event_call *call, void *rec,
507 struct trace_buffer *buffer,
508 struct ring_buffer_event *event)
510 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
511 !filter_match_preds(call->filter, rec)) {
512 __trace_event_discard_commit(buffer, event);
520 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
521 * @filtered_pids: The list of pids to check
522 * @search_pid: The PID to find in @filtered_pids
524 * Returns true if @search_pid is found in @filtered_pids, and false otherwise.
527 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
529 return trace_pid_list_is_set(filtered_pids, search_pid);
533 * trace_ignore_this_task - should a task be ignored for tracing
534 * @filtered_pids: The list of pids to check
535 * @filtered_no_pids: The list of pids not to be traced
536 * @task: The task that should be ignored if not filtered
538 * Checks if @task should be traced or not from @filtered_pids.
539 * Returns true if @task should *NOT* be traced.
540 * Returns false if @task should be traced.
543 trace_ignore_this_task(struct trace_pid_list *filtered_pids,
544 struct trace_pid_list *filtered_no_pids,
545 struct task_struct *task)
548 * If filtered_no_pids is not empty, and the task's pid is listed
549 * in filtered_no_pids, then return true.
550 * Otherwise, if filtered_pids is empty, that means we can
551 * trace all tasks. If it has content, then only trace pids
552 * within filtered_pids.
555 return (filtered_pids &&
556 !trace_find_filtered_pid(filtered_pids, task->pid)) ||
558 trace_find_filtered_pid(filtered_no_pids, task->pid));
562 * trace_filter_add_remove_task - Add or remove a task from a pid_list
563 * @pid_list: The list to modify
564 * @self: The current task for fork or NULL for exit
565 * @task: The task to add or remove
567 * If adding a task, if @self is defined, the task is only added if @self
568 * is also included in @pid_list. This happens on fork and tasks should
569 * only be added when the parent is listed. If @self is NULL, then the
570 * @task pid will be removed from the list, which would happen on exit
573 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
574 struct task_struct *self,
575 struct task_struct *task)
580 /* For forks, we only add if the forking task is listed */
582 if (!trace_find_filtered_pid(pid_list, self->pid))
586 /* "self" is set for forks, and NULL for exits */
588 trace_pid_list_set(pid_list, task->pid);
590 trace_pid_list_clear(pid_list, task->pid);
594 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
595 * @pid_list: The pid list to show
596 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
597 * @pos: The position of the file
599 * This is used by the seq_file "next" operation to iterate the pids
600 * listed in a trace_pid_list structure.
602 * Returns the pid+1 as we want to display pid of zero, but NULL would
603 * stop the iteration.
605 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
607 long pid = (unsigned long)v;
612 /* pid already is +1 of the actual previous bit */
613 if (trace_pid_list_next(pid_list, pid, &next) < 0)
618 /* Return pid + 1 to allow zero to be represented */
619 return (void *)(pid + 1);
623 * trace_pid_start - Used for seq_file to start reading pid lists
624 * @pid_list: The pid list to show
625 * @pos: The position of the file
627 * This is used by seq_file "start" operation to start the iteration
630 * Returns the pid+1 as we want to display pid of zero, but NULL would
631 * stop the iteration.
633 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
639 if (trace_pid_list_first(pid_list, &first) < 0)
644 /* Return pid + 1 so that zero can be the exit value */
645 for (pid++; pid && l < *pos;
646 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
652 * trace_pid_show - show the current pid in seq_file processing
653 * @m: The seq_file structure to write into
654 * @v: A void pointer of the pid (+1) value to display
656 * Can be directly used by seq_file operations to display the current
659 int trace_pid_show(struct seq_file *m, void *v)
661 unsigned long pid = (unsigned long)v - 1;
663 seq_printf(m, "%lu\n", pid);
667 /* 128 should be much more than enough */
668 #define PID_BUF_SIZE 127
670 int trace_pid_write(struct trace_pid_list *filtered_pids,
671 struct trace_pid_list **new_pid_list,
672 const char __user *ubuf, size_t cnt)
674 struct trace_pid_list *pid_list;
675 struct trace_parser parser;
683 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
687 * Always recreate a new array. The write is an all or nothing
688 * operation. Always create a new array when adding new pids by
689 * the user. If the operation fails, then the current list is
692 pid_list = trace_pid_list_alloc();
694 trace_parser_put(&parser);
699 /* copy the current bits to the new max */
700 ret = trace_pid_list_first(filtered_pids, &pid);
702 trace_pid_list_set(pid_list, pid);
703 ret = trace_pid_list_next(filtered_pids, pid + 1, &pid);
713 ret = trace_get_user(&parser, ubuf, cnt, &pos);
714 if (ret < 0 || !trace_parser_loaded(&parser))
722 if (kstrtoul(parser.buffer, 0, &val))
727 if (trace_pid_list_set(pid_list, pid) < 0) {
733 trace_parser_clear(&parser);
736 trace_parser_put(&parser);
739 trace_pid_list_free(pid_list);
744 /* Cleared the list of pids */
745 trace_pid_list_free(pid_list);
750 *new_pid_list = pid_list;
755 static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
759 /* Early boot up does not have a buffer yet */
761 return trace_clock_local();
763 ts = ring_buffer_time_stamp(buf->buffer);
764 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
769 u64 ftrace_now(int cpu)
771 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
775 * tracing_is_enabled - Show if global_trace has been enabled
777 * Shows if the global trace has been enabled or not. It uses the
778 * mirror flag "buffer_disabled" to be used in fast paths such as for
779 * the irqsoff tracer. But it may be inaccurate due to races. If you
780 * need to know the accurate state, use tracing_is_on() which is a little
781 * slower, but accurate.
783 int tracing_is_enabled(void)
786 * For quick access (irqsoff uses this in fast path), just
787 * return the mirror variable of the state of the ring buffer.
788 * It's a little racy, but we don't really care.
791 return !global_trace.buffer_disabled;
795 * trace_buf_size is the size in bytes that is allocated
796 * for a buffer. Note, the number of bytes is always rounded
799 * This number is purposely set to a low number of 16384.
800 * If the dump on oops happens, it will be much appreciated
801 * to not have to wait for all that output. Anyway this can be
802 * boot time and run time configurable.
804 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
806 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
808 /* trace_types holds a link list of available tracers. */
809 static struct tracer *trace_types __read_mostly;
812 * trace_types_lock is used to protect the trace_types list.
814 DEFINE_MUTEX(trace_types_lock);
817 * serialize the access of the ring buffer
819 * ring buffer serializes readers, but it is low level protection.
820 * The validity of the events (which returns by ring_buffer_peek() ..etc)
821 * are not protected by ring buffer.
823 * The content of events may become garbage if we allow other process consumes
824 * these events concurrently:
825 * A) the page of the consumed events may become a normal page
826 * (not reader page) in ring buffer, and this page will be rewritten
827 * by events producer.
828 * B) The page of the consumed events may become a page for splice_read,
829 * and this page will be returned to system.
831 * These primitives allow multi process access to different cpu ring buffer
834 * These primitives don't distinguish read-only and read-consume access.
835 * Multi read-only access are also serialized.
839 static DECLARE_RWSEM(all_cpu_access_lock);
840 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
842 static inline void trace_access_lock(int cpu)
844 if (cpu == RING_BUFFER_ALL_CPUS) {
845 /* gain it for accessing the whole ring buffer. */
846 down_write(&all_cpu_access_lock);
848 /* gain it for accessing a cpu ring buffer. */
850 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
851 down_read(&all_cpu_access_lock);
853 /* Secondly block other access to this @cpu ring buffer. */
854 mutex_lock(&per_cpu(cpu_access_lock, cpu));
858 static inline void trace_access_unlock(int cpu)
860 if (cpu == RING_BUFFER_ALL_CPUS) {
861 up_write(&all_cpu_access_lock);
863 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
864 up_read(&all_cpu_access_lock);
868 static inline void trace_access_lock_init(void)
872 for_each_possible_cpu(cpu)
873 mutex_init(&per_cpu(cpu_access_lock, cpu));
878 static DEFINE_MUTEX(access_lock);
880 static inline void trace_access_lock(int cpu)
883 mutex_lock(&access_lock);
886 static inline void trace_access_unlock(int cpu)
889 mutex_unlock(&access_lock);
892 static inline void trace_access_lock_init(void)
898 #ifdef CONFIG_STACKTRACE
899 static void __ftrace_trace_stack(struct trace_buffer *buffer,
900 unsigned int trace_ctx,
901 int skip, struct pt_regs *regs);
902 static inline void ftrace_trace_stack(struct trace_array *tr,
903 struct trace_buffer *buffer,
904 unsigned int trace_ctx,
905 int skip, struct pt_regs *regs);
908 static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
909 unsigned int trace_ctx,
910 int skip, struct pt_regs *regs)
913 static inline void ftrace_trace_stack(struct trace_array *tr,
914 struct trace_buffer *buffer,
915 unsigned long trace_ctx,
916 int skip, struct pt_regs *regs)
922 static __always_inline void
923 trace_event_setup(struct ring_buffer_event *event,
924 int type, unsigned int trace_ctx)
926 struct trace_entry *ent = ring_buffer_event_data(event);
928 tracing_generic_entry_update(ent, type, trace_ctx);
931 static __always_inline struct ring_buffer_event *
932 __trace_buffer_lock_reserve(struct trace_buffer *buffer,
935 unsigned int trace_ctx)
937 struct ring_buffer_event *event;
939 event = ring_buffer_lock_reserve(buffer, len);
941 trace_event_setup(event, type, trace_ctx);
946 void tracer_tracing_on(struct trace_array *tr)
948 if (tr->array_buffer.buffer)
949 ring_buffer_record_on(tr->array_buffer.buffer);
951 * This flag is looked at when buffers haven't been allocated
952 * yet, or by some tracers (like irqsoff), that just want to
953 * know if the ring buffer has been disabled, but it can handle
954 * races of where it gets disabled but we still do a record.
955 * As the check is in the fast path of the tracers, it is more
956 * important to be fast than accurate.
958 tr->buffer_disabled = 0;
959 /* Make the flag seen by readers */
964 * tracing_on - enable tracing buffers
966 * This function enables tracing buffers that may have been
967 * disabled with tracing_off.
969 void tracing_on(void)
971 tracer_tracing_on(&global_trace);
973 EXPORT_SYMBOL_GPL(tracing_on);
976 static __always_inline void
977 __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
979 __this_cpu_write(trace_taskinfo_save, true);
981 /* If this is the temp buffer, we need to commit fully */
982 if (this_cpu_read(trace_buffered_event) == event) {
983 /* Length is in event->array[0] */
984 ring_buffer_write(buffer, event->array[0], &event->array[1]);
985 /* Release the temp buffer */
986 this_cpu_dec(trace_buffered_event_cnt);
987 /* ring_buffer_unlock_commit() enables preemption */
988 preempt_enable_notrace();
990 ring_buffer_unlock_commit(buffer, event);
994 * __trace_puts - write a constant string into the trace buffer.
995 * @ip: The address of the caller
996 * @str: The constant string to write
997 * @size: The size of the string.
999 int __trace_puts(unsigned long ip, const char *str, int size)
1001 struct ring_buffer_event *event;
1002 struct trace_buffer *buffer;
1003 struct print_entry *entry;
1004 unsigned int trace_ctx;
1007 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1010 if (unlikely(tracing_selftest_running || tracing_disabled))
1013 alloc = sizeof(*entry) + size + 2; /* possible \n added */
1015 trace_ctx = tracing_gen_ctx();
1016 buffer = global_trace.array_buffer.buffer;
1017 ring_buffer_nest_start(buffer);
1018 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
1025 entry = ring_buffer_event_data(event);
1028 memcpy(&entry->buf, str, size);
1030 /* Add a newline if necessary */
1031 if (entry->buf[size - 1] != '\n') {
1032 entry->buf[size] = '\n';
1033 entry->buf[size + 1] = '\0';
1035 entry->buf[size] = '\0';
1037 __buffer_unlock_commit(buffer, event);
1038 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1040 ring_buffer_nest_end(buffer);
1043 EXPORT_SYMBOL_GPL(__trace_puts);
1046 * __trace_bputs - write the pointer to a constant string into trace buffer
1047 * @ip: The address of the caller
1048 * @str: The constant string to write to the buffer to
1050 int __trace_bputs(unsigned long ip, const char *str)
1052 struct ring_buffer_event *event;
1053 struct trace_buffer *buffer;
1054 struct bputs_entry *entry;
1055 unsigned int trace_ctx;
1056 int size = sizeof(struct bputs_entry);
1059 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1062 if (unlikely(tracing_selftest_running || tracing_disabled))
1065 trace_ctx = tracing_gen_ctx();
1066 buffer = global_trace.array_buffer.buffer;
1068 ring_buffer_nest_start(buffer);
1069 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
1074 entry = ring_buffer_event_data(event);
1078 __buffer_unlock_commit(buffer, event);
1079 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
1083 ring_buffer_nest_end(buffer);
1086 EXPORT_SYMBOL_GPL(__trace_bputs);
1088 #ifdef CONFIG_TRACER_SNAPSHOT
1089 static void tracing_snapshot_instance_cond(struct trace_array *tr,
1092 struct tracer *tracer = tr->current_trace;
1093 unsigned long flags;
1096 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
1097 internal_trace_puts("*** snapshot is being ignored ***\n");
1101 if (!tr->allocated_snapshot) {
1102 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
1103 internal_trace_puts("*** stopping trace here! ***\n");
1108 /* Note, snapshot can not be used when the tracer uses it */
1109 if (tracer->use_max_tr) {
1110 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
1111 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
1115 local_irq_save(flags);
1116 update_max_tr(tr, current, smp_processor_id(), cond_data);
1117 local_irq_restore(flags);
1120 void tracing_snapshot_instance(struct trace_array *tr)
1122 tracing_snapshot_instance_cond(tr, NULL);
1126 * tracing_snapshot - take a snapshot of the current buffer.
1128 * This causes a swap between the snapshot buffer and the current live
1129 * tracing buffer. You can use this to take snapshots of the live
1130 * trace when some condition is triggered, but continue to trace.
1132 * Note, make sure to allocate the snapshot with either
1133 * a tracing_snapshot_alloc(), or by doing it manually
1134 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
1136 * If the snapshot buffer is not allocated, it will stop tracing.
1137 * Basically making a permanent snapshot.
1139 void tracing_snapshot(void)
1141 struct trace_array *tr = &global_trace;
1143 tracing_snapshot_instance(tr);
1145 EXPORT_SYMBOL_GPL(tracing_snapshot);
1148 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1149 * @tr: The tracing instance to snapshot
1150 * @cond_data: The data to be tested conditionally, and possibly saved
1152 * This is the same as tracing_snapshot() except that the snapshot is
1153 * conditional - the snapshot will only happen if the
1154 * cond_snapshot.update() implementation receiving the cond_data
1155 * returns true, which means that the trace array's cond_snapshot
1156 * update() operation used the cond_data to determine whether the
1157 * snapshot should be taken, and if it was, presumably saved it along
1158 * with the snapshot.
1160 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1162 tracing_snapshot_instance_cond(tr, cond_data);
1164 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1167 * tracing_snapshot_cond_data - get the user data associated with a snapshot
1168 * @tr: The tracing instance
1170 * When the user enables a conditional snapshot using
1171 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1172 * with the snapshot. This accessor is used to retrieve it.
1174 * Should not be called from cond_snapshot.update(), since it takes
1175 * the tr->max_lock lock, which the code calling
1176 * cond_snapshot.update() has already done.
1178 * Returns the cond_data associated with the trace array's snapshot.
1180 void *tracing_cond_snapshot_data(struct trace_array *tr)
1182 void *cond_data = NULL;
1184 arch_spin_lock(&tr->max_lock);
1186 if (tr->cond_snapshot)
1187 cond_data = tr->cond_snapshot->cond_data;
1189 arch_spin_unlock(&tr->max_lock);
1193 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1195 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1196 struct array_buffer *size_buf, int cpu_id);
1197 static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1199 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1203 if (!tr->allocated_snapshot) {
1205 /* allocate spare buffer */
1206 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1207 &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1211 tr->allocated_snapshot = true;
1217 static void free_snapshot(struct trace_array *tr)
1220 * We don't free the ring buffer. instead, resize it because
1221 * The max_tr ring buffer has some state (e.g. ring->clock) and
1222 * we want preserve it.
1224 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1225 set_buffer_entries(&tr->max_buffer, 1);
1226 tracing_reset_online_cpus(&tr->max_buffer);
1227 tr->allocated_snapshot = false;
1231 * tracing_alloc_snapshot - allocate snapshot buffer.
1233 * This only allocates the snapshot buffer if it isn't already
1234 * allocated - it doesn't also take a snapshot.
1236 * This is meant to be used in cases where the snapshot buffer needs
1237 * to be set up for events that can't sleep but need to be able to
1238 * trigger a snapshot.
1240 int tracing_alloc_snapshot(void)
1242 struct trace_array *tr = &global_trace;
1245 ret = tracing_alloc_snapshot_instance(tr);
1250 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1253 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1255 * This is similar to tracing_snapshot(), but it will allocate the
1256 * snapshot buffer if it isn't already allocated. Use this only
1257 * where it is safe to sleep, as the allocation may sleep.
1259 * This causes a swap between the snapshot buffer and the current live
1260 * tracing buffer. You can use this to take snapshots of the live
1261 * trace when some condition is triggered, but continue to trace.
1263 void tracing_snapshot_alloc(void)
1267 ret = tracing_alloc_snapshot();
1273 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1276 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1277 * @tr: The tracing instance
1278 * @cond_data: User data to associate with the snapshot
1279 * @update: Implementation of the cond_snapshot update function
1281 * Check whether the conditional snapshot for the given instance has
1282 * already been enabled, or if the current tracer is already using a
1283 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1284 * save the cond_data and update function inside.
1286 * Returns 0 if successful, error otherwise.
1288 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1289 cond_update_fn_t update)
1291 struct cond_snapshot *cond_snapshot;
1294 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1298 cond_snapshot->cond_data = cond_data;
1299 cond_snapshot->update = update;
1301 mutex_lock(&trace_types_lock);
1303 ret = tracing_alloc_snapshot_instance(tr);
1307 if (tr->current_trace->use_max_tr) {
1313 * The cond_snapshot can only change to NULL without the
1314 * trace_types_lock. We don't care if we race with it going
1315 * to NULL, but we want to make sure that it's not set to
1316 * something other than NULL when we get here, which we can
1317 * do safely with only holding the trace_types_lock and not
1318 * having to take the max_lock.
1320 if (tr->cond_snapshot) {
1325 arch_spin_lock(&tr->max_lock);
1326 tr->cond_snapshot = cond_snapshot;
1327 arch_spin_unlock(&tr->max_lock);
1329 mutex_unlock(&trace_types_lock);
1334 mutex_unlock(&trace_types_lock);
1335 kfree(cond_snapshot);
1338 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1341 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1342 * @tr: The tracing instance
1344 * Check whether the conditional snapshot for the given instance is
1345 * enabled; if so, free the cond_snapshot associated with it,
1346 * otherwise return -EINVAL.
1348 * Returns 0 if successful, error otherwise.
1350 int tracing_snapshot_cond_disable(struct trace_array *tr)
1354 arch_spin_lock(&tr->max_lock);
1356 if (!tr->cond_snapshot)
1359 kfree(tr->cond_snapshot);
1360 tr->cond_snapshot = NULL;
1363 arch_spin_unlock(&tr->max_lock);
1367 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1369 void tracing_snapshot(void)
1371 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1373 EXPORT_SYMBOL_GPL(tracing_snapshot);
1374 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1376 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1378 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1379 int tracing_alloc_snapshot(void)
1381 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1384 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1385 void tracing_snapshot_alloc(void)
1390 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1391 void *tracing_cond_snapshot_data(struct trace_array *tr)
1395 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1396 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1400 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1401 int tracing_snapshot_cond_disable(struct trace_array *tr)
1405 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1406 #endif /* CONFIG_TRACER_SNAPSHOT */
1408 void tracer_tracing_off(struct trace_array *tr)
1410 if (tr->array_buffer.buffer)
1411 ring_buffer_record_off(tr->array_buffer.buffer);
1413 * This flag is looked at when buffers haven't been allocated
1414 * yet, or by some tracers (like irqsoff), that just want to
1415 * know if the ring buffer has been disabled, but it can handle
1416 * races of where it gets disabled but we still do a record.
1417 * As the check is in the fast path of the tracers, it is more
1418 * important to be fast than accurate.
1420 tr->buffer_disabled = 1;
1421 /* Make the flag seen by readers */
1426 * tracing_off - turn off tracing buffers
1428 * This function stops the tracing buffers from recording data.
1429 * It does not disable any overhead the tracers themselves may
1430 * be causing. This function simply causes all recording to
1431 * the ring buffers to fail.
1433 void tracing_off(void)
1435 tracer_tracing_off(&global_trace);
1437 EXPORT_SYMBOL_GPL(tracing_off);
1439 void disable_trace_on_warning(void)
1441 if (__disable_trace_on_warning) {
1442 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
1443 "Disabling tracing due to warning\n");
1449 * tracer_tracing_is_on - show real state of ring buffer enabled
1450 * @tr : the trace array to know if ring buffer is enabled
1452 * Shows real state of the ring buffer if it is enabled or not.
1454 bool tracer_tracing_is_on(struct trace_array *tr)
1456 if (tr->array_buffer.buffer)
1457 return ring_buffer_record_is_on(tr->array_buffer.buffer);
1458 return !tr->buffer_disabled;
1462 * tracing_is_on - show state of ring buffers enabled
1464 int tracing_is_on(void)
1466 return tracer_tracing_is_on(&global_trace);
1468 EXPORT_SYMBOL_GPL(tracing_is_on);
1470 static int __init set_buf_size(char *str)
1472 unsigned long buf_size;
1476 buf_size = memparse(str, &str);
1477 /* nr_entries can not be zero */
1480 trace_buf_size = buf_size;
1483 __setup("trace_buf_size=", set_buf_size);
1485 static int __init set_tracing_thresh(char *str)
1487 unsigned long threshold;
1492 ret = kstrtoul(str, 0, &threshold);
1495 tracing_thresh = threshold * 1000;
1498 __setup("tracing_thresh=", set_tracing_thresh);
1500 unsigned long nsecs_to_usecs(unsigned long nsecs)
1502 return nsecs / 1000;
1506 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1507 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1508 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1509 * of strings in the order that the evals (enum) were defined.
1514 /* These must match the bit positions in trace_iterator_flags */
1515 static const char *trace_options[] = {
1523 int in_ns; /* is this clock in nanoseconds? */
1524 } trace_clocks[] = {
1525 { trace_clock_local, "local", 1 },
1526 { trace_clock_global, "global", 1 },
1527 { trace_clock_counter, "counter", 0 },
1528 { trace_clock_jiffies, "uptime", 0 },
1529 { trace_clock, "perf", 1 },
1530 { ktime_get_mono_fast_ns, "mono", 1 },
1531 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1532 { ktime_get_boot_fast_ns, "boot", 1 },
1536 bool trace_clock_in_ns(struct trace_array *tr)
1538 if (trace_clocks[tr->clock_id].in_ns)
1545 * trace_parser_get_init - gets the buffer for trace parser
1547 int trace_parser_get_init(struct trace_parser *parser, int size)
1549 memset(parser, 0, sizeof(*parser));
1551 parser->buffer = kmalloc(size, GFP_KERNEL);
1552 if (!parser->buffer)
1555 parser->size = size;
1560 * trace_parser_put - frees the buffer for trace parser
1562 void trace_parser_put(struct trace_parser *parser)
1564 kfree(parser->buffer);
1565 parser->buffer = NULL;
1569 * trace_get_user - reads the user input string separated by space
1570 * (matched by isspace(ch))
1572 * For each string found the 'struct trace_parser' is updated,
1573 * and the function returns.
1575 * Returns number of bytes read.
1577 * See kernel/trace/trace.h for 'struct trace_parser' details.
1579 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1580 size_t cnt, loff_t *ppos)
1587 trace_parser_clear(parser);
1589 ret = get_user(ch, ubuf++);
1597 * The parser is not finished with the last write,
1598 * continue reading the user input without skipping spaces.
1600 if (!parser->cont) {
1601 /* skip white space */
1602 while (cnt && isspace(ch)) {
1603 ret = get_user(ch, ubuf++);
1612 /* only spaces were written */
1613 if (isspace(ch) || !ch) {
1620 /* read the non-space input */
1621 while (cnt && !isspace(ch) && ch) {
1622 if (parser->idx < parser->size - 1)
1623 parser->buffer[parser->idx++] = ch;
1628 ret = get_user(ch, ubuf++);
1635 /* We either got finished input or we have to wait for another call. */
1636 if (isspace(ch) || !ch) {
1637 parser->buffer[parser->idx] = 0;
1638 parser->cont = false;
1639 } else if (parser->idx < parser->size - 1) {
1640 parser->cont = true;
1641 parser->buffer[parser->idx++] = ch;
1642 /* Make sure the parsed string always terminates with '\0'. */
1643 parser->buffer[parser->idx] = 0;
1656 /* TODO add a seq_buf_to_buffer() */
1657 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1661 if (trace_seq_used(s) <= s->seq.readpos)
1664 len = trace_seq_used(s) - s->seq.readpos;
1667 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1669 s->seq.readpos += cnt;
1673 unsigned long __read_mostly tracing_thresh;
1674 static const struct file_operations tracing_max_lat_fops;
1676 #ifdef LATENCY_FS_NOTIFY
1678 static struct workqueue_struct *fsnotify_wq;
1680 static void latency_fsnotify_workfn(struct work_struct *work)
1682 struct trace_array *tr = container_of(work, struct trace_array,
1684 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
1687 static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1689 struct trace_array *tr = container_of(iwork, struct trace_array,
1691 queue_work(fsnotify_wq, &tr->fsnotify_work);
1694 static void trace_create_maxlat_file(struct trace_array *tr,
1695 struct dentry *d_tracer)
1697 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1698 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1699 tr->d_max_latency = trace_create_file("tracing_max_latency",
1701 d_tracer, &tr->max_latency,
1702 &tracing_max_lat_fops);
1705 __init static int latency_fsnotify_init(void)
1707 fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1708 WQ_UNBOUND | WQ_HIGHPRI, 0);
1710 pr_err("Unable to allocate tr_max_lat_wq\n");
1716 late_initcall_sync(latency_fsnotify_init);
1718 void latency_fsnotify(struct trace_array *tr)
1723 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1724 * possible that we are called from __schedule() or do_idle(), which
1725 * could cause a deadlock.
1727 irq_work_queue(&tr->fsnotify_irqwork);
1730 #elif defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) \
1731 || defined(CONFIG_OSNOISE_TRACER)
1733 #define trace_create_maxlat_file(tr, d_tracer) \
1734 trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \
1735 d_tracer, &tr->max_latency, &tracing_max_lat_fops)
1738 #define trace_create_maxlat_file(tr, d_tracer) do { } while (0)
1741 #ifdef CONFIG_TRACER_MAX_TRACE
1743 * Copy the new maximum trace into the separate maximum-trace
1744 * structure. (this way the maximum trace is permanently saved,
1745 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1748 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1750 struct array_buffer *trace_buf = &tr->array_buffer;
1751 struct array_buffer *max_buf = &tr->max_buffer;
1752 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1753 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1756 max_buf->time_start = data->preempt_timestamp;
1758 max_data->saved_latency = tr->max_latency;
1759 max_data->critical_start = data->critical_start;
1760 max_data->critical_end = data->critical_end;
1762 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1763 max_data->pid = tsk->pid;
1765 * If tsk == current, then use current_uid(), as that does not use
1766 * RCU. The irq tracer can be called out of RCU scope.
1769 max_data->uid = current_uid();
1771 max_data->uid = task_uid(tsk);
1773 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1774 max_data->policy = tsk->policy;
1775 max_data->rt_priority = tsk->rt_priority;
1777 /* record this tasks comm */
1778 tracing_record_cmdline(tsk);
1779 latency_fsnotify(tr);
1783 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1785 * @tsk: the task with the latency
1786 * @cpu: The cpu that initiated the trace.
1787 * @cond_data: User data associated with a conditional snapshot
1789 * Flip the buffers between the @tr and the max_tr and record information
1790 * about which task was the cause of this latency.
1793 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1799 WARN_ON_ONCE(!irqs_disabled());
1801 if (!tr->allocated_snapshot) {
1802 /* Only the nop tracer should hit this when disabling */
1803 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1807 arch_spin_lock(&tr->max_lock);
1809 /* Inherit the recordable setting from array_buffer */
1810 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1811 ring_buffer_record_on(tr->max_buffer.buffer);
1813 ring_buffer_record_off(tr->max_buffer.buffer);
1815 #ifdef CONFIG_TRACER_SNAPSHOT
1816 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1819 swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1821 __update_max_tr(tr, tsk, cpu);
1824 arch_spin_unlock(&tr->max_lock);
1828 * update_max_tr_single - only copy one trace over, and reset the rest
1830 * @tsk: task with the latency
1831 * @cpu: the cpu of the buffer to copy.
1833 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1836 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1843 WARN_ON_ONCE(!irqs_disabled());
1844 if (!tr->allocated_snapshot) {
1845 /* Only the nop tracer should hit this when disabling */
1846 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1850 arch_spin_lock(&tr->max_lock);
1852 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1854 if (ret == -EBUSY) {
1856 * We failed to swap the buffer due to a commit taking
1857 * place on this CPU. We fail to record, but we reset
1858 * the max trace buffer (no one writes directly to it)
1859 * and flag that it failed.
1861 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1862 "Failed to swap buffers due to commit in progress\n");
1865 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1867 __update_max_tr(tr, tsk, cpu);
1868 arch_spin_unlock(&tr->max_lock);
1870 #endif /* CONFIG_TRACER_MAX_TRACE */
1872 static int wait_on_pipe(struct trace_iterator *iter, int full)
1874 /* Iterators are static, they should be filled or empty */
1875 if (trace_buffer_iter(iter, iter->cpu_file))
1878 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1882 #ifdef CONFIG_FTRACE_STARTUP_TEST
1883 static bool selftests_can_run;
1885 struct trace_selftests {
1886 struct list_head list;
1887 struct tracer *type;
1890 static LIST_HEAD(postponed_selftests);
1892 static int save_selftest(struct tracer *type)
1894 struct trace_selftests *selftest;
1896 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1900 selftest->type = type;
1901 list_add(&selftest->list, &postponed_selftests);
1905 static int run_tracer_selftest(struct tracer *type)
1907 struct trace_array *tr = &global_trace;
1908 struct tracer *saved_tracer = tr->current_trace;
1911 if (!type->selftest || tracing_selftest_disabled)
1915 * If a tracer registers early in boot up (before scheduling is
1916 * initialized and such), then do not run its selftests yet.
1917 * Instead, run it a little later in the boot process.
1919 if (!selftests_can_run)
1920 return save_selftest(type);
1922 if (!tracing_is_on()) {
1923 pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
1929 * Run a selftest on this tracer.
1930 * Here we reset the trace buffer, and set the current
1931 * tracer to be this tracer. The tracer can then run some
1932 * internal tracing to verify that everything is in order.
1933 * If we fail, we do not register this tracer.
1935 tracing_reset_online_cpus(&tr->array_buffer);
1937 tr->current_trace = type;
1939 #ifdef CONFIG_TRACER_MAX_TRACE
1940 if (type->use_max_tr) {
1941 /* If we expanded the buffers, make sure the max is expanded too */
1942 if (ring_buffer_expanded)
1943 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1944 RING_BUFFER_ALL_CPUS);
1945 tr->allocated_snapshot = true;
1949 /* the test is responsible for initializing and enabling */
1950 pr_info("Testing tracer %s: ", type->name);
1951 ret = type->selftest(type, tr);
1952 /* the test is responsible for resetting too */
1953 tr->current_trace = saved_tracer;
1955 printk(KERN_CONT "FAILED!\n");
1956 /* Add the warning after printing 'FAILED' */
1960 /* Only reset on passing, to avoid touching corrupted buffers */
1961 tracing_reset_online_cpus(&tr->array_buffer);
1963 #ifdef CONFIG_TRACER_MAX_TRACE
1964 if (type->use_max_tr) {
1965 tr->allocated_snapshot = false;
1967 /* Shrink the max buffer again */
1968 if (ring_buffer_expanded)
1969 ring_buffer_resize(tr->max_buffer.buffer, 1,
1970 RING_BUFFER_ALL_CPUS);
1974 printk(KERN_CONT "PASSED\n");
1978 static __init int init_trace_selftests(void)
1980 struct trace_selftests *p, *n;
1981 struct tracer *t, **last;
1984 selftests_can_run = true;
1986 mutex_lock(&trace_types_lock);
1988 if (list_empty(&postponed_selftests))
1991 pr_info("Running postponed tracer tests:\n");
1993 tracing_selftest_running = true;
1994 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1995 /* This loop can take minutes when sanitizers are enabled, so
1996 * lets make sure we allow RCU processing.
1999 ret = run_tracer_selftest(p->type);
2000 /* If the test fails, then warn and remove from available_tracers */
2002 WARN(1, "tracer: %s failed selftest, disabling\n",
2004 last = &trace_types;
2005 for (t = trace_types; t; t = t->next) {
2016 tracing_selftest_running = false;
2019 mutex_unlock(&trace_types_lock);
2023 core_initcall(init_trace_selftests);
2025 static inline int run_tracer_selftest(struct tracer *type)
2029 #endif /* CONFIG_FTRACE_STARTUP_TEST */
2031 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
2033 static void __init apply_trace_boot_options(void);
2036 * register_tracer - register a tracer with the ftrace system.
2037 * @type: the plugin for the tracer
2039 * Register a new plugin tracer.
2041 int __init register_tracer(struct tracer *type)
2047 pr_info("Tracer must have a name\n");
2051 if (strlen(type->name) >= MAX_TRACER_SIZE) {
2052 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
2056 if (security_locked_down(LOCKDOWN_TRACEFS)) {
2057 pr_warn("Can not register tracer %s due to lockdown\n",
2062 mutex_lock(&trace_types_lock);
2064 tracing_selftest_running = true;
2066 for (t = trace_types; t; t = t->next) {
2067 if (strcmp(type->name, t->name) == 0) {
2069 pr_info("Tracer %s already registered\n",
2076 if (!type->set_flag)
2077 type->set_flag = &dummy_set_flag;
2079 /*allocate a dummy tracer_flags*/
2080 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
2085 type->flags->val = 0;
2086 type->flags->opts = dummy_tracer_opt;
2088 if (!type->flags->opts)
2089 type->flags->opts = dummy_tracer_opt;
2091 /* store the tracer for __set_tracer_option */
2092 type->flags->trace = type;
2094 ret = run_tracer_selftest(type);
2098 type->next = trace_types;
2100 add_tracer_options(&global_trace, type);
2103 tracing_selftest_running = false;
2104 mutex_unlock(&trace_types_lock);
2106 if (ret || !default_bootup_tracer)
2109 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
2112 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
2113 /* Do we want this tracer to start on bootup? */
2114 tracing_set_tracer(&global_trace, type->name);
2115 default_bootup_tracer = NULL;
2117 apply_trace_boot_options();
2119 /* disable other selftests, since this will break it. */
2120 disable_tracing_selftest("running a tracer");
2126 static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
2128 struct trace_buffer *buffer = buf->buffer;
2133 ring_buffer_record_disable(buffer);
2135 /* Make sure all commits have finished */
2137 ring_buffer_reset_cpu(buffer, cpu);
2139 ring_buffer_record_enable(buffer);
2142 void tracing_reset_online_cpus(struct array_buffer *buf)
2144 struct trace_buffer *buffer = buf->buffer;
2149 ring_buffer_record_disable(buffer);
2151 /* Make sure all commits have finished */
2154 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2156 ring_buffer_reset_online_cpus(buffer);
2158 ring_buffer_record_enable(buffer);
2161 /* Must have trace_types_lock held */
2162 void tracing_reset_all_online_cpus(void)
2164 struct trace_array *tr;
2166 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2167 if (!tr->clear_trace)
2169 tr->clear_trace = false;
2170 tracing_reset_online_cpus(&tr->array_buffer);
2171 #ifdef CONFIG_TRACER_MAX_TRACE
2172 tracing_reset_online_cpus(&tr->max_buffer);
2178 * The tgid_map array maps from pid to tgid; i.e. the value stored at index i
2179 * is the tgid last observed corresponding to pid=i.
2181 static int *tgid_map;
2183 /* The maximum valid index into tgid_map. */
2184 static size_t tgid_map_max;
2186 #define SAVED_CMDLINES_DEFAULT 128
2187 #define NO_CMDLINE_MAP UINT_MAX
2188 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2189 struct saved_cmdlines_buffer {
2190 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2191 unsigned *map_cmdline_to_pid;
2192 unsigned cmdline_num;
2194 char *saved_cmdlines;
2196 static struct saved_cmdlines_buffer *savedcmd;
2198 static inline char *get_saved_cmdlines(int idx)
2200 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2203 static inline void set_cmdline(int idx, const char *cmdline)
2205 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2208 static int allocate_cmdlines_buffer(unsigned int val,
2209 struct saved_cmdlines_buffer *s)
2211 s->map_cmdline_to_pid = kmalloc_array(val,
2212 sizeof(*s->map_cmdline_to_pid),
2214 if (!s->map_cmdline_to_pid)
2217 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2218 if (!s->saved_cmdlines) {
2219 kfree(s->map_cmdline_to_pid);
2224 s->cmdline_num = val;
2225 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2226 sizeof(s->map_pid_to_cmdline));
2227 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2228 val * sizeof(*s->map_cmdline_to_pid));
2233 static int trace_create_savedcmd(void)
2237 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2241 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2251 int is_tracing_stopped(void)
2253 return global_trace.stop_count;
2257 * tracing_start - quick start of the tracer
2259 * If tracing is enabled but was stopped by tracing_stop,
2260 * this will start the tracer back up.
2262 void tracing_start(void)
2264 struct trace_buffer *buffer;
2265 unsigned long flags;
2267 if (tracing_disabled)
2270 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2271 if (--global_trace.stop_count) {
2272 if (global_trace.stop_count < 0) {
2273 /* Someone screwed up their debugging */
2275 global_trace.stop_count = 0;
2280 /* Prevent the buffers from switching */
2281 arch_spin_lock(&global_trace.max_lock);
2283 buffer = global_trace.array_buffer.buffer;
2285 ring_buffer_record_enable(buffer);
2287 #ifdef CONFIG_TRACER_MAX_TRACE
2288 buffer = global_trace.max_buffer.buffer;
2290 ring_buffer_record_enable(buffer);
2293 arch_spin_unlock(&global_trace.max_lock);
2296 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2299 static void tracing_start_tr(struct trace_array *tr)
2301 struct trace_buffer *buffer;
2302 unsigned long flags;
2304 if (tracing_disabled)
2307 /* If global, we need to also start the max tracer */
2308 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2309 return tracing_start();
2311 raw_spin_lock_irqsave(&tr->start_lock, flags);
2313 if (--tr->stop_count) {
2314 if (tr->stop_count < 0) {
2315 /* Someone screwed up their debugging */
2322 buffer = tr->array_buffer.buffer;
2324 ring_buffer_record_enable(buffer);
2327 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2331 * tracing_stop - quick stop of the tracer
2333 * Light weight way to stop tracing. Use in conjunction with
2336 void tracing_stop(void)
2338 struct trace_buffer *buffer;
2339 unsigned long flags;
2341 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2342 if (global_trace.stop_count++)
2345 /* Prevent the buffers from switching */
2346 arch_spin_lock(&global_trace.max_lock);
2348 buffer = global_trace.array_buffer.buffer;
2350 ring_buffer_record_disable(buffer);
2352 #ifdef CONFIG_TRACER_MAX_TRACE
2353 buffer = global_trace.max_buffer.buffer;
2355 ring_buffer_record_disable(buffer);
2358 arch_spin_unlock(&global_trace.max_lock);
2361 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2364 static void tracing_stop_tr(struct trace_array *tr)
2366 struct trace_buffer *buffer;
2367 unsigned long flags;
2369 /* If global, we need to also stop the max tracer */
2370 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2371 return tracing_stop();
2373 raw_spin_lock_irqsave(&tr->start_lock, flags);
2374 if (tr->stop_count++)
2377 buffer = tr->array_buffer.buffer;
2379 ring_buffer_record_disable(buffer);
2382 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2385 static int trace_save_cmdline(struct task_struct *tsk)
2389 /* treat recording of idle task as a success */
2393 tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
2396 * It's not the end of the world if we don't get
2397 * the lock, but we also don't want to spin
2398 * nor do we want to disable interrupts,
2399 * so if we miss here, then better luck next time.
2401 if (!arch_spin_trylock(&trace_cmdline_lock))
2404 idx = savedcmd->map_pid_to_cmdline[tpid];
2405 if (idx == NO_CMDLINE_MAP) {
2406 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2408 savedcmd->map_pid_to_cmdline[tpid] = idx;
2409 savedcmd->cmdline_idx = idx;
2412 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2413 set_cmdline(idx, tsk->comm);
2415 arch_spin_unlock(&trace_cmdline_lock);
2420 static void __trace_find_cmdline(int pid, char comm[])
2426 strcpy(comm, "<idle>");
2430 if (WARN_ON_ONCE(pid < 0)) {
2431 strcpy(comm, "<XXX>");
2435 tpid = pid & (PID_MAX_DEFAULT - 1);
2436 map = savedcmd->map_pid_to_cmdline[tpid];
2437 if (map != NO_CMDLINE_MAP) {
2438 tpid = savedcmd->map_cmdline_to_pid[map];
2440 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2444 strcpy(comm, "<...>");
2447 void trace_find_cmdline(int pid, char comm[])
2450 arch_spin_lock(&trace_cmdline_lock);
2452 __trace_find_cmdline(pid, comm);
2454 arch_spin_unlock(&trace_cmdline_lock);
2458 static int *trace_find_tgid_ptr(int pid)
2461 * Pairs with the smp_store_release in set_tracer_flag() to ensure that
2462 * if we observe a non-NULL tgid_map then we also observe the correct
2465 int *map = smp_load_acquire(&tgid_map);
2467 if (unlikely(!map || pid > tgid_map_max))
2473 int trace_find_tgid(int pid)
2475 int *ptr = trace_find_tgid_ptr(pid);
2477 return ptr ? *ptr : 0;
2480 static int trace_save_tgid(struct task_struct *tsk)
2484 /* treat recording of idle task as a success */
2488 ptr = trace_find_tgid_ptr(tsk->pid);
2496 static bool tracing_record_taskinfo_skip(int flags)
2498 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2500 if (!__this_cpu_read(trace_taskinfo_save))
2506 * tracing_record_taskinfo - record the task info of a task
2508 * @task: task to record
2509 * @flags: TRACE_RECORD_CMDLINE for recording comm
2510 * TRACE_RECORD_TGID for recording tgid
2512 void tracing_record_taskinfo(struct task_struct *task, int flags)
2516 if (tracing_record_taskinfo_skip(flags))
2520 * Record as much task information as possible. If some fail, continue
2521 * to try to record the others.
2523 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2524 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2526 /* If recording any information failed, retry again soon. */
2530 __this_cpu_write(trace_taskinfo_save, false);
2534 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2536 * @prev: previous task during sched_switch
2537 * @next: next task during sched_switch
2538 * @flags: TRACE_RECORD_CMDLINE for recording comm
2539 * TRACE_RECORD_TGID for recording tgid
2541 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2542 struct task_struct *next, int flags)
2546 if (tracing_record_taskinfo_skip(flags))
2550 * Record as much task information as possible. If some fail, continue
2551 * to try to record the others.
2553 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2554 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2555 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2556 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2558 /* If recording any information failed, retry again soon. */
2562 __this_cpu_write(trace_taskinfo_save, false);
2565 /* Helpers to record a specific task information */
2566 void tracing_record_cmdline(struct task_struct *task)
2568 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2571 void tracing_record_tgid(struct task_struct *task)
2573 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2577 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2578 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2579 * simplifies those functions and keeps them in sync.
2581 enum print_line_t trace_handle_return(struct trace_seq *s)
2583 return trace_seq_has_overflowed(s) ?
2584 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2586 EXPORT_SYMBOL_GPL(trace_handle_return);
2588 static unsigned short migration_disable_value(void)
2590 #if defined(CONFIG_SMP)
2591 return current->migration_disabled;
2597 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
2599 unsigned int trace_flags = irqs_status;
2602 pc = preempt_count();
2605 trace_flags |= TRACE_FLAG_NMI;
2606 if (pc & HARDIRQ_MASK)
2607 trace_flags |= TRACE_FLAG_HARDIRQ;
2608 if (in_serving_softirq())
2609 trace_flags |= TRACE_FLAG_SOFTIRQ;
2610 if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
2611 trace_flags |= TRACE_FLAG_BH_OFF;
2613 if (tif_need_resched())
2614 trace_flags |= TRACE_FLAG_NEED_RESCHED;
2615 if (test_preempt_need_resched())
2616 trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
2617 return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) |
2618 (min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
2621 struct ring_buffer_event *
2622 trace_buffer_lock_reserve(struct trace_buffer *buffer,
2625 unsigned int trace_ctx)
2627 return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2630 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2631 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2632 static int trace_buffered_event_ref;
2635 * trace_buffered_event_enable - enable buffering events
2637 * When events are being filtered, it is quicker to use a temporary
2638 * buffer to write the event data into if there's a likely chance
2639 * that it will not be committed. The discard of the ring buffer
2640 * is not as fast as committing, and is much slower than copying
2643 * When an event is to be filtered, allocate per cpu buffers to
2644 * write the event data into, and if the event is filtered and discarded
2645 * it is simply dropped, otherwise, the entire data is to be committed
2648 void trace_buffered_event_enable(void)
2650 struct ring_buffer_event *event;
2654 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2656 if (trace_buffered_event_ref++)
2659 for_each_tracing_cpu(cpu) {
2660 page = alloc_pages_node(cpu_to_node(cpu),
2661 GFP_KERNEL | __GFP_NORETRY, 0);
2665 event = page_address(page);
2666 memset(event, 0, sizeof(*event));
2668 per_cpu(trace_buffered_event, cpu) = event;
2671 if (cpu == smp_processor_id() &&
2672 __this_cpu_read(trace_buffered_event) !=
2673 per_cpu(trace_buffered_event, cpu))
2680 trace_buffered_event_disable();
2683 static void enable_trace_buffered_event(void *data)
2685 /* Probably not needed, but do it anyway */
2687 this_cpu_dec(trace_buffered_event_cnt);
2690 static void disable_trace_buffered_event(void *data)
2692 this_cpu_inc(trace_buffered_event_cnt);
2696 * trace_buffered_event_disable - disable buffering events
2698 * When a filter is removed, it is faster to not use the buffered
2699 * events, and to commit directly into the ring buffer. Free up
2700 * the temp buffers when there are no more users. This requires
2701 * special synchronization with current events.
2703 void trace_buffered_event_disable(void)
2707 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2709 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2712 if (--trace_buffered_event_ref)
2716 /* For each CPU, set the buffer as used. */
2717 smp_call_function_many(tracing_buffer_mask,
2718 disable_trace_buffered_event, NULL, 1);
2721 /* Wait for all current users to finish */
2724 for_each_tracing_cpu(cpu) {
2725 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2726 per_cpu(trace_buffered_event, cpu) = NULL;
2729 * Make sure trace_buffered_event is NULL before clearing
2730 * trace_buffered_event_cnt.
2735 /* Do the work on each cpu */
2736 smp_call_function_many(tracing_buffer_mask,
2737 enable_trace_buffered_event, NULL, 1);
2741 static struct trace_buffer *temp_buffer;
2743 struct ring_buffer_event *
2744 trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2745 struct trace_event_file *trace_file,
2746 int type, unsigned long len,
2747 unsigned int trace_ctx)
2749 struct ring_buffer_event *entry;
2750 struct trace_array *tr = trace_file->tr;
2753 *current_rb = tr->array_buffer.buffer;
2755 if (!tr->no_filter_buffering_ref &&
2756 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) {
2757 preempt_disable_notrace();
2759 * Filtering is on, so try to use the per cpu buffer first.
2760 * This buffer will simulate a ring_buffer_event,
2761 * where the type_len is zero and the array[0] will
2762 * hold the full length.
2763 * (see include/linux/ring-buffer.h for details on
2764 * how the ring_buffer_event is structured).
2766 * Using a temp buffer during filtering and copying it
2767 * on a matched filter is quicker than writing directly
2768 * into the ring buffer and then discarding it when
2769 * it doesn't match. That is because the discard
2770 * requires several atomic operations to get right.
2771 * Copying on match and doing nothing on a failed match
2772 * is still quicker than no copy on match, but having
2773 * to discard out of the ring buffer on a failed match.
2775 if ((entry = __this_cpu_read(trace_buffered_event))) {
2776 int max_len = PAGE_SIZE - struct_size(entry, array, 1);
2778 val = this_cpu_inc_return(trace_buffered_event_cnt);
2781 * Preemption is disabled, but interrupts and NMIs
2782 * can still come in now. If that happens after
2783 * the above increment, then it will have to go
2784 * back to the old method of allocating the event
2785 * on the ring buffer, and if the filter fails, it
2786 * will have to call ring_buffer_discard_commit()
2789 * Need to also check the unlikely case that the
2790 * length is bigger than the temp buffer size.
2791 * If that happens, then the reserve is pretty much
2792 * guaranteed to fail, as the ring buffer currently
2793 * only allows events less than a page. But that may
2794 * change in the future, so let the ring buffer reserve
2795 * handle the failure in that case.
2797 if (val == 1 && likely(len <= max_len)) {
2798 trace_event_setup(entry, type, trace_ctx);
2799 entry->array[0] = len;
2800 /* Return with preemption disabled */
2803 this_cpu_dec(trace_buffered_event_cnt);
2805 /* __trace_buffer_lock_reserve() disables preemption */
2806 preempt_enable_notrace();
2809 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2812 * If tracing is off, but we have triggers enabled
2813 * we still need to look at the event data. Use the temp_buffer
2814 * to store the trace event for the trigger to use. It's recursive
2815 * safe and will not be recorded anywhere.
2817 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2818 *current_rb = temp_buffer;
2819 entry = __trace_buffer_lock_reserve(*current_rb, type, len,
2824 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2826 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2827 static DEFINE_MUTEX(tracepoint_printk_mutex);
2829 static void output_printk(struct trace_event_buffer *fbuffer)
2831 struct trace_event_call *event_call;
2832 struct trace_event_file *file;
2833 struct trace_event *event;
2834 unsigned long flags;
2835 struct trace_iterator *iter = tracepoint_print_iter;
2837 /* We should never get here if iter is NULL */
2838 if (WARN_ON_ONCE(!iter))
2841 event_call = fbuffer->trace_file->event_call;
2842 if (!event_call || !event_call->event.funcs ||
2843 !event_call->event.funcs->trace)
2846 file = fbuffer->trace_file;
2847 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2848 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2849 !filter_match_preds(file->filter, fbuffer->entry)))
2852 event = &fbuffer->trace_file->event_call->event;
2854 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2855 trace_seq_init(&iter->seq);
2856 iter->ent = fbuffer->entry;
2857 event_call->event.funcs->trace(iter, 0, event);
2858 trace_seq_putc(&iter->seq, 0);
2859 printk("%s", iter->seq.buffer);
2861 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2864 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2865 void *buffer, size_t *lenp,
2868 int save_tracepoint_printk;
2871 mutex_lock(&tracepoint_printk_mutex);
2872 save_tracepoint_printk = tracepoint_printk;
2874 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2877 * This will force exiting early, as tracepoint_printk
2878 * is always zero when tracepoint_printk_iter is not allocated
2880 if (!tracepoint_print_iter)
2881 tracepoint_printk = 0;
2883 if (save_tracepoint_printk == tracepoint_printk)
2886 if (tracepoint_printk)
2887 static_key_enable(&tracepoint_printk_key.key);
2889 static_key_disable(&tracepoint_printk_key.key);
2892 mutex_unlock(&tracepoint_printk_mutex);
2897 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2899 enum event_trigger_type tt = ETT_NONE;
2900 struct trace_event_file *file = fbuffer->trace_file;
2902 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,
2903 fbuffer->entry, &tt))
2906 if (static_key_false(&tracepoint_printk_key.key))
2907 output_printk(fbuffer);
2909 if (static_branch_unlikely(&trace_event_exports_enabled))
2910 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
2912 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,
2913 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);
2917 event_triggers_post_call(file, tt);
2920 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2925 * trace_buffer_unlock_commit_regs()
2926 * trace_event_buffer_commit()
2927 * trace_event_raw_event_xxx()
2929 # define STACK_SKIP 3
2931 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2932 struct trace_buffer *buffer,
2933 struct ring_buffer_event *event,
2934 unsigned int trace_ctx,
2935 struct pt_regs *regs)
2937 __buffer_unlock_commit(buffer, event);
2940 * If regs is not set, then skip the necessary functions.
2941 * Note, we can still get here via blktrace, wakeup tracer
2942 * and mmiotrace, but that's ok if they lose a function or
2943 * two. They are not that meaningful.
2945 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
2946 ftrace_trace_userstack(tr, buffer, trace_ctx);
2950 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2953 trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2954 struct ring_buffer_event *event)
2956 __buffer_unlock_commit(buffer, event);
2960 trace_function(struct trace_array *tr, unsigned long ip, unsigned long
2961 parent_ip, unsigned int trace_ctx)
2963 struct trace_event_call *call = &event_function;
2964 struct trace_buffer *buffer = tr->array_buffer.buffer;
2965 struct ring_buffer_event *event;
2966 struct ftrace_entry *entry;
2968 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2972 entry = ring_buffer_event_data(event);
2974 entry->parent_ip = parent_ip;
2976 if (!call_filter_check_discard(call, entry, buffer, event)) {
2977 if (static_branch_unlikely(&trace_function_exports_enabled))
2978 ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2979 __buffer_unlock_commit(buffer, event);
2983 #ifdef CONFIG_STACKTRACE
2985 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2986 #define FTRACE_KSTACK_NESTING 4
2988 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2990 struct ftrace_stack {
2991 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2995 struct ftrace_stacks {
2996 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2999 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
3000 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
3002 static void __ftrace_trace_stack(struct trace_buffer *buffer,
3003 unsigned int trace_ctx,
3004 int skip, struct pt_regs *regs)
3006 struct trace_event_call *call = &event_kernel_stack;
3007 struct ring_buffer_event *event;
3008 unsigned int size, nr_entries;
3009 struct ftrace_stack *fstack;
3010 struct stack_entry *entry;
3014 * Add one, for this function and the call to save_stack_trace()
3015 * If regs is set, then these functions will not be in the way.
3017 #ifndef CONFIG_UNWINDER_ORC
3022 preempt_disable_notrace();
3024 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
3026 /* This should never happen. If it does, yell once and skip */
3027 if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
3031 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
3032 * interrupt will either see the value pre increment or post
3033 * increment. If the interrupt happens pre increment it will have
3034 * restored the counter when it returns. We just need a barrier to
3035 * keep gcc from moving things around.
3039 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
3040 size = ARRAY_SIZE(fstack->calls);
3043 nr_entries = stack_trace_save_regs(regs, fstack->calls,
3046 nr_entries = stack_trace_save(fstack->calls, size, skip);
3049 size = nr_entries * sizeof(unsigned long);
3050 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
3051 (sizeof(*entry) - sizeof(entry->caller)) + size,
3055 entry = ring_buffer_event_data(event);
3057 memcpy(&entry->caller, fstack->calls, size);
3058 entry->size = nr_entries;
3060 if (!call_filter_check_discard(call, entry, buffer, event))
3061 __buffer_unlock_commit(buffer, event);
3064 /* Again, don't let gcc optimize things here */
3066 __this_cpu_dec(ftrace_stack_reserve);
3067 preempt_enable_notrace();
3071 static inline void ftrace_trace_stack(struct trace_array *tr,
3072 struct trace_buffer *buffer,
3073 unsigned int trace_ctx,
3074 int skip, struct pt_regs *regs)
3076 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
3079 __ftrace_trace_stack(buffer, trace_ctx, skip, regs);
3082 void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
3085 struct trace_buffer *buffer = tr->array_buffer.buffer;
3087 if (rcu_is_watching()) {
3088 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3093 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
3094 * but if the above rcu_is_watching() failed, then the NMI
3095 * triggered someplace critical, and rcu_irq_enter() should
3096 * not be called from NMI.
3098 if (unlikely(in_nmi()))
3101 rcu_irq_enter_irqson();
3102 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3103 rcu_irq_exit_irqson();
3107 * trace_dump_stack - record a stack back trace in the trace buffer
3108 * @skip: Number of functions to skip (helper handlers)
3110 void trace_dump_stack(int skip)
3112 if (tracing_disabled || tracing_selftest_running)
3115 #ifndef CONFIG_UNWINDER_ORC
3116 /* Skip 1 to skip this function. */
3119 __ftrace_trace_stack(global_trace.array_buffer.buffer,
3120 tracing_gen_ctx(), skip, NULL);
3122 EXPORT_SYMBOL_GPL(trace_dump_stack);
3124 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
3125 static DEFINE_PER_CPU(int, user_stack_count);
3128 ftrace_trace_userstack(struct trace_array *tr,
3129 struct trace_buffer *buffer, unsigned int trace_ctx)
3131 struct trace_event_call *call = &event_user_stack;
3132 struct ring_buffer_event *event;
3133 struct userstack_entry *entry;
3135 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
3139 * NMIs can not handle page faults, even with fix ups.
3140 * The save user stack can (and often does) fault.
3142 if (unlikely(in_nmi()))
3146 * prevent recursion, since the user stack tracing may
3147 * trigger other kernel events.
3150 if (__this_cpu_read(user_stack_count))
3153 __this_cpu_inc(user_stack_count);
3155 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3156 sizeof(*entry), trace_ctx);
3158 goto out_drop_count;
3159 entry = ring_buffer_event_data(event);
3161 entry->tgid = current->tgid;
3162 memset(&entry->caller, 0, sizeof(entry->caller));
3164 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3165 if (!call_filter_check_discard(call, entry, buffer, event))
3166 __buffer_unlock_commit(buffer, event);
3169 __this_cpu_dec(user_stack_count);
3173 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
3174 static void ftrace_trace_userstack(struct trace_array *tr,
3175 struct trace_buffer *buffer,
3176 unsigned int trace_ctx)
3179 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3181 #endif /* CONFIG_STACKTRACE */
3184 func_repeats_set_delta_ts(struct func_repeats_entry *entry,
3185 unsigned long long delta)
3187 entry->bottom_delta_ts = delta & U32_MAX;
3188 entry->top_delta_ts = (delta >> 32);
3191 void trace_last_func_repeats(struct trace_array *tr,
3192 struct trace_func_repeats *last_info,
3193 unsigned int trace_ctx)
3195 struct trace_buffer *buffer = tr->array_buffer.buffer;
3196 struct func_repeats_entry *entry;
3197 struct ring_buffer_event *event;
3200 event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS,
3201 sizeof(*entry), trace_ctx);
3205 delta = ring_buffer_event_time_stamp(buffer, event) -
3206 last_info->ts_last_call;
3208 entry = ring_buffer_event_data(event);
3209 entry->ip = last_info->ip;
3210 entry->parent_ip = last_info->parent_ip;
3211 entry->count = last_info->count;
3212 func_repeats_set_delta_ts(entry, delta);
3214 __buffer_unlock_commit(buffer, event);
3217 /* created for use with alloc_percpu */
3218 struct trace_buffer_struct {
3220 char buffer[4][TRACE_BUF_SIZE];
3223 static struct trace_buffer_struct __percpu *trace_percpu_buffer;
3226 * This allows for lockless recording. If we're nested too deeply, then
3227 * this returns NULL.
3229 static char *get_trace_buf(void)
3231 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3233 if (!trace_percpu_buffer || buffer->nesting >= 4)
3238 /* Interrupts must see nesting incremented before we use the buffer */
3240 return &buffer->buffer[buffer->nesting - 1][0];
3243 static void put_trace_buf(void)
3245 /* Don't let the decrement of nesting leak before this */
3247 this_cpu_dec(trace_percpu_buffer->nesting);
3250 static int alloc_percpu_trace_buffer(void)
3252 struct trace_buffer_struct __percpu *buffers;
3254 if (trace_percpu_buffer)
3257 buffers = alloc_percpu(struct trace_buffer_struct);
3258 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3261 trace_percpu_buffer = buffers;
3265 static int buffers_allocated;
3267 void trace_printk_init_buffers(void)
3269 if (buffers_allocated)
3272 if (alloc_percpu_trace_buffer())
3275 /* trace_printk() is for debug use only. Don't use it in production. */
3278 pr_warn("**********************************************************\n");
3279 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3281 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3283 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3284 pr_warn("** unsafe for production use. **\n");
3286 pr_warn("** If you see this message and you are not debugging **\n");
3287 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3289 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3290 pr_warn("**********************************************************\n");
3292 /* Expand the buffers to set size */
3293 tracing_update_buffers();
3295 buffers_allocated = 1;
3298 * trace_printk_init_buffers() can be called by modules.
3299 * If that happens, then we need to start cmdline recording
3300 * directly here. If the global_trace.buffer is already
3301 * allocated here, then this was called by module code.
3303 if (global_trace.array_buffer.buffer)
3304 tracing_start_cmdline_record();
3306 EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3308 void trace_printk_start_comm(void)
3310 /* Start tracing comms if trace printk is set */
3311 if (!buffers_allocated)
3313 tracing_start_cmdline_record();
3316 static void trace_printk_start_stop_comm(int enabled)
3318 if (!buffers_allocated)
3322 tracing_start_cmdline_record();
3324 tracing_stop_cmdline_record();
3328 * trace_vbprintk - write binary msg to tracing buffer
3329 * @ip: The address of the caller
3330 * @fmt: The string format to write to the buffer
3331 * @args: Arguments for @fmt
3333 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3335 struct trace_event_call *call = &event_bprint;
3336 struct ring_buffer_event *event;
3337 struct trace_buffer *buffer;
3338 struct trace_array *tr = &global_trace;
3339 struct bprint_entry *entry;
3340 unsigned int trace_ctx;
3344 if (unlikely(tracing_selftest_running || tracing_disabled))
3347 /* Don't pollute graph traces with trace_vprintk internals */
3348 pause_graph_tracing();
3350 trace_ctx = tracing_gen_ctx();
3351 preempt_disable_notrace();
3353 tbuffer = get_trace_buf();
3359 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3361 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3364 size = sizeof(*entry) + sizeof(u32) * len;
3365 buffer = tr->array_buffer.buffer;
3366 ring_buffer_nest_start(buffer);
3367 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3371 entry = ring_buffer_event_data(event);
3375 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3376 if (!call_filter_check_discard(call, entry, buffer, event)) {
3377 __buffer_unlock_commit(buffer, event);
3378 ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL);
3382 ring_buffer_nest_end(buffer);
3387 preempt_enable_notrace();
3388 unpause_graph_tracing();
3392 EXPORT_SYMBOL_GPL(trace_vbprintk);
3396 __trace_array_vprintk(struct trace_buffer *buffer,
3397 unsigned long ip, const char *fmt, va_list args)
3399 struct trace_event_call *call = &event_print;
3400 struct ring_buffer_event *event;
3402 struct print_entry *entry;
3403 unsigned int trace_ctx;
3406 if (tracing_disabled || tracing_selftest_running)
3409 /* Don't pollute graph traces with trace_vprintk internals */
3410 pause_graph_tracing();
3412 trace_ctx = tracing_gen_ctx();
3413 preempt_disable_notrace();
3416 tbuffer = get_trace_buf();
3422 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3424 size = sizeof(*entry) + len + 1;
3425 ring_buffer_nest_start(buffer);
3426 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3430 entry = ring_buffer_event_data(event);
3433 memcpy(&entry->buf, tbuffer, len + 1);
3434 if (!call_filter_check_discard(call, entry, buffer, event)) {
3435 __buffer_unlock_commit(buffer, event);
3436 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL);
3440 ring_buffer_nest_end(buffer);
3444 preempt_enable_notrace();
3445 unpause_graph_tracing();
3451 int trace_array_vprintk(struct trace_array *tr,
3452 unsigned long ip, const char *fmt, va_list args)
3454 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3458 * trace_array_printk - Print a message to a specific instance
3459 * @tr: The instance trace_array descriptor
3460 * @ip: The instruction pointer that this is called from.
3461 * @fmt: The format to print (printf format)
3463 * If a subsystem sets up its own instance, they have the right to
3464 * printk strings into their tracing instance buffer using this
3465 * function. Note, this function will not write into the top level
3466 * buffer (use trace_printk() for that), as writing into the top level
3467 * buffer should only have events that can be individually disabled.
3468 * trace_printk() is only used for debugging a kernel, and should not
3469 * be ever incorporated in normal use.
3471 * trace_array_printk() can be used, as it will not add noise to the
3472 * top level tracing buffer.
3474 * Note, trace_array_init_printk() must be called on @tr before this
3478 int trace_array_printk(struct trace_array *tr,
3479 unsigned long ip, const char *fmt, ...)
3487 /* This is only allowed for created instances */
3488 if (tr == &global_trace)
3491 if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3495 ret = trace_array_vprintk(tr, ip, fmt, ap);
3499 EXPORT_SYMBOL_GPL(trace_array_printk);
3502 * trace_array_init_printk - Initialize buffers for trace_array_printk()
3503 * @tr: The trace array to initialize the buffers for
3505 * As trace_array_printk() only writes into instances, they are OK to
3506 * have in the kernel (unlike trace_printk()). This needs to be called
3507 * before trace_array_printk() can be used on a trace_array.
3509 int trace_array_init_printk(struct trace_array *tr)
3514 /* This is only allowed for created instances */
3515 if (tr == &global_trace)
3518 return alloc_percpu_trace_buffer();
3520 EXPORT_SYMBOL_GPL(trace_array_init_printk);
3523 int trace_array_printk_buf(struct trace_buffer *buffer,
3524 unsigned long ip, const char *fmt, ...)
3529 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3533 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3539 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3541 return trace_array_vprintk(&global_trace, ip, fmt, args);
3543 EXPORT_SYMBOL_GPL(trace_vprintk);
3545 static void trace_iterator_increment(struct trace_iterator *iter)
3547 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3551 ring_buffer_iter_advance(buf_iter);
3554 static struct trace_entry *
3555 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3556 unsigned long *lost_events)
3558 struct ring_buffer_event *event;
3559 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3562 event = ring_buffer_iter_peek(buf_iter, ts);
3564 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
3565 (unsigned long)-1 : 0;
3567 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3572 iter->ent_size = ring_buffer_event_length(event);
3573 return ring_buffer_event_data(event);
3579 static struct trace_entry *
3580 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3581 unsigned long *missing_events, u64 *ent_ts)
3583 struct trace_buffer *buffer = iter->array_buffer->buffer;
3584 struct trace_entry *ent, *next = NULL;
3585 unsigned long lost_events = 0, next_lost = 0;
3586 int cpu_file = iter->cpu_file;
3587 u64 next_ts = 0, ts;
3593 * If we are in a per_cpu trace file, don't bother by iterating over
3594 * all cpu and peek directly.
3596 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3597 if (ring_buffer_empty_cpu(buffer, cpu_file))
3599 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3601 *ent_cpu = cpu_file;
3606 for_each_tracing_cpu(cpu) {
3608 if (ring_buffer_empty_cpu(buffer, cpu))
3611 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3614 * Pick the entry with the smallest timestamp:
3616 if (ent && (!next || ts < next_ts)) {
3620 next_lost = lost_events;
3621 next_size = iter->ent_size;
3625 iter->ent_size = next_size;
3628 *ent_cpu = next_cpu;
3634 *missing_events = next_lost;
3639 #define STATIC_FMT_BUF_SIZE 128
3640 static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
3642 static char *trace_iter_expand_format(struct trace_iterator *iter)
3647 * iter->tr is NULL when used with tp_printk, which makes
3648 * this get called where it is not safe to call krealloc().
3650 if (!iter->tr || iter->fmt == static_fmt_buf)
3653 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,
3656 iter->fmt_size += STATIC_FMT_BUF_SIZE;
3663 /* Returns true if the string is safe to dereference from an event */
3664 static bool trace_safe_str(struct trace_iterator *iter, const char *str)
3666 unsigned long addr = (unsigned long)str;
3667 struct trace_event *trace_event;
3668 struct trace_event_call *event;
3670 /* OK if part of the event data */
3671 if ((addr >= (unsigned long)iter->ent) &&
3672 (addr < (unsigned long)iter->ent + iter->ent_size))
3675 /* OK if part of the temp seq buffer */
3676 if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
3677 (addr < (unsigned long)iter->tmp_seq.buffer + PAGE_SIZE))
3680 /* Core rodata can not be freed */
3681 if (is_kernel_rodata(addr))
3684 if (trace_is_tracepoint_string(str))
3688 * Now this could be a module event, referencing core module
3689 * data, which is OK.
3694 trace_event = ftrace_find_event(iter->ent->type);
3698 event = container_of(trace_event, struct trace_event_call, event);
3699 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module)
3702 /* Would rather have rodata, but this will suffice */
3703 if (within_module_core(addr, event->module))
3709 static const char *show_buffer(struct trace_seq *s)
3711 struct seq_buf *seq = &s->seq;
3713 seq_buf_terminate(seq);
3718 static DEFINE_STATIC_KEY_FALSE(trace_no_verify);
3720 static int test_can_verify_check(const char *fmt, ...)
3727 * The verifier is dependent on vsnprintf() modifies the va_list
3728 * passed to it, where it is sent as a reference. Some architectures
3729 * (like x86_32) passes it by value, which means that vsnprintf()
3730 * does not modify the va_list passed to it, and the verifier
3731 * would then need to be able to understand all the values that
3732 * vsnprintf can use. If it is passed by value, then the verifier
3736 vsnprintf(buf, 16, "%d", ap);
3737 ret = va_arg(ap, int);
3743 static void test_can_verify(void)
3745 if (!test_can_verify_check("%d %d", 0, 1)) {
3746 pr_info("trace event string verifier disabled\n");
3747 static_branch_inc(&trace_no_verify);
3752 * trace_check_vprintf - Check dereferenced strings while writing to the seq buffer
3753 * @iter: The iterator that holds the seq buffer and the event being printed
3754 * @fmt: The format used to print the event
3755 * @ap: The va_list holding the data to print from @fmt.
3757 * This writes the data into the @iter->seq buffer using the data from
3758 * @fmt and @ap. If the format has a %s, then the source of the string
3759 * is examined to make sure it is safe to print, otherwise it will
3760 * warn and print "[UNSAFE MEMORY]" in place of the dereferenced string
3763 void trace_check_vprintf(struct trace_iterator *iter, const char *fmt,
3766 const char *p = fmt;
3770 if (WARN_ON_ONCE(!fmt))
3773 if (static_branch_unlikely(&trace_no_verify))
3776 /* Don't bother checking when doing a ftrace_dump() */
3777 if (iter->fmt == static_fmt_buf)
3786 /* We only care about %s and variants */
3787 for (i = 0; p[i]; i++) {
3788 if (i + 1 >= iter->fmt_size) {
3790 * If we can't expand the copy buffer,
3793 if (!trace_iter_expand_format(iter))
3797 if (p[i] == '\\' && p[i+1]) {
3802 /* Need to test cases like %08.*s */
3803 for (j = 1; p[i+j]; j++) {
3804 if (isdigit(p[i+j]) ||
3807 if (p[i+j] == '*') {
3819 /* If no %s found then just print normally */
3823 /* Copy up to the %s, and print that */
3824 strncpy(iter->fmt, p, i);
3825 iter->fmt[i] = '\0';
3826 trace_seq_vprintf(&iter->seq, iter->fmt, ap);
3829 * If iter->seq is full, the above call no longer guarantees
3830 * that ap is in sync with fmt processing, and further calls
3831 * to va_arg() can return wrong positional arguments.
3833 * Ensure that ap is no longer used in this case.
3835 if (iter->seq.full) {
3841 len = va_arg(ap, int);
3843 /* The ap now points to the string data of the %s */
3844 str = va_arg(ap, const char *);
3847 * If you hit this warning, it is likely that the
3848 * trace event in question used %s on a string that
3849 * was saved at the time of the event, but may not be
3850 * around when the trace is read. Use __string(),
3851 * __assign_str() and __get_str() helpers in the TRACE_EVENT()
3852 * instead. See samples/trace_events/trace-events-sample.h
3855 if (WARN_ONCE(!trace_safe_str(iter, str),
3856 "fmt: '%s' current_buffer: '%s'",
3857 fmt, show_buffer(&iter->seq))) {
3860 /* Try to safely read the string */
3862 if (len + 1 > iter->fmt_size)
3863 len = iter->fmt_size - 1;
3866 ret = copy_from_kernel_nofault(iter->fmt, str, len);
3870 ret = strncpy_from_kernel_nofault(iter->fmt, str,
3874 trace_seq_printf(&iter->seq, "(0x%px)", str);
3876 trace_seq_printf(&iter->seq, "(0x%px:%s)",
3878 str = "[UNSAFE-MEMORY]";
3879 strcpy(iter->fmt, "%s");
3881 strncpy(iter->fmt, p + i, j + 1);
3882 iter->fmt[j+1] = '\0';
3885 trace_seq_printf(&iter->seq, iter->fmt, len, str);
3887 trace_seq_printf(&iter->seq, iter->fmt, str);
3893 trace_seq_vprintf(&iter->seq, p, ap);
3896 const char *trace_event_format(struct trace_iterator *iter, const char *fmt)
3898 const char *p, *new_fmt;
3901 if (WARN_ON_ONCE(!fmt))
3904 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR)
3908 new_fmt = q = iter->fmt;
3910 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
3911 if (!trace_iter_expand_format(iter))
3914 q += iter->fmt - new_fmt;
3915 new_fmt = iter->fmt;
3920 /* Replace %p with %px */
3924 } else if (p[0] == 'p' && !isalnum(p[1])) {
3935 #define STATIC_TEMP_BUF_SIZE 128
3936 static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
3938 /* Find the next real entry, without updating the iterator itself */
3939 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3940 int *ent_cpu, u64 *ent_ts)
3942 /* __find_next_entry will reset ent_size */
3943 int ent_size = iter->ent_size;
3944 struct trace_entry *entry;
3947 * If called from ftrace_dump(), then the iter->temp buffer
3948 * will be the static_temp_buf and not created from kmalloc.
3949 * If the entry size is greater than the buffer, we can
3950 * not save it. Just return NULL in that case. This is only
3951 * used to add markers when two consecutive events' time
3952 * stamps have a large delta. See trace_print_lat_context()
3954 if (iter->temp == static_temp_buf &&
3955 STATIC_TEMP_BUF_SIZE < ent_size)
3959 * The __find_next_entry() may call peek_next_entry(), which may
3960 * call ring_buffer_peek() that may make the contents of iter->ent
3961 * undefined. Need to copy iter->ent now.
3963 if (iter->ent && iter->ent != iter->temp) {
3964 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
3965 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3967 temp = kmalloc(iter->ent_size, GFP_KERNEL);
3972 iter->temp_size = iter->ent_size;
3974 memcpy(iter->temp, iter->ent, iter->ent_size);
3975 iter->ent = iter->temp;
3977 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3978 /* Put back the original ent_size */
3979 iter->ent_size = ent_size;
3984 /* Find the next real entry, and increment the iterator to the next entry */
3985 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3987 iter->ent = __find_next_entry(iter, &iter->cpu,
3988 &iter->lost_events, &iter->ts);
3991 trace_iterator_increment(iter);
3993 return iter->ent ? iter : NULL;
3996 static void trace_consume(struct trace_iterator *iter)
3998 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
3999 &iter->lost_events);
4002 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
4004 struct trace_iterator *iter = m->private;
4008 WARN_ON_ONCE(iter->leftover);
4012 /* can't go backwards */
4017 ent = trace_find_next_entry_inc(iter);
4021 while (ent && iter->idx < i)
4022 ent = trace_find_next_entry_inc(iter);
4029 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
4031 struct ring_buffer_iter *buf_iter;
4032 unsigned long entries = 0;
4035 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
4037 buf_iter = trace_buffer_iter(iter, cpu);
4041 ring_buffer_iter_reset(buf_iter);
4044 * We could have the case with the max latency tracers
4045 * that a reset never took place on a cpu. This is evident
4046 * by the timestamp being before the start of the buffer.
4048 while (ring_buffer_iter_peek(buf_iter, &ts)) {
4049 if (ts >= iter->array_buffer->time_start)
4052 ring_buffer_iter_advance(buf_iter);
4055 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
4059 * The current tracer is copied to avoid a global locking
4062 static void *s_start(struct seq_file *m, loff_t *pos)
4064 struct trace_iterator *iter = m->private;
4065 struct trace_array *tr = iter->tr;
4066 int cpu_file = iter->cpu_file;
4072 * copy the tracer to avoid using a global lock all around.
4073 * iter->trace is a copy of current_trace, the pointer to the
4074 * name may be used instead of a strcmp(), as iter->trace->name
4075 * will point to the same string as current_trace->name.
4077 mutex_lock(&trace_types_lock);
4078 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
4079 *iter->trace = *tr->current_trace;
4080 mutex_unlock(&trace_types_lock);
4082 #ifdef CONFIG_TRACER_MAX_TRACE
4083 if (iter->snapshot && iter->trace->use_max_tr)
4084 return ERR_PTR(-EBUSY);
4087 if (*pos != iter->pos) {
4092 if (cpu_file == RING_BUFFER_ALL_CPUS) {
4093 for_each_tracing_cpu(cpu)
4094 tracing_iter_reset(iter, cpu);
4096 tracing_iter_reset(iter, cpu_file);
4099 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
4104 * If we overflowed the seq_file before, then we want
4105 * to just reuse the trace_seq buffer again.
4111 p = s_next(m, p, &l);
4115 trace_event_read_lock();
4116 trace_access_lock(cpu_file);
4120 static void s_stop(struct seq_file *m, void *p)
4122 struct trace_iterator *iter = m->private;
4124 #ifdef CONFIG_TRACER_MAX_TRACE
4125 if (iter->snapshot && iter->trace->use_max_tr)
4129 trace_access_unlock(iter->cpu_file);
4130 trace_event_read_unlock();
4134 get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
4135 unsigned long *entries, int cpu)
4137 unsigned long count;
4139 count = ring_buffer_entries_cpu(buf->buffer, cpu);
4141 * If this buffer has skipped entries, then we hold all
4142 * entries for the trace and we need to ignore the
4143 * ones before the time stamp.
4145 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
4146 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
4147 /* total is the same as the entries */
4151 ring_buffer_overrun_cpu(buf->buffer, cpu);
4156 get_total_entries(struct array_buffer *buf,
4157 unsigned long *total, unsigned long *entries)
4165 for_each_tracing_cpu(cpu) {
4166 get_total_entries_cpu(buf, &t, &e, cpu);
4172 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
4174 unsigned long total, entries;
4179 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
4184 unsigned long trace_total_entries(struct trace_array *tr)
4186 unsigned long total, entries;
4191 get_total_entries(&tr->array_buffer, &total, &entries);
4196 static void print_lat_help_header(struct seq_file *m)
4198 seq_puts(m, "# _------=> CPU# \n"
4199 "# / _-----=> irqs-off/BH-disabled\n"
4200 "# | / _----=> need-resched \n"
4201 "# || / _---=> hardirq/softirq \n"
4202 "# ||| / _--=> preempt-depth \n"
4203 "# |||| / _-=> migrate-disable \n"
4204 "# ||||| / delay \n"
4205 "# cmd pid |||||| time | caller \n"
4206 "# \\ / |||||| \\ | / \n");
4209 static void print_event_info(struct array_buffer *buf, struct seq_file *m)
4211 unsigned long total;
4212 unsigned long entries;
4214 get_total_entries(buf, &total, &entries);
4215 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
4216 entries, total, num_online_cpus());
4220 static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
4223 bool tgid = flags & TRACE_ITER_RECORD_TGID;
4225 print_event_info(buf, m);
4227 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
4228 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
4231 static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
4234 bool tgid = flags & TRACE_ITER_RECORD_TGID;
4235 const char *space = " ";
4236 int prec = tgid ? 12 : 2;
4238 print_event_info(buf, m);
4240 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space);
4241 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
4242 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
4243 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
4244 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space);
4245 seq_printf(m, "# %.*s|||| / delay\n", prec, space);
4246 seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID ");
4247 seq_printf(m, "# | | %.*s | ||||| | |\n", prec, " | ");
4251 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
4253 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
4254 struct array_buffer *buf = iter->array_buffer;
4255 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
4256 struct tracer *type = iter->trace;
4257 unsigned long entries;
4258 unsigned long total;
4259 const char *name = "preemption";
4263 get_total_entries(buf, &total, &entries);
4265 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
4267 seq_puts(m, "# -----------------------------------"
4268 "---------------------------------\n");
4269 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
4270 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
4271 nsecs_to_usecs(data->saved_latency),
4275 #if defined(CONFIG_PREEMPT_NONE)
4277 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
4279 #elif defined(CONFIG_PREEMPT)
4281 #elif defined(CONFIG_PREEMPT_RT)
4286 /* These are reserved for later use */
4289 seq_printf(m, " #P:%d)\n", num_online_cpus());
4293 seq_puts(m, "# -----------------\n");
4294 seq_printf(m, "# | task: %.16s-%d "
4295 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
4296 data->comm, data->pid,
4297 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
4298 data->policy, data->rt_priority);
4299 seq_puts(m, "# -----------------\n");
4301 if (data->critical_start) {
4302 seq_puts(m, "# => started at: ");
4303 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
4304 trace_print_seq(m, &iter->seq);
4305 seq_puts(m, "\n# => ended at: ");
4306 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
4307 trace_print_seq(m, &iter->seq);
4308 seq_puts(m, "\n#\n");
4314 static void test_cpu_buff_start(struct trace_iterator *iter)
4316 struct trace_seq *s = &iter->seq;
4317 struct trace_array *tr = iter->tr;
4319 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
4322 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
4325 if (cpumask_available(iter->started) &&
4326 cpumask_test_cpu(iter->cpu, iter->started))
4329 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
4332 if (cpumask_available(iter->started))
4333 cpumask_set_cpu(iter->cpu, iter->started);
4335 /* Don't print started cpu buffer for the first entry of the trace */
4337 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
4341 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
4343 struct trace_array *tr = iter->tr;
4344 struct trace_seq *s = &iter->seq;
4345 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4346 struct trace_entry *entry;
4347 struct trace_event *event;
4351 test_cpu_buff_start(iter);
4353 event = ftrace_find_event(entry->type);
4355 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4356 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4357 trace_print_lat_context(iter);
4359 trace_print_context(iter);
4362 if (trace_seq_has_overflowed(s))
4363 return TRACE_TYPE_PARTIAL_LINE;
4366 return event->funcs->trace(iter, sym_flags, event);
4368 trace_seq_printf(s, "Unknown type %d\n", entry->type);
4370 return trace_handle_return(s);
4373 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
4375 struct trace_array *tr = iter->tr;
4376 struct trace_seq *s = &iter->seq;
4377 struct trace_entry *entry;
4378 struct trace_event *event;
4382 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
4383 trace_seq_printf(s, "%d %d %llu ",
4384 entry->pid, iter->cpu, iter->ts);
4386 if (trace_seq_has_overflowed(s))
4387 return TRACE_TYPE_PARTIAL_LINE;
4389 event = ftrace_find_event(entry->type);
4391 return event->funcs->raw(iter, 0, event);
4393 trace_seq_printf(s, "%d ?\n", entry->type);
4395 return trace_handle_return(s);
4398 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
4400 struct trace_array *tr = iter->tr;
4401 struct trace_seq *s = &iter->seq;
4402 unsigned char newline = '\n';
4403 struct trace_entry *entry;
4404 struct trace_event *event;
4408 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4409 SEQ_PUT_HEX_FIELD(s, entry->pid);
4410 SEQ_PUT_HEX_FIELD(s, iter->cpu);
4411 SEQ_PUT_HEX_FIELD(s, iter->ts);
4412 if (trace_seq_has_overflowed(s))
4413 return TRACE_TYPE_PARTIAL_LINE;
4416 event = ftrace_find_event(entry->type);
4418 enum print_line_t ret = event->funcs->hex(iter, 0, event);
4419 if (ret != TRACE_TYPE_HANDLED)
4423 SEQ_PUT_FIELD(s, newline);
4425 return trace_handle_return(s);
4428 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
4430 struct trace_array *tr = iter->tr;
4431 struct trace_seq *s = &iter->seq;
4432 struct trace_entry *entry;
4433 struct trace_event *event;
4437 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4438 SEQ_PUT_FIELD(s, entry->pid);
4439 SEQ_PUT_FIELD(s, iter->cpu);
4440 SEQ_PUT_FIELD(s, iter->ts);
4441 if (trace_seq_has_overflowed(s))
4442 return TRACE_TYPE_PARTIAL_LINE;
4445 event = ftrace_find_event(entry->type);
4446 return event ? event->funcs->binary(iter, 0, event) :
4450 int trace_empty(struct trace_iterator *iter)
4452 struct ring_buffer_iter *buf_iter;
4455 /* If we are looking at one CPU buffer, only check that one */
4456 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4457 cpu = iter->cpu_file;
4458 buf_iter = trace_buffer_iter(iter, cpu);
4460 if (!ring_buffer_iter_empty(buf_iter))
4463 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4469 for_each_tracing_cpu(cpu) {
4470 buf_iter = trace_buffer_iter(iter, cpu);
4472 if (!ring_buffer_iter_empty(buf_iter))
4475 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4483 /* Called with trace_event_read_lock() held. */
4484 enum print_line_t print_trace_line(struct trace_iterator *iter)
4486 struct trace_array *tr = iter->tr;
4487 unsigned long trace_flags = tr->trace_flags;
4488 enum print_line_t ret;
4490 if (iter->lost_events) {
4491 if (iter->lost_events == (unsigned long)-1)
4492 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4495 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4496 iter->cpu, iter->lost_events);
4497 if (trace_seq_has_overflowed(&iter->seq))
4498 return TRACE_TYPE_PARTIAL_LINE;
4501 if (iter->trace && iter->trace->print_line) {
4502 ret = iter->trace->print_line(iter);
4503 if (ret != TRACE_TYPE_UNHANDLED)
4507 if (iter->ent->type == TRACE_BPUTS &&
4508 trace_flags & TRACE_ITER_PRINTK &&
4509 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4510 return trace_print_bputs_msg_only(iter);
4512 if (iter->ent->type == TRACE_BPRINT &&
4513 trace_flags & TRACE_ITER_PRINTK &&
4514 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4515 return trace_print_bprintk_msg_only(iter);
4517 if (iter->ent->type == TRACE_PRINT &&
4518 trace_flags & TRACE_ITER_PRINTK &&
4519 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4520 return trace_print_printk_msg_only(iter);
4522 if (trace_flags & TRACE_ITER_BIN)
4523 return print_bin_fmt(iter);
4525 if (trace_flags & TRACE_ITER_HEX)
4526 return print_hex_fmt(iter);
4528 if (trace_flags & TRACE_ITER_RAW)
4529 return print_raw_fmt(iter);
4531 return print_trace_fmt(iter);
4534 void trace_latency_header(struct seq_file *m)
4536 struct trace_iterator *iter = m->private;
4537 struct trace_array *tr = iter->tr;
4539 /* print nothing if the buffers are empty */
4540 if (trace_empty(iter))
4543 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4544 print_trace_header(m, iter);
4546 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4547 print_lat_help_header(m);
4550 void trace_default_header(struct seq_file *m)
4552 struct trace_iterator *iter = m->private;
4553 struct trace_array *tr = iter->tr;
4554 unsigned long trace_flags = tr->trace_flags;
4556 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4559 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4560 /* print nothing if the buffers are empty */
4561 if (trace_empty(iter))
4563 print_trace_header(m, iter);
4564 if (!(trace_flags & TRACE_ITER_VERBOSE))
4565 print_lat_help_header(m);
4567 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4568 if (trace_flags & TRACE_ITER_IRQ_INFO)
4569 print_func_help_header_irq(iter->array_buffer,
4572 print_func_help_header(iter->array_buffer, m,
4578 static void test_ftrace_alive(struct seq_file *m)
4580 if (!ftrace_is_dead())
4582 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4583 "# MAY BE MISSING FUNCTION EVENTS\n");
4586 #ifdef CONFIG_TRACER_MAX_TRACE
4587 static void show_snapshot_main_help(struct seq_file *m)
4589 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4590 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4591 "# Takes a snapshot of the main buffer.\n"
4592 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4593 "# (Doesn't have to be '2' works with any number that\n"
4594 "# is not a '0' or '1')\n");
4597 static void show_snapshot_percpu_help(struct seq_file *m)
4599 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4600 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4601 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4602 "# Takes a snapshot of the main buffer for this cpu.\n");
4604 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4605 "# Must use main snapshot file to allocate.\n");
4607 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4608 "# (Doesn't have to be '2' works with any number that\n"
4609 "# is not a '0' or '1')\n");
4612 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4614 if (iter->tr->allocated_snapshot)
4615 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4617 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4619 seq_puts(m, "# Snapshot commands:\n");
4620 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4621 show_snapshot_main_help(m);
4623 show_snapshot_percpu_help(m);
4626 /* Should never be called */
4627 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4630 static int s_show(struct seq_file *m, void *v)
4632 struct trace_iterator *iter = v;
4635 if (iter->ent == NULL) {
4637 seq_printf(m, "# tracer: %s\n", iter->trace->name);
4639 test_ftrace_alive(m);
4641 if (iter->snapshot && trace_empty(iter))
4642 print_snapshot_help(m, iter);
4643 else if (iter->trace && iter->trace->print_header)
4644 iter->trace->print_header(m);
4646 trace_default_header(m);
4648 } else if (iter->leftover) {
4650 * If we filled the seq_file buffer earlier, we
4651 * want to just show it now.
4653 ret = trace_print_seq(m, &iter->seq);
4655 /* ret should this time be zero, but you never know */
4656 iter->leftover = ret;
4659 print_trace_line(iter);
4660 ret = trace_print_seq(m, &iter->seq);
4662 * If we overflow the seq_file buffer, then it will
4663 * ask us for this data again at start up.
4665 * ret is 0 if seq_file write succeeded.
4668 iter->leftover = ret;
4675 * Should be used after trace_array_get(), trace_types_lock
4676 * ensures that i_cdev was already initialized.
4678 static inline int tracing_get_cpu(struct inode *inode)
4680 if (inode->i_cdev) /* See trace_create_cpu_file() */
4681 return (long)inode->i_cdev - 1;
4682 return RING_BUFFER_ALL_CPUS;
4685 static const struct seq_operations tracer_seq_ops = {
4692 static struct trace_iterator *
4693 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4695 struct trace_array *tr = inode->i_private;
4696 struct trace_iterator *iter;
4699 if (tracing_disabled)
4700 return ERR_PTR(-ENODEV);
4702 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4704 return ERR_PTR(-ENOMEM);
4706 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4708 if (!iter->buffer_iter)
4712 * trace_find_next_entry() may need to save off iter->ent.
4713 * It will place it into the iter->temp buffer. As most
4714 * events are less than 128, allocate a buffer of that size.
4715 * If one is greater, then trace_find_next_entry() will
4716 * allocate a new buffer to adjust for the bigger iter->ent.
4717 * It's not critical if it fails to get allocated here.
4719 iter->temp = kmalloc(128, GFP_KERNEL);
4721 iter->temp_size = 128;
4724 * trace_event_printf() may need to modify given format
4725 * string to replace %p with %px so that it shows real address
4726 * instead of hash value. However, that is only for the event
4727 * tracing, other tracer may not need. Defer the allocation
4728 * until it is needed.
4734 * We make a copy of the current tracer to avoid concurrent
4735 * changes on it while we are reading.
4737 mutex_lock(&trace_types_lock);
4738 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4742 *iter->trace = *tr->current_trace;
4744 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4749 #ifdef CONFIG_TRACER_MAX_TRACE
4750 /* Currently only the top directory has a snapshot */
4751 if (tr->current_trace->print_max || snapshot)
4752 iter->array_buffer = &tr->max_buffer;
4755 iter->array_buffer = &tr->array_buffer;
4756 iter->snapshot = snapshot;
4758 iter->cpu_file = tracing_get_cpu(inode);
4759 mutex_init(&iter->mutex);
4761 /* Notify the tracer early; before we stop tracing. */
4762 if (iter->trace->open)
4763 iter->trace->open(iter);
4765 /* Annotate start of buffers if we had overruns */
4766 if (ring_buffer_overruns(iter->array_buffer->buffer))
4767 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4769 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4770 if (trace_clocks[tr->clock_id].in_ns)
4771 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4774 * If pause-on-trace is enabled, then stop the trace while
4775 * dumping, unless this is the "snapshot" file
4777 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4778 tracing_stop_tr(tr);
4780 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4781 for_each_tracing_cpu(cpu) {
4782 iter->buffer_iter[cpu] =
4783 ring_buffer_read_prepare(iter->array_buffer->buffer,
4786 ring_buffer_read_prepare_sync();
4787 for_each_tracing_cpu(cpu) {
4788 ring_buffer_read_start(iter->buffer_iter[cpu]);
4789 tracing_iter_reset(iter, cpu);
4792 cpu = iter->cpu_file;
4793 iter->buffer_iter[cpu] =
4794 ring_buffer_read_prepare(iter->array_buffer->buffer,
4796 ring_buffer_read_prepare_sync();
4797 ring_buffer_read_start(iter->buffer_iter[cpu]);
4798 tracing_iter_reset(iter, cpu);
4801 mutex_unlock(&trace_types_lock);
4806 mutex_unlock(&trace_types_lock);
4809 kfree(iter->buffer_iter);
4811 seq_release_private(inode, file);
4812 return ERR_PTR(-ENOMEM);
4815 int tracing_open_generic(struct inode *inode, struct file *filp)
4819 ret = tracing_check_open_get_tr(NULL);
4823 filp->private_data = inode->i_private;
4827 bool tracing_is_disabled(void)
4829 return (tracing_disabled) ? true: false;
4833 * Open and update trace_array ref count.
4834 * Must have the current trace_array passed to it.
4836 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4838 struct trace_array *tr = inode->i_private;
4841 ret = tracing_check_open_get_tr(tr);
4845 filp->private_data = inode->i_private;
4850 static int tracing_mark_open(struct inode *inode, struct file *filp)
4852 stream_open(inode, filp);
4853 return tracing_open_generic_tr(inode, filp);
4856 static int tracing_release(struct inode *inode, struct file *file)
4858 struct trace_array *tr = inode->i_private;
4859 struct seq_file *m = file->private_data;
4860 struct trace_iterator *iter;
4863 if (!(file->f_mode & FMODE_READ)) {
4864 trace_array_put(tr);
4868 /* Writes do not use seq_file */
4870 mutex_lock(&trace_types_lock);
4872 for_each_tracing_cpu(cpu) {
4873 if (iter->buffer_iter[cpu])
4874 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4877 if (iter->trace && iter->trace->close)
4878 iter->trace->close(iter);
4880 if (!iter->snapshot && tr->stop_count)
4881 /* reenable tracing if it was previously enabled */
4882 tracing_start_tr(tr);
4884 __trace_array_put(tr);
4886 mutex_unlock(&trace_types_lock);
4888 mutex_destroy(&iter->mutex);
4889 free_cpumask_var(iter->started);
4893 kfree(iter->buffer_iter);
4894 seq_release_private(inode, file);
4899 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4901 struct trace_array *tr = inode->i_private;
4903 trace_array_put(tr);
4907 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4909 struct trace_array *tr = inode->i_private;
4911 trace_array_put(tr);
4913 return single_release(inode, file);
4916 static int tracing_open(struct inode *inode, struct file *file)
4918 struct trace_array *tr = inode->i_private;
4919 struct trace_iterator *iter;
4922 ret = tracing_check_open_get_tr(tr);
4926 /* If this file was open for write, then erase contents */
4927 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4928 int cpu = tracing_get_cpu(inode);
4929 struct array_buffer *trace_buf = &tr->array_buffer;
4931 #ifdef CONFIG_TRACER_MAX_TRACE
4932 if (tr->current_trace->print_max)
4933 trace_buf = &tr->max_buffer;
4936 if (cpu == RING_BUFFER_ALL_CPUS)
4937 tracing_reset_online_cpus(trace_buf);
4939 tracing_reset_cpu(trace_buf, cpu);
4942 if (file->f_mode & FMODE_READ) {
4943 iter = __tracing_open(inode, file, false);
4945 ret = PTR_ERR(iter);
4946 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4947 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4951 trace_array_put(tr);
4957 * Some tracers are not suitable for instance buffers.
4958 * A tracer is always available for the global array (toplevel)
4959 * or if it explicitly states that it is.
4962 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4964 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4967 /* Find the next tracer that this trace array may use */
4968 static struct tracer *
4969 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4971 while (t && !trace_ok_for_array(t, tr))
4978 t_next(struct seq_file *m, void *v, loff_t *pos)
4980 struct trace_array *tr = m->private;
4981 struct tracer *t = v;
4986 t = get_tracer_for_array(tr, t->next);
4991 static void *t_start(struct seq_file *m, loff_t *pos)
4993 struct trace_array *tr = m->private;
4997 mutex_lock(&trace_types_lock);
4999 t = get_tracer_for_array(tr, trace_types);
5000 for (; t && l < *pos; t = t_next(m, t, &l))
5006 static void t_stop(struct seq_file *m, void *p)
5008 mutex_unlock(&trace_types_lock);
5011 static int t_show(struct seq_file *m, void *v)
5013 struct tracer *t = v;
5018 seq_puts(m, t->name);
5027 static const struct seq_operations show_traces_seq_ops = {
5034 static int show_traces_open(struct inode *inode, struct file *file)
5036 struct trace_array *tr = inode->i_private;
5040 ret = tracing_check_open_get_tr(tr);
5044 ret = seq_open(file, &show_traces_seq_ops);
5046 trace_array_put(tr);
5050 m = file->private_data;
5056 static int show_traces_release(struct inode *inode, struct file *file)
5058 struct trace_array *tr = inode->i_private;
5060 trace_array_put(tr);
5061 return seq_release(inode, file);
5065 tracing_write_stub(struct file *filp, const char __user *ubuf,
5066 size_t count, loff_t *ppos)
5071 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
5075 if (file->f_mode & FMODE_READ)
5076 ret = seq_lseek(file, offset, whence);
5078 file->f_pos = ret = 0;
5083 static const struct file_operations tracing_fops = {
5084 .open = tracing_open,
5086 .write = tracing_write_stub,
5087 .llseek = tracing_lseek,
5088 .release = tracing_release,
5091 static const struct file_operations show_traces_fops = {
5092 .open = show_traces_open,
5094 .llseek = seq_lseek,
5095 .release = show_traces_release,
5099 tracing_cpumask_read(struct file *filp, char __user *ubuf,
5100 size_t count, loff_t *ppos)
5102 struct trace_array *tr = file_inode(filp)->i_private;
5106 len = snprintf(NULL, 0, "%*pb\n",
5107 cpumask_pr_args(tr->tracing_cpumask)) + 1;
5108 mask_str = kmalloc(len, GFP_KERNEL);
5112 len = snprintf(mask_str, len, "%*pb\n",
5113 cpumask_pr_args(tr->tracing_cpumask));
5118 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
5126 int tracing_set_cpumask(struct trace_array *tr,
5127 cpumask_var_t tracing_cpumask_new)
5134 local_irq_disable();
5135 arch_spin_lock(&tr->max_lock);
5136 for_each_tracing_cpu(cpu) {
5138 * Increase/decrease the disabled counter if we are
5139 * about to flip a bit in the cpumask:
5141 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5142 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5143 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5144 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
5146 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
5147 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
5148 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5149 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
5152 arch_spin_unlock(&tr->max_lock);
5155 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
5161 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
5162 size_t count, loff_t *ppos)
5164 struct trace_array *tr = file_inode(filp)->i_private;
5165 cpumask_var_t tracing_cpumask_new;
5168 if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
5171 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
5175 err = tracing_set_cpumask(tr, tracing_cpumask_new);
5179 free_cpumask_var(tracing_cpumask_new);
5184 free_cpumask_var(tracing_cpumask_new);
5189 static const struct file_operations tracing_cpumask_fops = {
5190 .open = tracing_open_generic_tr,
5191 .read = tracing_cpumask_read,
5192 .write = tracing_cpumask_write,
5193 .release = tracing_release_generic_tr,
5194 .llseek = generic_file_llseek,
5197 static int tracing_trace_options_show(struct seq_file *m, void *v)
5199 struct tracer_opt *trace_opts;
5200 struct trace_array *tr = m->private;
5204 mutex_lock(&trace_types_lock);
5205 tracer_flags = tr->current_trace->flags->val;
5206 trace_opts = tr->current_trace->flags->opts;
5208 for (i = 0; trace_options[i]; i++) {
5209 if (tr->trace_flags & (1 << i))
5210 seq_printf(m, "%s\n", trace_options[i]);
5212 seq_printf(m, "no%s\n", trace_options[i]);
5215 for (i = 0; trace_opts[i].name; i++) {
5216 if (tracer_flags & trace_opts[i].bit)
5217 seq_printf(m, "%s\n", trace_opts[i].name);
5219 seq_printf(m, "no%s\n", trace_opts[i].name);
5221 mutex_unlock(&trace_types_lock);
5226 static int __set_tracer_option(struct trace_array *tr,
5227 struct tracer_flags *tracer_flags,
5228 struct tracer_opt *opts, int neg)
5230 struct tracer *trace = tracer_flags->trace;
5233 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
5238 tracer_flags->val &= ~opts->bit;
5240 tracer_flags->val |= opts->bit;
5244 /* Try to assign a tracer specific option */
5245 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
5247 struct tracer *trace = tr->current_trace;
5248 struct tracer_flags *tracer_flags = trace->flags;
5249 struct tracer_opt *opts = NULL;
5252 for (i = 0; tracer_flags->opts[i].name; i++) {
5253 opts = &tracer_flags->opts[i];
5255 if (strcmp(cmp, opts->name) == 0)
5256 return __set_tracer_option(tr, trace->flags, opts, neg);
5262 /* Some tracers require overwrite to stay enabled */
5263 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
5265 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
5271 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
5275 if ((mask == TRACE_ITER_RECORD_TGID) ||
5276 (mask == TRACE_ITER_RECORD_CMD))
5277 lockdep_assert_held(&event_mutex);
5279 /* do nothing if flag is already set */
5280 if (!!(tr->trace_flags & mask) == !!enabled)
5283 /* Give the tracer a chance to approve the change */
5284 if (tr->current_trace->flag_changed)
5285 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
5289 tr->trace_flags |= mask;
5291 tr->trace_flags &= ~mask;
5293 if (mask == TRACE_ITER_RECORD_CMD)
5294 trace_event_enable_cmd_record(enabled);
5296 if (mask == TRACE_ITER_RECORD_TGID) {
5298 tgid_map_max = pid_max;
5299 map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map),
5303 * Pairs with smp_load_acquire() in
5304 * trace_find_tgid_ptr() to ensure that if it observes
5305 * the tgid_map we just allocated then it also observes
5306 * the corresponding tgid_map_max value.
5308 smp_store_release(&tgid_map, map);
5311 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
5315 trace_event_enable_tgid_record(enabled);
5318 if (mask == TRACE_ITER_EVENT_FORK)
5319 trace_event_follow_fork(tr, enabled);
5321 if (mask == TRACE_ITER_FUNC_FORK)
5322 ftrace_pid_follow_fork(tr, enabled);
5324 if (mask == TRACE_ITER_OVERWRITE) {
5325 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
5326 #ifdef CONFIG_TRACER_MAX_TRACE
5327 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
5331 if (mask == TRACE_ITER_PRINTK) {
5332 trace_printk_start_stop_comm(enabled);
5333 trace_printk_control(enabled);
5339 int trace_set_options(struct trace_array *tr, char *option)
5344 size_t orig_len = strlen(option);
5347 cmp = strstrip(option);
5349 len = str_has_prefix(cmp, "no");
5355 mutex_lock(&event_mutex);
5356 mutex_lock(&trace_types_lock);
5358 ret = match_string(trace_options, -1, cmp);
5359 /* If no option could be set, test the specific tracer options */
5361 ret = set_tracer_option(tr, cmp, neg);
5363 ret = set_tracer_flag(tr, 1 << ret, !neg);
5365 mutex_unlock(&trace_types_lock);
5366 mutex_unlock(&event_mutex);
5369 * If the first trailing whitespace is replaced with '\0' by strstrip,
5370 * turn it back into a space.
5372 if (orig_len > strlen(option))
5373 option[strlen(option)] = ' ';
5378 static void __init apply_trace_boot_options(void)
5380 char *buf = trace_boot_options_buf;
5384 option = strsep(&buf, ",");
5390 trace_set_options(&global_trace, option);
5392 /* Put back the comma to allow this to be called again */
5399 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
5400 size_t cnt, loff_t *ppos)
5402 struct seq_file *m = filp->private_data;
5403 struct trace_array *tr = m->private;
5407 if (cnt >= sizeof(buf))
5410 if (copy_from_user(buf, ubuf, cnt))
5415 ret = trace_set_options(tr, buf);
5424 static int tracing_trace_options_open(struct inode *inode, struct file *file)
5426 struct trace_array *tr = inode->i_private;
5429 ret = tracing_check_open_get_tr(tr);
5433 ret = single_open(file, tracing_trace_options_show, inode->i_private);
5435 trace_array_put(tr);
5440 static const struct file_operations tracing_iter_fops = {
5441 .open = tracing_trace_options_open,
5443 .llseek = seq_lseek,
5444 .release = tracing_single_release_tr,
5445 .write = tracing_trace_options_write,
5448 static const char readme_msg[] =
5449 "tracing mini-HOWTO:\n\n"
5450 "# echo 0 > tracing_on : quick way to disable tracing\n"
5451 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5452 " Important files:\n"
5453 " trace\t\t\t- The static contents of the buffer\n"
5454 "\t\t\t To clear the buffer write into this file: echo > trace\n"
5455 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5456 " current_tracer\t- function and latency tracers\n"
5457 " available_tracers\t- list of configured tracers for current_tracer\n"
5458 " error_log\t- error log for failed commands (that support it)\n"
5459 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
5460 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
5461 " trace_clock\t\t-change the clock used to order events\n"
5462 " local: Per cpu clock but may not be synced across CPUs\n"
5463 " global: Synced across CPUs but slows tracing down.\n"
5464 " counter: Not a clock, but just an increment\n"
5465 " uptime: Jiffy counter from time of boot\n"
5466 " perf: Same clock that perf events use\n"
5467 #ifdef CONFIG_X86_64
5468 " x86-tsc: TSC cycle counter\n"
5470 "\n timestamp_mode\t-view the mode used to timestamp events\n"
5471 " delta: Delta difference against a buffer-wide timestamp\n"
5472 " absolute: Absolute (standalone) timestamp\n"
5473 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5474 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5475 " tracing_cpumask\t- Limit which CPUs to trace\n"
5476 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5477 "\t\t\t Remove sub-buffer with rmdir\n"
5478 " trace_options\t\t- Set format or modify how tracing happens\n"
5479 "\t\t\t Disable an option by prefixing 'no' to the\n"
5480 "\t\t\t option name\n"
5481 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5482 #ifdef CONFIG_DYNAMIC_FTRACE
5483 "\n available_filter_functions - list of functions that can be filtered on\n"
5484 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5485 "\t\t\t functions\n"
5486 "\t accepts: func_full_name or glob-matching-pattern\n"
5487 "\t modules: Can select a group via module\n"
5488 "\t Format: :mod:<module-name>\n"
5489 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
5490 "\t triggers: a command to perform when function is hit\n"
5491 "\t Format: <function>:<trigger>[:count]\n"
5492 "\t trigger: traceon, traceoff\n"
5493 "\t\t enable_event:<system>:<event>\n"
5494 "\t\t disable_event:<system>:<event>\n"
5495 #ifdef CONFIG_STACKTRACE
5498 #ifdef CONFIG_TRACER_SNAPSHOT
5503 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5504 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5505 "\t The first one will disable tracing every time do_fault is hit\n"
5506 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
5507 "\t The first time do trap is hit and it disables tracing, the\n"
5508 "\t counter will decrement to 2. If tracing is already disabled,\n"
5509 "\t the counter will not decrement. It only decrements when the\n"
5510 "\t trigger did work\n"
5511 "\t To remove trigger without count:\n"
5512 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5513 "\t To remove trigger with a count:\n"
5514 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5515 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5516 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5517 "\t modules: Can select a group via module command :mod:\n"
5518 "\t Does not accept triggers\n"
5519 #endif /* CONFIG_DYNAMIC_FTRACE */
5520 #ifdef CONFIG_FUNCTION_TRACER
5521 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5523 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5526 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5527 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5528 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5529 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5531 #ifdef CONFIG_TRACER_SNAPSHOT
5532 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5533 "\t\t\t snapshot buffer. Read the contents for more\n"
5534 "\t\t\t information\n"
5536 #ifdef CONFIG_STACK_TRACER
5537 " stack_trace\t\t- Shows the max stack trace when active\n"
5538 " stack_max_size\t- Shows current max stack size that was traced\n"
5539 "\t\t\t Write into this file to reset the max size (trigger a\n"
5540 "\t\t\t new trace)\n"
5541 #ifdef CONFIG_DYNAMIC_FTRACE
5542 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5545 #endif /* CONFIG_STACK_TRACER */
5546 #ifdef CONFIG_DYNAMIC_EVENTS
5547 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5548 "\t\t\t Write into this file to define/undefine new trace events.\n"
5550 #ifdef CONFIG_KPROBE_EVENTS
5551 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5552 "\t\t\t Write into this file to define/undefine new trace events.\n"
5554 #ifdef CONFIG_UPROBE_EVENTS
5555 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5556 "\t\t\t Write into this file to define/undefine new trace events.\n"
5558 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5559 "\t accepts: event-definitions (one definition per line)\n"
5560 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
5561 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
5562 #ifdef CONFIG_HIST_TRIGGERS
5563 "\t s:[synthetic/]<event> <field> [<field>]\n"
5565 "\t e[:[<group>/]<event>] <attached-group>.<attached-event> [<args>]\n"
5566 "\t -:[<group>/]<event>\n"
5567 #ifdef CONFIG_KPROBE_EVENTS
5568 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5569 "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
5571 #ifdef CONFIG_UPROBE_EVENTS
5572 " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
5574 "\t args: <name>=fetcharg[:type]\n"
5575 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
5576 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5577 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5579 "\t $stack<index>, $stack, $retval, $comm,\n"
5581 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5582 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
5583 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5584 "\t <type>\\[<array-size>\\]\n"
5585 #ifdef CONFIG_HIST_TRIGGERS
5586 "\t field: <stype> <name>;\n"
5587 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5588 "\t [unsigned] char/int/long\n"
5590 "\t efield: For event probes ('e' types), the field is on of the fields\n"
5591 "\t of the <attached-group>/<attached-event>.\n"
5593 " events/\t\t- Directory containing all trace event subsystems:\n"
5594 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5595 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5596 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5598 " filter\t\t- If set, only events passing filter are traced\n"
5599 " events/<system>/<event>/\t- Directory containing control files for\n"
5601 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5602 " filter\t\t- If set, only events passing filter are traced\n"
5603 " trigger\t\t- If set, a command to perform when event is hit\n"
5604 "\t Format: <trigger>[:count][if <filter>]\n"
5605 "\t trigger: traceon, traceoff\n"
5606 "\t enable_event:<system>:<event>\n"
5607 "\t disable_event:<system>:<event>\n"
5608 #ifdef CONFIG_HIST_TRIGGERS
5609 "\t enable_hist:<system>:<event>\n"
5610 "\t disable_hist:<system>:<event>\n"
5612 #ifdef CONFIG_STACKTRACE
5615 #ifdef CONFIG_TRACER_SNAPSHOT
5618 #ifdef CONFIG_HIST_TRIGGERS
5619 "\t\t hist (see below)\n"
5621 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
5622 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5623 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5624 "\t events/block/block_unplug/trigger\n"
5625 "\t The first disables tracing every time block_unplug is hit.\n"
5626 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
5627 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
5628 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5629 "\t Like function triggers, the counter is only decremented if it\n"
5630 "\t enabled or disabled tracing.\n"
5631 "\t To remove a trigger without a count:\n"
5632 "\t echo '!<trigger> > <system>/<event>/trigger\n"
5633 "\t To remove a trigger with a count:\n"
5634 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5635 "\t Filters can be ignored when removing a trigger.\n"
5636 #ifdef CONFIG_HIST_TRIGGERS
5637 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5638 "\t Format: hist:keys=<field1[,field2,...]>\n"
5639 "\t [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n"
5640 "\t [:values=<field1[,field2,...]>]\n"
5641 "\t [:sort=<field1[,field2,...]>]\n"
5642 "\t [:size=#entries]\n"
5643 "\t [:pause][:continue][:clear]\n"
5644 "\t [:name=histname1]\n"
5645 "\t [:<handler>.<action>]\n"
5646 "\t [if <filter>]\n\n"
5647 "\t Note, special fields can be used as well:\n"
5648 "\t common_timestamp - to record current timestamp\n"
5649 "\t common_cpu - to record the CPU the event happened on\n"
5651 "\t A hist trigger variable can be:\n"
5652 "\t - a reference to a field e.g. x=current_timestamp,\n"
5653 "\t - a reference to another variable e.g. y=$x,\n"
5654 "\t - a numeric literal: e.g. ms_per_sec=1000,\n"
5655 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5657 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5658 "\t multiplication(*) and division(/) operators. An operand can be either a\n"
5659 "\t variable reference, field or numeric literal.\n"
5661 "\t When a matching event is hit, an entry is added to a hash\n"
5662 "\t table using the key(s) and value(s) named, and the value of a\n"
5663 "\t sum called 'hitcount' is incremented. Keys and values\n"
5664 "\t correspond to fields in the event's format description. Keys\n"
5665 "\t can be any field, or the special string 'stacktrace'.\n"
5666 "\t Compound keys consisting of up to two fields can be specified\n"
5667 "\t by the 'keys' keyword. Values must correspond to numeric\n"
5668 "\t fields. Sort keys consisting of up to two fields can be\n"
5669 "\t specified using the 'sort' keyword. The sort direction can\n"
5670 "\t be modified by appending '.descending' or '.ascending' to a\n"
5671 "\t sort field. The 'size' parameter can be used to specify more\n"
5672 "\t or fewer than the default 2048 entries for the hashtable size.\n"
5673 "\t If a hist trigger is given a name using the 'name' parameter,\n"
5674 "\t its histogram data will be shared with other triggers of the\n"
5675 "\t same name, and trigger hits will update this common data.\n\n"
5676 "\t Reading the 'hist' file for the event will dump the hash\n"
5677 "\t table in its entirety to stdout. If there are multiple hist\n"
5678 "\t triggers attached to an event, there will be a table for each\n"
5679 "\t trigger in the output. The table displayed for a named\n"
5680 "\t trigger will be the same as any other instance having the\n"
5681 "\t same name. The default format used to display a given field\n"
5682 "\t can be modified by appending any of the following modifiers\n"
5683 "\t to the field name, as applicable:\n\n"
5684 "\t .hex display a number as a hex value\n"
5685 "\t .sym display an address as a symbol\n"
5686 "\t .sym-offset display an address as a symbol and offset\n"
5687 "\t .execname display a common_pid as a program name\n"
5688 "\t .syscall display a syscall id as a syscall name\n"
5689 "\t .log2 display log2 value rather than raw number\n"
5690 "\t .buckets=size display values in groups of size rather than raw number\n"
5691 "\t .usecs display a common_timestamp in microseconds\n\n"
5692 "\t The 'pause' parameter can be used to pause an existing hist\n"
5693 "\t trigger or to start a hist trigger but not log any events\n"
5694 "\t until told to do so. 'continue' can be used to start or\n"
5695 "\t restart a paused hist trigger.\n\n"
5696 "\t The 'clear' parameter will clear the contents of a running\n"
5697 "\t hist trigger and leave its current paused/active state\n"
5699 "\t The enable_hist and disable_hist triggers can be used to\n"
5700 "\t have one event conditionally start and stop another event's\n"
5701 "\t already-attached hist trigger. The syntax is analogous to\n"
5702 "\t the enable_event and disable_event triggers.\n\n"
5703 "\t Hist trigger handlers and actions are executed whenever a\n"
5704 "\t a histogram entry is added or updated. They take the form:\n\n"
5705 "\t <handler>.<action>\n\n"
5706 "\t The available handlers are:\n\n"
5707 "\t onmatch(matching.event) - invoke on addition or update\n"
5708 "\t onmax(var) - invoke if var exceeds current max\n"
5709 "\t onchange(var) - invoke action if var changes\n\n"
5710 "\t The available actions are:\n\n"
5711 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5712 "\t save(field,...) - save current event fields\n"
5713 #ifdef CONFIG_TRACER_SNAPSHOT
5714 "\t snapshot() - snapshot the trace buffer\n\n"
5716 #ifdef CONFIG_SYNTH_EVENTS
5717 " events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5718 "\t Write into this file to define/undefine new synthetic events.\n"
5719 "\t example: echo 'myevent u64 lat; char name[]' >> synthetic_events\n"
5725 tracing_readme_read(struct file *filp, char __user *ubuf,
5726 size_t cnt, loff_t *ppos)
5728 return simple_read_from_buffer(ubuf, cnt, ppos,
5729 readme_msg, strlen(readme_msg));
5732 static const struct file_operations tracing_readme_fops = {
5733 .open = tracing_open_generic,
5734 .read = tracing_readme_read,
5735 .llseek = generic_file_llseek,
5738 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5742 return trace_find_tgid_ptr(pid);
5745 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5749 return trace_find_tgid_ptr(pid);
5752 static void saved_tgids_stop(struct seq_file *m, void *v)
5756 static int saved_tgids_show(struct seq_file *m, void *v)
5758 int *entry = (int *)v;
5759 int pid = entry - tgid_map;
5765 seq_printf(m, "%d %d\n", pid, tgid);
5769 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5770 .start = saved_tgids_start,
5771 .stop = saved_tgids_stop,
5772 .next = saved_tgids_next,
5773 .show = saved_tgids_show,
5776 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5780 ret = tracing_check_open_get_tr(NULL);
5784 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5788 static const struct file_operations tracing_saved_tgids_fops = {
5789 .open = tracing_saved_tgids_open,
5791 .llseek = seq_lseek,
5792 .release = seq_release,
5795 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5797 unsigned int *ptr = v;
5799 if (*pos || m->count)
5804 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5806 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5815 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5821 arch_spin_lock(&trace_cmdline_lock);
5823 v = &savedcmd->map_cmdline_to_pid[0];
5825 v = saved_cmdlines_next(m, v, &l);
5833 static void saved_cmdlines_stop(struct seq_file *m, void *v)
5835 arch_spin_unlock(&trace_cmdline_lock);
5839 static int saved_cmdlines_show(struct seq_file *m, void *v)
5841 char buf[TASK_COMM_LEN];
5842 unsigned int *pid = v;
5844 __trace_find_cmdline(*pid, buf);
5845 seq_printf(m, "%d %s\n", *pid, buf);
5849 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5850 .start = saved_cmdlines_start,
5851 .next = saved_cmdlines_next,
5852 .stop = saved_cmdlines_stop,
5853 .show = saved_cmdlines_show,
5856 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5860 ret = tracing_check_open_get_tr(NULL);
5864 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5867 static const struct file_operations tracing_saved_cmdlines_fops = {
5868 .open = tracing_saved_cmdlines_open,
5870 .llseek = seq_lseek,
5871 .release = seq_release,
5875 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5876 size_t cnt, loff_t *ppos)
5881 arch_spin_lock(&trace_cmdline_lock);
5882 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5883 arch_spin_unlock(&trace_cmdline_lock);
5885 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5888 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5890 kfree(s->saved_cmdlines);
5891 kfree(s->map_cmdline_to_pid);
5895 static int tracing_resize_saved_cmdlines(unsigned int val)
5897 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5899 s = kmalloc(sizeof(*s), GFP_KERNEL);
5903 if (allocate_cmdlines_buffer(val, s) < 0) {
5908 arch_spin_lock(&trace_cmdline_lock);
5909 savedcmd_temp = savedcmd;
5911 arch_spin_unlock(&trace_cmdline_lock);
5912 free_saved_cmdlines_buffer(savedcmd_temp);
5918 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5919 size_t cnt, loff_t *ppos)
5924 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5928 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5929 if (!val || val > PID_MAX_DEFAULT)
5932 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5941 static const struct file_operations tracing_saved_cmdlines_size_fops = {
5942 .open = tracing_open_generic,
5943 .read = tracing_saved_cmdlines_size_read,
5944 .write = tracing_saved_cmdlines_size_write,
5947 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5948 static union trace_eval_map_item *
5949 update_eval_map(union trace_eval_map_item *ptr)
5951 if (!ptr->map.eval_string) {
5952 if (ptr->tail.next) {
5953 ptr = ptr->tail.next;
5954 /* Set ptr to the next real item (skip head) */
5962 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5964 union trace_eval_map_item *ptr = v;
5967 * Paranoid! If ptr points to end, we don't want to increment past it.
5968 * This really should never happen.
5971 ptr = update_eval_map(ptr);
5972 if (WARN_ON_ONCE(!ptr))
5976 ptr = update_eval_map(ptr);
5981 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5983 union trace_eval_map_item *v;
5986 mutex_lock(&trace_eval_mutex);
5988 v = trace_eval_maps;
5992 while (v && l < *pos) {
5993 v = eval_map_next(m, v, &l);
5999 static void eval_map_stop(struct seq_file *m, void *v)
6001 mutex_unlock(&trace_eval_mutex);
6004 static int eval_map_show(struct seq_file *m, void *v)
6006 union trace_eval_map_item *ptr = v;
6008 seq_printf(m, "%s %ld (%s)\n",
6009 ptr->map.eval_string, ptr->map.eval_value,
6015 static const struct seq_operations tracing_eval_map_seq_ops = {
6016 .start = eval_map_start,
6017 .next = eval_map_next,
6018 .stop = eval_map_stop,
6019 .show = eval_map_show,
6022 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
6026 ret = tracing_check_open_get_tr(NULL);
6030 return seq_open(filp, &tracing_eval_map_seq_ops);
6033 static const struct file_operations tracing_eval_map_fops = {
6034 .open = tracing_eval_map_open,
6036 .llseek = seq_lseek,
6037 .release = seq_release,
6040 static inline union trace_eval_map_item *
6041 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
6043 /* Return tail of array given the head */
6044 return ptr + ptr->head.length + 1;
6048 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
6051 struct trace_eval_map **stop;
6052 struct trace_eval_map **map;
6053 union trace_eval_map_item *map_array;
6054 union trace_eval_map_item *ptr;
6059 * The trace_eval_maps contains the map plus a head and tail item,
6060 * where the head holds the module and length of array, and the
6061 * tail holds a pointer to the next list.
6063 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
6065 pr_warn("Unable to allocate trace eval mapping\n");
6069 mutex_lock(&trace_eval_mutex);
6071 if (!trace_eval_maps)
6072 trace_eval_maps = map_array;
6074 ptr = trace_eval_maps;
6076 ptr = trace_eval_jmp_to_tail(ptr);
6077 if (!ptr->tail.next)
6079 ptr = ptr->tail.next;
6082 ptr->tail.next = map_array;
6084 map_array->head.mod = mod;
6085 map_array->head.length = len;
6088 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
6089 map_array->map = **map;
6092 memset(map_array, 0, sizeof(*map_array));
6094 mutex_unlock(&trace_eval_mutex);
6097 static void trace_create_eval_file(struct dentry *d_tracer)
6099 trace_create_file("eval_map", TRACE_MODE_READ, d_tracer,
6100 NULL, &tracing_eval_map_fops);
6103 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
6104 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
6105 static inline void trace_insert_eval_map_file(struct module *mod,
6106 struct trace_eval_map **start, int len) { }
6107 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
6109 static void trace_insert_eval_map(struct module *mod,
6110 struct trace_eval_map **start, int len)
6112 struct trace_eval_map **map;
6119 trace_event_eval_update(map, len);
6121 trace_insert_eval_map_file(mod, start, len);
6125 tracing_set_trace_read(struct file *filp, char __user *ubuf,
6126 size_t cnt, loff_t *ppos)
6128 struct trace_array *tr = filp->private_data;
6129 char buf[MAX_TRACER_SIZE+2];
6132 mutex_lock(&trace_types_lock);
6133 r = sprintf(buf, "%s\n", tr->current_trace->name);
6134 mutex_unlock(&trace_types_lock);
6136 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6139 int tracer_init(struct tracer *t, struct trace_array *tr)
6141 tracing_reset_online_cpus(&tr->array_buffer);
6145 static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
6149 for_each_tracing_cpu(cpu)
6150 per_cpu_ptr(buf->data, cpu)->entries = val;
6153 #ifdef CONFIG_TRACER_MAX_TRACE
6154 /* resize @tr's buffer to the size of @size_tr's entries */
6155 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
6156 struct array_buffer *size_buf, int cpu_id)
6160 if (cpu_id == RING_BUFFER_ALL_CPUS) {
6161 for_each_tracing_cpu(cpu) {
6162 ret = ring_buffer_resize(trace_buf->buffer,
6163 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
6166 per_cpu_ptr(trace_buf->data, cpu)->entries =
6167 per_cpu_ptr(size_buf->data, cpu)->entries;
6170 ret = ring_buffer_resize(trace_buf->buffer,
6171 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
6173 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
6174 per_cpu_ptr(size_buf->data, cpu_id)->entries;
6179 #endif /* CONFIG_TRACER_MAX_TRACE */
6181 static int __tracing_resize_ring_buffer(struct trace_array *tr,
6182 unsigned long size, int cpu)
6187 * If kernel or user changes the size of the ring buffer
6188 * we use the size that was given, and we can forget about
6189 * expanding it later.
6191 ring_buffer_expanded = true;
6193 /* May be called before buffers are initialized */
6194 if (!tr->array_buffer.buffer)
6197 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
6201 #ifdef CONFIG_TRACER_MAX_TRACE
6202 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
6203 !tr->current_trace->use_max_tr)
6206 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
6208 int r = resize_buffer_duplicate_size(&tr->array_buffer,
6209 &tr->array_buffer, cpu);
6212 * AARGH! We are left with different
6213 * size max buffer!!!!
6214 * The max buffer is our "snapshot" buffer.
6215 * When a tracer needs a snapshot (one of the
6216 * latency tracers), it swaps the max buffer
6217 * with the saved snap shot. We succeeded to
6218 * update the size of the main buffer, but failed to
6219 * update the size of the max buffer. But when we tried
6220 * to reset the main buffer to the original size, we
6221 * failed there too. This is very unlikely to
6222 * happen, but if it does, warn and kill all
6226 tracing_disabled = 1;
6231 if (cpu == RING_BUFFER_ALL_CPUS)
6232 set_buffer_entries(&tr->max_buffer, size);
6234 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
6237 #endif /* CONFIG_TRACER_MAX_TRACE */
6239 if (cpu == RING_BUFFER_ALL_CPUS)
6240 set_buffer_entries(&tr->array_buffer, size);
6242 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
6247 ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
6248 unsigned long size, int cpu_id)
6252 mutex_lock(&trace_types_lock);
6254 if (cpu_id != RING_BUFFER_ALL_CPUS) {
6255 /* make sure, this cpu is enabled in the mask */
6256 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
6262 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
6267 mutex_unlock(&trace_types_lock);
6274 * tracing_update_buffers - used by tracing facility to expand ring buffers
6276 * To save on memory when the tracing is never used on a system with it
6277 * configured in. The ring buffers are set to a minimum size. But once
6278 * a user starts to use the tracing facility, then they need to grow
6279 * to their default size.
6281 * This function is to be called when a tracer is about to be used.
6283 int tracing_update_buffers(void)
6287 mutex_lock(&trace_types_lock);
6288 if (!ring_buffer_expanded)
6289 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
6290 RING_BUFFER_ALL_CPUS);
6291 mutex_unlock(&trace_types_lock);
6296 struct trace_option_dentry;
6299 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
6302 * Used to clear out the tracer before deletion of an instance.
6303 * Must have trace_types_lock held.
6305 static void tracing_set_nop(struct trace_array *tr)
6307 if (tr->current_trace == &nop_trace)
6310 tr->current_trace->enabled--;
6312 if (tr->current_trace->reset)
6313 tr->current_trace->reset(tr);
6315 tr->current_trace = &nop_trace;
6318 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
6320 /* Only enable if the directory has been created already. */
6324 create_trace_option_files(tr, t);
6327 int tracing_set_tracer(struct trace_array *tr, const char *buf)
6330 #ifdef CONFIG_TRACER_MAX_TRACE
6335 mutex_lock(&trace_types_lock);
6337 if (!ring_buffer_expanded) {
6338 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
6339 RING_BUFFER_ALL_CPUS);
6345 for (t = trace_types; t; t = t->next) {
6346 if (strcmp(t->name, buf) == 0)
6353 if (t == tr->current_trace)
6356 #ifdef CONFIG_TRACER_SNAPSHOT
6357 if (t->use_max_tr) {
6358 arch_spin_lock(&tr->max_lock);
6359 if (tr->cond_snapshot)
6361 arch_spin_unlock(&tr->max_lock);
6366 /* Some tracers won't work on kernel command line */
6367 if (system_state < SYSTEM_RUNNING && t->noboot) {
6368 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
6373 /* Some tracers are only allowed for the top level buffer */
6374 if (!trace_ok_for_array(t, tr)) {
6379 /* If trace pipe files are being read, we can't change the tracer */
6380 if (tr->trace_ref) {
6385 trace_branch_disable();
6387 tr->current_trace->enabled--;
6389 if (tr->current_trace->reset)
6390 tr->current_trace->reset(tr);
6392 /* Current trace needs to be nop_trace before synchronize_rcu */
6393 tr->current_trace = &nop_trace;
6395 #ifdef CONFIG_TRACER_MAX_TRACE
6396 had_max_tr = tr->allocated_snapshot;
6398 if (had_max_tr && !t->use_max_tr) {
6400 * We need to make sure that the update_max_tr sees that
6401 * current_trace changed to nop_trace to keep it from
6402 * swapping the buffers after we resize it.
6403 * The update_max_tr is called from interrupts disabled
6404 * so a synchronized_sched() is sufficient.
6411 #ifdef CONFIG_TRACER_MAX_TRACE
6412 if (t->use_max_tr && !had_max_tr) {
6413 ret = tracing_alloc_snapshot_instance(tr);
6420 ret = tracer_init(t, tr);
6425 tr->current_trace = t;
6426 tr->current_trace->enabled++;
6427 trace_branch_enable(tr);
6429 mutex_unlock(&trace_types_lock);
6435 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
6436 size_t cnt, loff_t *ppos)
6438 struct trace_array *tr = filp->private_data;
6439 char buf[MAX_TRACER_SIZE+1];
6446 if (cnt > MAX_TRACER_SIZE)
6447 cnt = MAX_TRACER_SIZE;
6449 if (copy_from_user(buf, ubuf, cnt))
6454 /* strip ending whitespace. */
6455 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
6458 err = tracing_set_tracer(tr, buf);
6468 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
6469 size_t cnt, loff_t *ppos)
6474 r = snprintf(buf, sizeof(buf), "%ld\n",
6475 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
6476 if (r > sizeof(buf))
6478 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6482 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
6483 size_t cnt, loff_t *ppos)
6488 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6498 tracing_thresh_read(struct file *filp, char __user *ubuf,
6499 size_t cnt, loff_t *ppos)
6501 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6505 tracing_thresh_write(struct file *filp, const char __user *ubuf,
6506 size_t cnt, loff_t *ppos)
6508 struct trace_array *tr = filp->private_data;
6511 mutex_lock(&trace_types_lock);
6512 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6516 if (tr->current_trace->update_thresh) {
6517 ret = tr->current_trace->update_thresh(tr);
6524 mutex_unlock(&trace_types_lock);
6529 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6532 tracing_max_lat_read(struct file *filp, char __user *ubuf,
6533 size_t cnt, loff_t *ppos)
6535 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
6539 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6540 size_t cnt, loff_t *ppos)
6542 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
6547 static int tracing_open_pipe(struct inode *inode, struct file *filp)
6549 struct trace_array *tr = inode->i_private;
6550 struct trace_iterator *iter;
6553 ret = tracing_check_open_get_tr(tr);
6557 mutex_lock(&trace_types_lock);
6559 /* create a buffer to store the information to pass to userspace */
6560 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6563 __trace_array_put(tr);
6567 trace_seq_init(&iter->seq);
6568 iter->trace = tr->current_trace;
6570 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6575 /* trace pipe does not show start of buffer */
6576 cpumask_setall(iter->started);
6578 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6579 iter->iter_flags |= TRACE_FILE_LAT_FMT;
6581 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6582 if (trace_clocks[tr->clock_id].in_ns)
6583 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6586 iter->array_buffer = &tr->array_buffer;
6587 iter->cpu_file = tracing_get_cpu(inode);
6588 mutex_init(&iter->mutex);
6589 filp->private_data = iter;
6591 if (iter->trace->pipe_open)
6592 iter->trace->pipe_open(iter);
6594 nonseekable_open(inode, filp);
6598 mutex_unlock(&trace_types_lock);
6603 __trace_array_put(tr);
6604 mutex_unlock(&trace_types_lock);
6608 static int tracing_release_pipe(struct inode *inode, struct file *file)
6610 struct trace_iterator *iter = file->private_data;
6611 struct trace_array *tr = inode->i_private;
6613 mutex_lock(&trace_types_lock);
6617 if (iter->trace->pipe_close)
6618 iter->trace->pipe_close(iter);
6620 mutex_unlock(&trace_types_lock);
6622 free_cpumask_var(iter->started);
6623 mutex_destroy(&iter->mutex);
6626 trace_array_put(tr);
6632 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6634 struct trace_array *tr = iter->tr;
6636 /* Iterators are static, they should be filled or empty */
6637 if (trace_buffer_iter(iter, iter->cpu_file))
6638 return EPOLLIN | EPOLLRDNORM;
6640 if (tr->trace_flags & TRACE_ITER_BLOCK)
6642 * Always select as readable when in blocking mode
6644 return EPOLLIN | EPOLLRDNORM;
6646 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6651 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6653 struct trace_iterator *iter = filp->private_data;
6655 return trace_poll(iter, filp, poll_table);
6658 /* Must be called with iter->mutex held. */
6659 static int tracing_wait_pipe(struct file *filp)
6661 struct trace_iterator *iter = filp->private_data;
6664 while (trace_empty(iter)) {
6666 if ((filp->f_flags & O_NONBLOCK)) {
6671 * We block until we read something and tracing is disabled.
6672 * We still block if tracing is disabled, but we have never
6673 * read anything. This allows a user to cat this file, and
6674 * then enable tracing. But after we have read something,
6675 * we give an EOF when tracing is again disabled.
6677 * iter->pos will be 0 if we haven't read anything.
6679 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6682 mutex_unlock(&iter->mutex);
6684 ret = wait_on_pipe(iter, 0);
6686 mutex_lock(&iter->mutex);
6699 tracing_read_pipe(struct file *filp, char __user *ubuf,
6700 size_t cnt, loff_t *ppos)
6702 struct trace_iterator *iter = filp->private_data;
6706 * Avoid more than one consumer on a single file descriptor
6707 * This is just a matter of traces coherency, the ring buffer itself
6710 mutex_lock(&iter->mutex);
6712 /* return any leftover data */
6713 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6717 trace_seq_init(&iter->seq);
6719 if (iter->trace->read) {
6720 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6726 sret = tracing_wait_pipe(filp);
6730 /* stop when tracing is finished */
6731 if (trace_empty(iter)) {
6736 if (cnt >= PAGE_SIZE)
6737 cnt = PAGE_SIZE - 1;
6739 /* reset all but tr, trace, and overruns */
6740 trace_iterator_reset(iter);
6741 cpumask_clear(iter->started);
6742 trace_seq_init(&iter->seq);
6744 trace_event_read_lock();
6745 trace_access_lock(iter->cpu_file);
6746 while (trace_find_next_entry_inc(iter) != NULL) {
6747 enum print_line_t ret;
6748 int save_len = iter->seq.seq.len;
6750 ret = print_trace_line(iter);
6751 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6752 /* don't print partial lines */
6753 iter->seq.seq.len = save_len;
6756 if (ret != TRACE_TYPE_NO_CONSUME)
6757 trace_consume(iter);
6759 if (trace_seq_used(&iter->seq) >= cnt)
6763 * Setting the full flag means we reached the trace_seq buffer
6764 * size and we should leave by partial output condition above.
6765 * One of the trace_seq_* functions is not used properly.
6767 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6770 trace_access_unlock(iter->cpu_file);
6771 trace_event_read_unlock();
6773 /* Now copy what we have to the user */
6774 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6775 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6776 trace_seq_init(&iter->seq);
6779 * If there was nothing to send to user, in spite of consuming trace
6780 * entries, go back to wait for more entries.
6786 mutex_unlock(&iter->mutex);
6791 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6794 __free_page(spd->pages[idx]);
6798 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6804 /* Seq buffer is page-sized, exactly what we need. */
6806 save_len = iter->seq.seq.len;
6807 ret = print_trace_line(iter);
6809 if (trace_seq_has_overflowed(&iter->seq)) {
6810 iter->seq.seq.len = save_len;
6815 * This should not be hit, because it should only
6816 * be set if the iter->seq overflowed. But check it
6817 * anyway to be safe.
6819 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6820 iter->seq.seq.len = save_len;
6824 count = trace_seq_used(&iter->seq) - save_len;
6827 iter->seq.seq.len = save_len;
6831 if (ret != TRACE_TYPE_NO_CONSUME)
6832 trace_consume(iter);
6834 if (!trace_find_next_entry_inc(iter)) {
6844 static ssize_t tracing_splice_read_pipe(struct file *filp,
6846 struct pipe_inode_info *pipe,
6850 struct page *pages_def[PIPE_DEF_BUFFERS];
6851 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6852 struct trace_iterator *iter = filp->private_data;
6853 struct splice_pipe_desc spd = {
6855 .partial = partial_def,
6856 .nr_pages = 0, /* This gets updated below. */
6857 .nr_pages_max = PIPE_DEF_BUFFERS,
6858 .ops = &default_pipe_buf_ops,
6859 .spd_release = tracing_spd_release_pipe,
6865 if (splice_grow_spd(pipe, &spd))
6868 mutex_lock(&iter->mutex);
6870 if (iter->trace->splice_read) {
6871 ret = iter->trace->splice_read(iter, filp,
6872 ppos, pipe, len, flags);
6877 ret = tracing_wait_pipe(filp);
6881 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6886 trace_event_read_lock();
6887 trace_access_lock(iter->cpu_file);
6889 /* Fill as many pages as possible. */
6890 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6891 spd.pages[i] = alloc_page(GFP_KERNEL);
6895 rem = tracing_fill_pipe_page(rem, iter);
6897 /* Copy the data into the page, so we can start over. */
6898 ret = trace_seq_to_buffer(&iter->seq,
6899 page_address(spd.pages[i]),
6900 trace_seq_used(&iter->seq));
6902 __free_page(spd.pages[i]);
6905 spd.partial[i].offset = 0;
6906 spd.partial[i].len = trace_seq_used(&iter->seq);
6908 trace_seq_init(&iter->seq);
6911 trace_access_unlock(iter->cpu_file);
6912 trace_event_read_unlock();
6913 mutex_unlock(&iter->mutex);
6918 ret = splice_to_pipe(pipe, &spd);
6922 splice_shrink_spd(&spd);
6926 mutex_unlock(&iter->mutex);
6931 tracing_entries_read(struct file *filp, char __user *ubuf,
6932 size_t cnt, loff_t *ppos)
6934 struct inode *inode = file_inode(filp);
6935 struct trace_array *tr = inode->i_private;
6936 int cpu = tracing_get_cpu(inode);
6941 mutex_lock(&trace_types_lock);
6943 if (cpu == RING_BUFFER_ALL_CPUS) {
6944 int cpu, buf_size_same;
6949 /* check if all cpu sizes are same */
6950 for_each_tracing_cpu(cpu) {
6951 /* fill in the size from first enabled cpu */
6953 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6954 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6960 if (buf_size_same) {
6961 if (!ring_buffer_expanded)
6962 r = sprintf(buf, "%lu (expanded: %lu)\n",
6964 trace_buf_size >> 10);
6966 r = sprintf(buf, "%lu\n", size >> 10);
6968 r = sprintf(buf, "X\n");
6970 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6972 mutex_unlock(&trace_types_lock);
6974 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6979 tracing_entries_write(struct file *filp, const char __user *ubuf,
6980 size_t cnt, loff_t *ppos)
6982 struct inode *inode = file_inode(filp);
6983 struct trace_array *tr = inode->i_private;
6987 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6991 /* must have at least 1 entry */
6995 /* value is in KB */
6997 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
7007 tracing_total_entries_read(struct file *filp, char __user *ubuf,
7008 size_t cnt, loff_t *ppos)
7010 struct trace_array *tr = filp->private_data;
7013 unsigned long size = 0, expanded_size = 0;
7015 mutex_lock(&trace_types_lock);
7016 for_each_tracing_cpu(cpu) {
7017 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
7018 if (!ring_buffer_expanded)
7019 expanded_size += trace_buf_size >> 10;
7021 if (ring_buffer_expanded)
7022 r = sprintf(buf, "%lu\n", size);
7024 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
7025 mutex_unlock(&trace_types_lock);
7027 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7031 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
7032 size_t cnt, loff_t *ppos)
7035 * There is no need to read what the user has written, this function
7036 * is just to make sure that there is no error when "echo" is used
7045 tracing_free_buffer_release(struct inode *inode, struct file *filp)
7047 struct trace_array *tr = inode->i_private;
7049 /* disable tracing ? */
7050 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
7051 tracer_tracing_off(tr);
7052 /* resize the ring buffer to 0 */
7053 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
7055 trace_array_put(tr);
7061 tracing_mark_write(struct file *filp, const char __user *ubuf,
7062 size_t cnt, loff_t *fpos)
7064 struct trace_array *tr = filp->private_data;
7065 struct ring_buffer_event *event;
7066 enum event_trigger_type tt = ETT_NONE;
7067 struct trace_buffer *buffer;
7068 struct print_entry *entry;
7073 /* Used in tracing_mark_raw_write() as well */
7074 #define FAULTED_STR "<faulted>"
7075 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
7077 if (tracing_disabled)
7080 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7083 if (cnt > TRACE_BUF_SIZE)
7084 cnt = TRACE_BUF_SIZE;
7086 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7088 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
7090 /* If less than "<faulted>", then make sure we can still add that */
7091 if (cnt < FAULTED_SIZE)
7092 size += FAULTED_SIZE - cnt;
7094 buffer = tr->array_buffer.buffer;
7095 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
7097 if (unlikely(!event))
7098 /* Ring buffer disabled, return as if not open for write */
7101 entry = ring_buffer_event_data(event);
7102 entry->ip = _THIS_IP_;
7104 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
7106 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7112 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
7113 /* do not add \n before testing triggers, but add \0 */
7114 entry->buf[cnt] = '\0';
7115 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);
7118 if (entry->buf[cnt - 1] != '\n') {
7119 entry->buf[cnt] = '\n';
7120 entry->buf[cnt + 1] = '\0';
7122 entry->buf[cnt] = '\0';
7124 if (static_branch_unlikely(&trace_marker_exports_enabled))
7125 ftrace_exports(event, TRACE_EXPORT_MARKER);
7126 __buffer_unlock_commit(buffer, event);
7129 event_triggers_post_call(tr->trace_marker_file, tt);
7134 /* Limit it for now to 3K (including tag) */
7135 #define RAW_DATA_MAX_SIZE (1024*3)
7138 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
7139 size_t cnt, loff_t *fpos)
7141 struct trace_array *tr = filp->private_data;
7142 struct ring_buffer_event *event;
7143 struct trace_buffer *buffer;
7144 struct raw_data_entry *entry;
7149 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
7151 if (tracing_disabled)
7154 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
7157 /* The marker must at least have a tag id */
7158 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
7161 if (cnt > TRACE_BUF_SIZE)
7162 cnt = TRACE_BUF_SIZE;
7164 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
7166 size = sizeof(*entry) + cnt;
7167 if (cnt < FAULT_SIZE_ID)
7168 size += FAULT_SIZE_ID - cnt;
7170 buffer = tr->array_buffer.buffer;
7171 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
7174 /* Ring buffer disabled, return as if not open for write */
7177 entry = ring_buffer_event_data(event);
7179 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
7182 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7187 __buffer_unlock_commit(buffer, event);
7192 static int tracing_clock_show(struct seq_file *m, void *v)
7194 struct trace_array *tr = m->private;
7197 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
7199 "%s%s%s%s", i ? " " : "",
7200 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
7201 i == tr->clock_id ? "]" : "");
7207 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
7211 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
7212 if (strcmp(trace_clocks[i].name, clockstr) == 0)
7215 if (i == ARRAY_SIZE(trace_clocks))
7218 mutex_lock(&trace_types_lock);
7222 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
7225 * New clock may not be consistent with the previous clock.
7226 * Reset the buffer so that it doesn't have incomparable timestamps.
7228 tracing_reset_online_cpus(&tr->array_buffer);
7230 #ifdef CONFIG_TRACER_MAX_TRACE
7231 if (tr->max_buffer.buffer)
7232 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
7233 tracing_reset_online_cpus(&tr->max_buffer);
7236 mutex_unlock(&trace_types_lock);
7241 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
7242 size_t cnt, loff_t *fpos)
7244 struct seq_file *m = filp->private_data;
7245 struct trace_array *tr = m->private;
7247 const char *clockstr;
7250 if (cnt >= sizeof(buf))
7253 if (copy_from_user(buf, ubuf, cnt))
7258 clockstr = strstrip(buf);
7260 ret = tracing_set_clock(tr, clockstr);
7269 static int tracing_clock_open(struct inode *inode, struct file *file)
7271 struct trace_array *tr = inode->i_private;
7274 ret = tracing_check_open_get_tr(tr);
7278 ret = single_open(file, tracing_clock_show, inode->i_private);
7280 trace_array_put(tr);
7285 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
7287 struct trace_array *tr = m->private;
7289 mutex_lock(&trace_types_lock);
7291 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
7292 seq_puts(m, "delta [absolute]\n");
7294 seq_puts(m, "[delta] absolute\n");
7296 mutex_unlock(&trace_types_lock);
7301 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
7303 struct trace_array *tr = inode->i_private;
7306 ret = tracing_check_open_get_tr(tr);
7310 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
7312 trace_array_put(tr);
7317 u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe)
7319 if (rbe == this_cpu_read(trace_buffered_event))
7320 return ring_buffer_time_stamp(buffer);
7322 return ring_buffer_event_time_stamp(buffer, rbe);
7326 * Set or disable using the per CPU trace_buffer_event when possible.
7328 int tracing_set_filter_buffering(struct trace_array *tr, bool set)
7332 mutex_lock(&trace_types_lock);
7334 if (set && tr->no_filter_buffering_ref++)
7338 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) {
7343 --tr->no_filter_buffering_ref;
7346 mutex_unlock(&trace_types_lock);
7351 struct ftrace_buffer_info {
7352 struct trace_iterator iter;
7354 unsigned int spare_cpu;
7358 #ifdef CONFIG_TRACER_SNAPSHOT
7359 static int tracing_snapshot_open(struct inode *inode, struct file *file)
7361 struct trace_array *tr = inode->i_private;
7362 struct trace_iterator *iter;
7366 ret = tracing_check_open_get_tr(tr);
7370 if (file->f_mode & FMODE_READ) {
7371 iter = __tracing_open(inode, file, true);
7373 ret = PTR_ERR(iter);
7375 /* Writes still need the seq_file to hold the private data */
7377 m = kzalloc(sizeof(*m), GFP_KERNEL);
7380 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
7388 iter->array_buffer = &tr->max_buffer;
7389 iter->cpu_file = tracing_get_cpu(inode);
7391 file->private_data = m;
7395 trace_array_put(tr);
7401 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
7404 struct seq_file *m = filp->private_data;
7405 struct trace_iterator *iter = m->private;
7406 struct trace_array *tr = iter->tr;
7410 ret = tracing_update_buffers();
7414 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7418 mutex_lock(&trace_types_lock);
7420 if (tr->current_trace->use_max_tr) {
7425 arch_spin_lock(&tr->max_lock);
7426 if (tr->cond_snapshot)
7428 arch_spin_unlock(&tr->max_lock);
7434 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7438 if (tr->allocated_snapshot)
7442 /* Only allow per-cpu swap if the ring buffer supports it */
7443 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
7444 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7449 if (tr->allocated_snapshot)
7450 ret = resize_buffer_duplicate_size(&tr->max_buffer,
7451 &tr->array_buffer, iter->cpu_file);
7453 ret = tracing_alloc_snapshot_instance(tr);
7456 local_irq_disable();
7457 /* Now, we're going to swap */
7458 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7459 update_max_tr(tr, current, smp_processor_id(), NULL);
7461 update_max_tr_single(tr, current, iter->cpu_file);
7465 if (tr->allocated_snapshot) {
7466 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7467 tracing_reset_online_cpus(&tr->max_buffer);
7469 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7479 mutex_unlock(&trace_types_lock);
7483 static int tracing_snapshot_release(struct inode *inode, struct file *file)
7485 struct seq_file *m = file->private_data;
7488 ret = tracing_release(inode, file);
7490 if (file->f_mode & FMODE_READ)
7493 /* If write only, the seq_file is just a stub */
7501 static int tracing_buffers_open(struct inode *inode, struct file *filp);
7502 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7503 size_t count, loff_t *ppos);
7504 static int tracing_buffers_release(struct inode *inode, struct file *file);
7505 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7506 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7508 static int snapshot_raw_open(struct inode *inode, struct file *filp)
7510 struct ftrace_buffer_info *info;
7513 /* The following checks for tracefs lockdown */
7514 ret = tracing_buffers_open(inode, filp);
7518 info = filp->private_data;
7520 if (info->iter.trace->use_max_tr) {
7521 tracing_buffers_release(inode, filp);
7525 info->iter.snapshot = true;
7526 info->iter.array_buffer = &info->iter.tr->max_buffer;
7531 #endif /* CONFIG_TRACER_SNAPSHOT */
7534 static const struct file_operations tracing_thresh_fops = {
7535 .open = tracing_open_generic,
7536 .read = tracing_thresh_read,
7537 .write = tracing_thresh_write,
7538 .llseek = generic_file_llseek,
7541 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7542 static const struct file_operations tracing_max_lat_fops = {
7543 .open = tracing_open_generic,
7544 .read = tracing_max_lat_read,
7545 .write = tracing_max_lat_write,
7546 .llseek = generic_file_llseek,
7550 static const struct file_operations set_tracer_fops = {
7551 .open = tracing_open_generic,
7552 .read = tracing_set_trace_read,
7553 .write = tracing_set_trace_write,
7554 .llseek = generic_file_llseek,
7557 static const struct file_operations tracing_pipe_fops = {
7558 .open = tracing_open_pipe,
7559 .poll = tracing_poll_pipe,
7560 .read = tracing_read_pipe,
7561 .splice_read = tracing_splice_read_pipe,
7562 .release = tracing_release_pipe,
7563 .llseek = no_llseek,
7566 static const struct file_operations tracing_entries_fops = {
7567 .open = tracing_open_generic_tr,
7568 .read = tracing_entries_read,
7569 .write = tracing_entries_write,
7570 .llseek = generic_file_llseek,
7571 .release = tracing_release_generic_tr,
7574 static const struct file_operations tracing_total_entries_fops = {
7575 .open = tracing_open_generic_tr,
7576 .read = tracing_total_entries_read,
7577 .llseek = generic_file_llseek,
7578 .release = tracing_release_generic_tr,
7581 static const struct file_operations tracing_free_buffer_fops = {
7582 .open = tracing_open_generic_tr,
7583 .write = tracing_free_buffer_write,
7584 .release = tracing_free_buffer_release,
7587 static const struct file_operations tracing_mark_fops = {
7588 .open = tracing_mark_open,
7589 .write = tracing_mark_write,
7590 .release = tracing_release_generic_tr,
7593 static const struct file_operations tracing_mark_raw_fops = {
7594 .open = tracing_mark_open,
7595 .write = tracing_mark_raw_write,
7596 .release = tracing_release_generic_tr,
7599 static const struct file_operations trace_clock_fops = {
7600 .open = tracing_clock_open,
7602 .llseek = seq_lseek,
7603 .release = tracing_single_release_tr,
7604 .write = tracing_clock_write,
7607 static const struct file_operations trace_time_stamp_mode_fops = {
7608 .open = tracing_time_stamp_mode_open,
7610 .llseek = seq_lseek,
7611 .release = tracing_single_release_tr,
7614 #ifdef CONFIG_TRACER_SNAPSHOT
7615 static const struct file_operations snapshot_fops = {
7616 .open = tracing_snapshot_open,
7618 .write = tracing_snapshot_write,
7619 .llseek = tracing_lseek,
7620 .release = tracing_snapshot_release,
7623 static const struct file_operations snapshot_raw_fops = {
7624 .open = snapshot_raw_open,
7625 .read = tracing_buffers_read,
7626 .release = tracing_buffers_release,
7627 .splice_read = tracing_buffers_splice_read,
7628 .llseek = no_llseek,
7631 #endif /* CONFIG_TRACER_SNAPSHOT */
7634 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7635 * @filp: The active open file structure
7636 * @ubuf: The userspace provided buffer to read value into
7637 * @cnt: The maximum number of bytes to read
7638 * @ppos: The current "file" position
7640 * This function implements the write interface for a struct trace_min_max_param.
7641 * The filp->private_data must point to a trace_min_max_param structure that
7642 * defines where to write the value, the min and the max acceptable values,
7643 * and a lock to protect the write.
7646 trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
7648 struct trace_min_max_param *param = filp->private_data;
7655 err = kstrtoull_from_user(ubuf, cnt, 10, &val);
7660 mutex_lock(param->lock);
7662 if (param->min && val < *param->min)
7665 if (param->max && val > *param->max)
7672 mutex_unlock(param->lock);
7681 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7682 * @filp: The active open file structure
7683 * @ubuf: The userspace provided buffer to read value into
7684 * @cnt: The maximum number of bytes to read
7685 * @ppos: The current "file" position
7687 * This function implements the read interface for a struct trace_min_max_param.
7688 * The filp->private_data must point to a trace_min_max_param struct with valid
7692 trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
7694 struct trace_min_max_param *param = filp->private_data;
7695 char buf[U64_STR_SIZE];
7704 if (cnt > sizeof(buf))
7707 len = snprintf(buf, sizeof(buf), "%llu\n", val);
7709 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
7712 const struct file_operations trace_min_max_fops = {
7713 .open = tracing_open_generic,
7714 .read = trace_min_max_read,
7715 .write = trace_min_max_write,
7718 #define TRACING_LOG_ERRS_MAX 8
7719 #define TRACING_LOG_LOC_MAX 128
7721 #define CMD_PREFIX " Command: "
7724 const char **errs; /* ptr to loc-specific array of err strings */
7725 u8 type; /* index into errs -> specific err string */
7726 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
7730 struct tracing_log_err {
7731 struct list_head list;
7732 struct err_info info;
7733 char loc[TRACING_LOG_LOC_MAX]; /* err location */
7734 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
7737 static DEFINE_MUTEX(tracing_err_log_lock);
7739 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
7741 struct tracing_log_err *err;
7743 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7744 err = kzalloc(sizeof(*err), GFP_KERNEL);
7746 err = ERR_PTR(-ENOMEM);
7748 tr->n_err_log_entries++;
7753 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7754 list_del(&err->list);
7760 * err_pos - find the position of a string within a command for error careting
7761 * @cmd: The tracing command that caused the error
7762 * @str: The string to position the caret at within @cmd
7764 * Finds the position of the first occurrence of @str within @cmd. The
7765 * return value can be passed to tracing_log_err() for caret placement
7768 * Returns the index within @cmd of the first occurrence of @str or 0
7769 * if @str was not found.
7771 unsigned int err_pos(char *cmd, const char *str)
7775 if (WARN_ON(!strlen(cmd)))
7778 found = strstr(cmd, str);
7786 * tracing_log_err - write an error to the tracing error log
7787 * @tr: The associated trace array for the error (NULL for top level array)
7788 * @loc: A string describing where the error occurred
7789 * @cmd: The tracing command that caused the error
7790 * @errs: The array of loc-specific static error strings
7791 * @type: The index into errs[], which produces the specific static err string
7792 * @pos: The position the caret should be placed in the cmd
7794 * Writes an error into tracing/error_log of the form:
7796 * <loc>: error: <text>
7800 * tracing/error_log is a small log file containing the last
7801 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7802 * unless there has been a tracing error, and the error log can be
7803 * cleared and have its memory freed by writing the empty string in
7804 * truncation mode to it i.e. echo > tracing/error_log.
7806 * NOTE: the @errs array along with the @type param are used to
7807 * produce a static error string - this string is not copied and saved
7808 * when the error is logged - only a pointer to it is saved. See
7809 * existing callers for examples of how static strings are typically
7810 * defined for use with tracing_log_err().
7812 void tracing_log_err(struct trace_array *tr,
7813 const char *loc, const char *cmd,
7814 const char **errs, u8 type, u8 pos)
7816 struct tracing_log_err *err;
7821 mutex_lock(&tracing_err_log_lock);
7822 err = get_tracing_log_err(tr);
7823 if (PTR_ERR(err) == -ENOMEM) {
7824 mutex_unlock(&tracing_err_log_lock);
7828 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7829 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7831 err->info.errs = errs;
7832 err->info.type = type;
7833 err->info.pos = pos;
7834 err->info.ts = local_clock();
7836 list_add_tail(&err->list, &tr->err_log);
7837 mutex_unlock(&tracing_err_log_lock);
7840 static void clear_tracing_err_log(struct trace_array *tr)
7842 struct tracing_log_err *err, *next;
7844 mutex_lock(&tracing_err_log_lock);
7845 list_for_each_entry_safe(err, next, &tr->err_log, list) {
7846 list_del(&err->list);
7850 tr->n_err_log_entries = 0;
7851 mutex_unlock(&tracing_err_log_lock);
7854 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7856 struct trace_array *tr = m->private;
7858 mutex_lock(&tracing_err_log_lock);
7860 return seq_list_start(&tr->err_log, *pos);
7863 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7865 struct trace_array *tr = m->private;
7867 return seq_list_next(v, &tr->err_log, pos);
7870 static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7872 mutex_unlock(&tracing_err_log_lock);
7875 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7879 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7881 for (i = 0; i < pos; i++)
7886 static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7888 struct tracing_log_err *err = v;
7891 const char *err_text = err->info.errs[err->info.type];
7892 u64 sec = err->info.ts;
7895 nsec = do_div(sec, NSEC_PER_SEC);
7896 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7897 err->loc, err_text);
7898 seq_printf(m, "%s", err->cmd);
7899 tracing_err_log_show_pos(m, err->info.pos);
7905 static const struct seq_operations tracing_err_log_seq_ops = {
7906 .start = tracing_err_log_seq_start,
7907 .next = tracing_err_log_seq_next,
7908 .stop = tracing_err_log_seq_stop,
7909 .show = tracing_err_log_seq_show
7912 static int tracing_err_log_open(struct inode *inode, struct file *file)
7914 struct trace_array *tr = inode->i_private;
7917 ret = tracing_check_open_get_tr(tr);
7921 /* If this file was opened for write, then erase contents */
7922 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7923 clear_tracing_err_log(tr);
7925 if (file->f_mode & FMODE_READ) {
7926 ret = seq_open(file, &tracing_err_log_seq_ops);
7928 struct seq_file *m = file->private_data;
7931 trace_array_put(tr);
7937 static ssize_t tracing_err_log_write(struct file *file,
7938 const char __user *buffer,
7939 size_t count, loff_t *ppos)
7944 static int tracing_err_log_release(struct inode *inode, struct file *file)
7946 struct trace_array *tr = inode->i_private;
7948 trace_array_put(tr);
7950 if (file->f_mode & FMODE_READ)
7951 seq_release(inode, file);
7956 static const struct file_operations tracing_err_log_fops = {
7957 .open = tracing_err_log_open,
7958 .write = tracing_err_log_write,
7960 .llseek = seq_lseek,
7961 .release = tracing_err_log_release,
7964 static int tracing_buffers_open(struct inode *inode, struct file *filp)
7966 struct trace_array *tr = inode->i_private;
7967 struct ftrace_buffer_info *info;
7970 ret = tracing_check_open_get_tr(tr);
7974 info = kvzalloc(sizeof(*info), GFP_KERNEL);
7976 trace_array_put(tr);
7980 mutex_lock(&trace_types_lock);
7983 info->iter.cpu_file = tracing_get_cpu(inode);
7984 info->iter.trace = tr->current_trace;
7985 info->iter.array_buffer = &tr->array_buffer;
7987 /* Force reading ring buffer for first read */
7988 info->read = (unsigned int)-1;
7990 filp->private_data = info;
7994 mutex_unlock(&trace_types_lock);
7996 ret = nonseekable_open(inode, filp);
7998 trace_array_put(tr);
8004 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
8006 struct ftrace_buffer_info *info = filp->private_data;
8007 struct trace_iterator *iter = &info->iter;
8009 return trace_poll(iter, filp, poll_table);
8013 tracing_buffers_read(struct file *filp, char __user *ubuf,
8014 size_t count, loff_t *ppos)
8016 struct ftrace_buffer_info *info = filp->private_data;
8017 struct trace_iterator *iter = &info->iter;
8024 #ifdef CONFIG_TRACER_MAX_TRACE
8025 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8030 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
8032 if (IS_ERR(info->spare)) {
8033 ret = PTR_ERR(info->spare);
8036 info->spare_cpu = iter->cpu_file;
8042 /* Do we have previous read data to read? */
8043 if (info->read < PAGE_SIZE)
8047 trace_access_lock(iter->cpu_file);
8048 ret = ring_buffer_read_page(iter->array_buffer->buffer,
8052 trace_access_unlock(iter->cpu_file);
8055 if (trace_empty(iter)) {
8056 if ((filp->f_flags & O_NONBLOCK))
8059 ret = wait_on_pipe(iter, 0);
8070 size = PAGE_SIZE - info->read;
8074 ret = copy_to_user(ubuf, info->spare + info->read, size);
8086 static int tracing_buffers_release(struct inode *inode, struct file *file)
8088 struct ftrace_buffer_info *info = file->private_data;
8089 struct trace_iterator *iter = &info->iter;
8091 mutex_lock(&trace_types_lock);
8093 iter->tr->trace_ref--;
8095 __trace_array_put(iter->tr);
8098 ring_buffer_free_read_page(iter->array_buffer->buffer,
8099 info->spare_cpu, info->spare);
8102 mutex_unlock(&trace_types_lock);
8108 struct trace_buffer *buffer;
8111 refcount_t refcount;
8114 static void buffer_ref_release(struct buffer_ref *ref)
8116 if (!refcount_dec_and_test(&ref->refcount))
8118 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
8122 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
8123 struct pipe_buffer *buf)
8125 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8127 buffer_ref_release(ref);
8131 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
8132 struct pipe_buffer *buf)
8134 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8136 if (refcount_read(&ref->refcount) > INT_MAX/2)
8139 refcount_inc(&ref->refcount);
8143 /* Pipe buffer operations for a buffer. */
8144 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
8145 .release = buffer_pipe_buf_release,
8146 .get = buffer_pipe_buf_get,
8150 * Callback from splice_to_pipe(), if we need to release some pages
8151 * at the end of the spd in case we error'ed out in filling the pipe.
8153 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
8155 struct buffer_ref *ref =
8156 (struct buffer_ref *)spd->partial[i].private;
8158 buffer_ref_release(ref);
8159 spd->partial[i].private = 0;
8163 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
8164 struct pipe_inode_info *pipe, size_t len,
8167 struct ftrace_buffer_info *info = file->private_data;
8168 struct trace_iterator *iter = &info->iter;
8169 struct partial_page partial_def[PIPE_DEF_BUFFERS];
8170 struct page *pages_def[PIPE_DEF_BUFFERS];
8171 struct splice_pipe_desc spd = {
8173 .partial = partial_def,
8174 .nr_pages_max = PIPE_DEF_BUFFERS,
8175 .ops = &buffer_pipe_buf_ops,
8176 .spd_release = buffer_spd_release,
8178 struct buffer_ref *ref;
8182 #ifdef CONFIG_TRACER_MAX_TRACE
8183 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8187 if (*ppos & (PAGE_SIZE - 1))
8190 if (len & (PAGE_SIZE - 1)) {
8191 if (len < PAGE_SIZE)
8196 if (splice_grow_spd(pipe, &spd))
8200 trace_access_lock(iter->cpu_file);
8201 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8203 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
8207 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
8213 refcount_set(&ref->refcount, 1);
8214 ref->buffer = iter->array_buffer->buffer;
8215 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
8216 if (IS_ERR(ref->page)) {
8217 ret = PTR_ERR(ref->page);
8222 ref->cpu = iter->cpu_file;
8224 r = ring_buffer_read_page(ref->buffer, &ref->page,
8225 len, iter->cpu_file, 1);
8227 ring_buffer_free_read_page(ref->buffer, ref->cpu,
8233 page = virt_to_page(ref->page);
8235 spd.pages[i] = page;
8236 spd.partial[i].len = PAGE_SIZE;
8237 spd.partial[i].offset = 0;
8238 spd.partial[i].private = (unsigned long)ref;
8242 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
8245 trace_access_unlock(iter->cpu_file);
8248 /* did we read anything? */
8249 if (!spd.nr_pages) {
8254 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
8257 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
8264 ret = splice_to_pipe(pipe, &spd);
8266 splice_shrink_spd(&spd);
8271 static const struct file_operations tracing_buffers_fops = {
8272 .open = tracing_buffers_open,
8273 .read = tracing_buffers_read,
8274 .poll = tracing_buffers_poll,
8275 .release = tracing_buffers_release,
8276 .splice_read = tracing_buffers_splice_read,
8277 .llseek = no_llseek,
8281 tracing_stats_read(struct file *filp, char __user *ubuf,
8282 size_t count, loff_t *ppos)
8284 struct inode *inode = file_inode(filp);
8285 struct trace_array *tr = inode->i_private;
8286 struct array_buffer *trace_buf = &tr->array_buffer;
8287 int cpu = tracing_get_cpu(inode);
8288 struct trace_seq *s;
8290 unsigned long long t;
8291 unsigned long usec_rem;
8293 s = kmalloc(sizeof(*s), GFP_KERNEL);
8299 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
8300 trace_seq_printf(s, "entries: %ld\n", cnt);
8302 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
8303 trace_seq_printf(s, "overrun: %ld\n", cnt);
8305 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
8306 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
8308 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
8309 trace_seq_printf(s, "bytes: %ld\n", cnt);
8311 if (trace_clocks[tr->clock_id].in_ns) {
8312 /* local or global for trace_clock */
8313 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8314 usec_rem = do_div(t, USEC_PER_SEC);
8315 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
8318 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));
8319 usec_rem = do_div(t, USEC_PER_SEC);
8320 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
8322 /* counter or tsc mode for trace_clock */
8323 trace_seq_printf(s, "oldest event ts: %llu\n",
8324 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
8326 trace_seq_printf(s, "now ts: %llu\n",
8327 ring_buffer_time_stamp(trace_buf->buffer));
8330 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
8331 trace_seq_printf(s, "dropped events: %ld\n", cnt);
8333 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
8334 trace_seq_printf(s, "read events: %ld\n", cnt);
8336 count = simple_read_from_buffer(ubuf, count, ppos,
8337 s->buffer, trace_seq_used(s));
8344 static const struct file_operations tracing_stats_fops = {
8345 .open = tracing_open_generic_tr,
8346 .read = tracing_stats_read,
8347 .llseek = generic_file_llseek,
8348 .release = tracing_release_generic_tr,
8351 #ifdef CONFIG_DYNAMIC_FTRACE
8354 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
8355 size_t cnt, loff_t *ppos)
8361 /* 256 should be plenty to hold the amount needed */
8362 buf = kmalloc(256, GFP_KERNEL);
8366 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
8367 ftrace_update_tot_cnt,
8368 ftrace_number_of_pages,
8369 ftrace_number_of_groups);
8371 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8376 static const struct file_operations tracing_dyn_info_fops = {
8377 .open = tracing_open_generic,
8378 .read = tracing_read_dyn_info,
8379 .llseek = generic_file_llseek,
8381 #endif /* CONFIG_DYNAMIC_FTRACE */
8383 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
8385 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
8386 struct trace_array *tr, struct ftrace_probe_ops *ops,
8389 tracing_snapshot_instance(tr);
8393 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
8394 struct trace_array *tr, struct ftrace_probe_ops *ops,
8397 struct ftrace_func_mapper *mapper = data;
8401 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8411 tracing_snapshot_instance(tr);
8415 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
8416 struct ftrace_probe_ops *ops, void *data)
8418 struct ftrace_func_mapper *mapper = data;
8421 seq_printf(m, "%ps:", (void *)ip);
8423 seq_puts(m, "snapshot");
8426 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8429 seq_printf(m, ":count=%ld\n", *count);
8431 seq_puts(m, ":unlimited\n");
8437 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
8438 unsigned long ip, void *init_data, void **data)
8440 struct ftrace_func_mapper *mapper = *data;
8443 mapper = allocate_ftrace_func_mapper();
8449 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
8453 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
8454 unsigned long ip, void *data)
8456 struct ftrace_func_mapper *mapper = data;
8461 free_ftrace_func_mapper(mapper, NULL);
8465 ftrace_func_mapper_remove_ip(mapper, ip);
8468 static struct ftrace_probe_ops snapshot_probe_ops = {
8469 .func = ftrace_snapshot,
8470 .print = ftrace_snapshot_print,
8473 static struct ftrace_probe_ops snapshot_count_probe_ops = {
8474 .func = ftrace_count_snapshot,
8475 .print = ftrace_snapshot_print,
8476 .init = ftrace_snapshot_init,
8477 .free = ftrace_snapshot_free,
8481 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
8482 char *glob, char *cmd, char *param, int enable)
8484 struct ftrace_probe_ops *ops;
8485 void *count = (void *)-1;
8492 /* hash funcs only work with set_ftrace_filter */
8496 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
8499 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
8504 number = strsep(¶m, ":");
8506 if (!strlen(number))
8510 * We use the callback data field (which is a pointer)
8513 ret = kstrtoul(number, 0, (unsigned long *)&count);
8518 ret = tracing_alloc_snapshot_instance(tr);
8522 ret = register_ftrace_function_probe(glob, tr, ops, count);
8525 return ret < 0 ? ret : 0;
8528 static struct ftrace_func_command ftrace_snapshot_cmd = {
8530 .func = ftrace_trace_snapshot_callback,
8533 static __init int register_snapshot_cmd(void)
8535 return register_ftrace_command(&ftrace_snapshot_cmd);
8538 static inline __init int register_snapshot_cmd(void) { return 0; }
8539 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
8541 static struct dentry *tracing_get_dentry(struct trace_array *tr)
8543 if (WARN_ON(!tr->dir))
8544 return ERR_PTR(-ENODEV);
8546 /* Top directory uses NULL as the parent */
8547 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
8550 /* All sub buffers have a descriptor */
8554 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
8556 struct dentry *d_tracer;
8559 return tr->percpu_dir;
8561 d_tracer = tracing_get_dentry(tr);
8562 if (IS_ERR(d_tracer))
8565 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8567 MEM_FAIL(!tr->percpu_dir,
8568 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8570 return tr->percpu_dir;
8573 static struct dentry *
8574 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8575 void *data, long cpu, const struct file_operations *fops)
8577 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8579 if (ret) /* See tracing_get_cpu() */
8580 d_inode(ret)->i_cdev = (void *)(cpu + 1);
8585 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8587 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8588 struct dentry *d_cpu;
8589 char cpu_dir[30]; /* 30 characters should be more than enough */
8594 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8595 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8597 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8601 /* per cpu trace_pipe */
8602 trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu,
8603 tr, cpu, &tracing_pipe_fops);
8606 trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu,
8607 tr, cpu, &tracing_fops);
8609 trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu,
8610 tr, cpu, &tracing_buffers_fops);
8612 trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu,
8613 tr, cpu, &tracing_stats_fops);
8615 trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu,
8616 tr, cpu, &tracing_entries_fops);
8618 #ifdef CONFIG_TRACER_SNAPSHOT
8619 trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu,
8620 tr, cpu, &snapshot_fops);
8622 trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu,
8623 tr, cpu, &snapshot_raw_fops);
8627 #ifdef CONFIG_FTRACE_SELFTEST
8628 /* Let selftest have access to static functions in this file */
8629 #include "trace_selftest.c"
8633 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8636 struct trace_option_dentry *topt = filp->private_data;
8639 if (topt->flags->val & topt->opt->bit)
8644 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8648 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8651 struct trace_option_dentry *topt = filp->private_data;
8655 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8659 if (val != 0 && val != 1)
8662 if (!!(topt->flags->val & topt->opt->bit) != val) {
8663 mutex_lock(&trace_types_lock);
8664 ret = __set_tracer_option(topt->tr, topt->flags,
8666 mutex_unlock(&trace_types_lock);
8677 static const struct file_operations trace_options_fops = {
8678 .open = tracing_open_generic,
8679 .read = trace_options_read,
8680 .write = trace_options_write,
8681 .llseek = generic_file_llseek,
8685 * In order to pass in both the trace_array descriptor as well as the index
8686 * to the flag that the trace option file represents, the trace_array
8687 * has a character array of trace_flags_index[], which holds the index
8688 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8689 * The address of this character array is passed to the flag option file
8690 * read/write callbacks.
8692 * In order to extract both the index and the trace_array descriptor,
8693 * get_tr_index() uses the following algorithm.
8697 * As the pointer itself contains the address of the index (remember
8700 * Then to get the trace_array descriptor, by subtracting that index
8701 * from the ptr, we get to the start of the index itself.
8703 * ptr - idx == &index[0]
8705 * Then a simple container_of() from that pointer gets us to the
8706 * trace_array descriptor.
8708 static void get_tr_index(void *data, struct trace_array **ptr,
8709 unsigned int *pindex)
8711 *pindex = *(unsigned char *)data;
8713 *ptr = container_of(data - *pindex, struct trace_array,
8718 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8721 void *tr_index = filp->private_data;
8722 struct trace_array *tr;
8726 get_tr_index(tr_index, &tr, &index);
8728 if (tr->trace_flags & (1 << index))
8733 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8737 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8740 void *tr_index = filp->private_data;
8741 struct trace_array *tr;
8746 get_tr_index(tr_index, &tr, &index);
8748 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8752 if (val != 0 && val != 1)
8755 mutex_lock(&event_mutex);
8756 mutex_lock(&trace_types_lock);
8757 ret = set_tracer_flag(tr, 1 << index, val);
8758 mutex_unlock(&trace_types_lock);
8759 mutex_unlock(&event_mutex);
8769 static const struct file_operations trace_options_core_fops = {
8770 .open = tracing_open_generic,
8771 .read = trace_options_core_read,
8772 .write = trace_options_core_write,
8773 .llseek = generic_file_llseek,
8776 struct dentry *trace_create_file(const char *name,
8778 struct dentry *parent,
8780 const struct file_operations *fops)
8784 ret = tracefs_create_file(name, mode, parent, data, fops);
8786 pr_warn("Could not create tracefs '%s' entry\n", name);
8792 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8794 struct dentry *d_tracer;
8799 d_tracer = tracing_get_dentry(tr);
8800 if (IS_ERR(d_tracer))
8803 tr->options = tracefs_create_dir("options", d_tracer);
8805 pr_warn("Could not create tracefs directory 'options'\n");
8813 create_trace_option_file(struct trace_array *tr,
8814 struct trace_option_dentry *topt,
8815 struct tracer_flags *flags,
8816 struct tracer_opt *opt)
8818 struct dentry *t_options;
8820 t_options = trace_options_init_dentry(tr);
8824 topt->flags = flags;
8828 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,
8829 t_options, topt, &trace_options_fops);
8834 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8836 struct trace_option_dentry *topts;
8837 struct trace_options *tr_topts;
8838 struct tracer_flags *flags;
8839 struct tracer_opt *opts;
8846 flags = tracer->flags;
8848 if (!flags || !flags->opts)
8852 * If this is an instance, only create flags for tracers
8853 * the instance may have.
8855 if (!trace_ok_for_array(tracer, tr))
8858 for (i = 0; i < tr->nr_topts; i++) {
8859 /* Make sure there's no duplicate flags. */
8860 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8866 for (cnt = 0; opts[cnt].name; cnt++)
8869 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8873 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8880 tr->topts = tr_topts;
8881 tr->topts[tr->nr_topts].tracer = tracer;
8882 tr->topts[tr->nr_topts].topts = topts;
8885 for (cnt = 0; opts[cnt].name; cnt++) {
8886 create_trace_option_file(tr, &topts[cnt], flags,
8888 MEM_FAIL(topts[cnt].entry == NULL,
8889 "Failed to create trace option: %s",
8894 static struct dentry *
8895 create_trace_option_core_file(struct trace_array *tr,
8896 const char *option, long index)
8898 struct dentry *t_options;
8900 t_options = trace_options_init_dentry(tr);
8904 return trace_create_file(option, TRACE_MODE_WRITE, t_options,
8905 (void *)&tr->trace_flags_index[index],
8906 &trace_options_core_fops);
8909 static void create_trace_options_dir(struct trace_array *tr)
8911 struct dentry *t_options;
8912 bool top_level = tr == &global_trace;
8915 t_options = trace_options_init_dentry(tr);
8919 for (i = 0; trace_options[i]; i++) {
8921 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8922 create_trace_option_core_file(tr, trace_options[i], i);
8927 rb_simple_read(struct file *filp, char __user *ubuf,
8928 size_t cnt, loff_t *ppos)
8930 struct trace_array *tr = filp->private_data;
8934 r = tracer_tracing_is_on(tr);
8935 r = sprintf(buf, "%d\n", r);
8937 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8941 rb_simple_write(struct file *filp, const char __user *ubuf,
8942 size_t cnt, loff_t *ppos)
8944 struct trace_array *tr = filp->private_data;
8945 struct trace_buffer *buffer = tr->array_buffer.buffer;
8949 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8954 mutex_lock(&trace_types_lock);
8955 if (!!val == tracer_tracing_is_on(tr)) {
8956 val = 0; /* do nothing */
8958 tracer_tracing_on(tr);
8959 if (tr->current_trace->start)
8960 tr->current_trace->start(tr);
8962 tracer_tracing_off(tr);
8963 if (tr->current_trace->stop)
8964 tr->current_trace->stop(tr);
8966 mutex_unlock(&trace_types_lock);
8974 static const struct file_operations rb_simple_fops = {
8975 .open = tracing_open_generic_tr,
8976 .read = rb_simple_read,
8977 .write = rb_simple_write,
8978 .release = tracing_release_generic_tr,
8979 .llseek = default_llseek,
8983 buffer_percent_read(struct file *filp, char __user *ubuf,
8984 size_t cnt, loff_t *ppos)
8986 struct trace_array *tr = filp->private_data;
8990 r = tr->buffer_percent;
8991 r = sprintf(buf, "%d\n", r);
8993 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8997 buffer_percent_write(struct file *filp, const char __user *ubuf,
8998 size_t cnt, loff_t *ppos)
9000 struct trace_array *tr = filp->private_data;
9004 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
9014 tr->buffer_percent = val;
9021 static const struct file_operations buffer_percent_fops = {
9022 .open = tracing_open_generic_tr,
9023 .read = buffer_percent_read,
9024 .write = buffer_percent_write,
9025 .release = tracing_release_generic_tr,
9026 .llseek = default_llseek,
9029 static struct dentry *trace_instance_dir;
9032 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
9035 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
9037 enum ring_buffer_flags rb_flags;
9039 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
9043 buf->buffer = ring_buffer_alloc(size, rb_flags);
9047 buf->data = alloc_percpu(struct trace_array_cpu);
9049 ring_buffer_free(buf->buffer);
9054 /* Allocate the first page for all buffers */
9055 set_buffer_entries(&tr->array_buffer,
9056 ring_buffer_size(tr->array_buffer.buffer, 0));
9061 static int allocate_trace_buffers(struct trace_array *tr, int size)
9065 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
9069 #ifdef CONFIG_TRACER_MAX_TRACE
9070 ret = allocate_trace_buffer(tr, &tr->max_buffer,
9071 allocate_snapshot ? size : 1);
9072 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
9073 ring_buffer_free(tr->array_buffer.buffer);
9074 tr->array_buffer.buffer = NULL;
9075 free_percpu(tr->array_buffer.data);
9076 tr->array_buffer.data = NULL;
9079 tr->allocated_snapshot = allocate_snapshot;
9082 * Only the top level trace array gets its snapshot allocated
9083 * from the kernel command line.
9085 allocate_snapshot = false;
9091 static void free_trace_buffer(struct array_buffer *buf)
9094 ring_buffer_free(buf->buffer);
9096 free_percpu(buf->data);
9101 static void free_trace_buffers(struct trace_array *tr)
9106 free_trace_buffer(&tr->array_buffer);
9108 #ifdef CONFIG_TRACER_MAX_TRACE
9109 free_trace_buffer(&tr->max_buffer);
9113 static void init_trace_flags_index(struct trace_array *tr)
9117 /* Used by the trace options files */
9118 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
9119 tr->trace_flags_index[i] = i;
9122 static void __update_tracer_options(struct trace_array *tr)
9126 for (t = trace_types; t; t = t->next)
9127 add_tracer_options(tr, t);
9130 static void update_tracer_options(struct trace_array *tr)
9132 mutex_lock(&trace_types_lock);
9133 __update_tracer_options(tr);
9134 mutex_unlock(&trace_types_lock);
9137 /* Must have trace_types_lock held */
9138 struct trace_array *trace_array_find(const char *instance)
9140 struct trace_array *tr, *found = NULL;
9142 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9143 if (tr->name && strcmp(tr->name, instance) == 0) {
9152 struct trace_array *trace_array_find_get(const char *instance)
9154 struct trace_array *tr;
9156 mutex_lock(&trace_types_lock);
9157 tr = trace_array_find(instance);
9160 mutex_unlock(&trace_types_lock);
9165 static int trace_array_create_dir(struct trace_array *tr)
9169 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
9173 ret = event_trace_add_tracer(tr->dir, tr);
9175 tracefs_remove(tr->dir);
9179 init_tracer_tracefs(tr, tr->dir);
9180 __update_tracer_options(tr);
9185 static struct trace_array *trace_array_create(const char *name)
9187 struct trace_array *tr;
9191 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
9193 return ERR_PTR(ret);
9195 tr->name = kstrdup(name, GFP_KERNEL);
9199 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
9202 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
9204 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
9206 raw_spin_lock_init(&tr->start_lock);
9208 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9210 tr->current_trace = &nop_trace;
9212 INIT_LIST_HEAD(&tr->systems);
9213 INIT_LIST_HEAD(&tr->events);
9214 INIT_LIST_HEAD(&tr->hist_vars);
9215 INIT_LIST_HEAD(&tr->err_log);
9217 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
9220 if (ftrace_allocate_ftrace_ops(tr) < 0)
9223 ftrace_init_trace_array(tr);
9225 init_trace_flags_index(tr);
9227 if (trace_instance_dir) {
9228 ret = trace_array_create_dir(tr);
9232 __trace_early_add_events(tr);
9234 list_add(&tr->list, &ftrace_trace_arrays);
9241 ftrace_free_ftrace_ops(tr);
9242 free_trace_buffers(tr);
9243 free_cpumask_var(tr->tracing_cpumask);
9247 return ERR_PTR(ret);
9250 static int instance_mkdir(const char *name)
9252 struct trace_array *tr;
9255 mutex_lock(&event_mutex);
9256 mutex_lock(&trace_types_lock);
9259 if (trace_array_find(name))
9262 tr = trace_array_create(name);
9264 ret = PTR_ERR_OR_ZERO(tr);
9267 mutex_unlock(&trace_types_lock);
9268 mutex_unlock(&event_mutex);
9273 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
9274 * @name: The name of the trace array to be looked up/created.
9276 * Returns pointer to trace array with given name.
9277 * NULL, if it cannot be created.
9279 * NOTE: This function increments the reference counter associated with the
9280 * trace array returned. This makes sure it cannot be freed while in use.
9281 * Use trace_array_put() once the trace array is no longer needed.
9282 * If the trace_array is to be freed, trace_array_destroy() needs to
9283 * be called after the trace_array_put(), or simply let user space delete
9284 * it from the tracefs instances directory. But until the
9285 * trace_array_put() is called, user space can not delete it.
9288 struct trace_array *trace_array_get_by_name(const char *name)
9290 struct trace_array *tr;
9292 mutex_lock(&event_mutex);
9293 mutex_lock(&trace_types_lock);
9295 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9296 if (tr->name && strcmp(tr->name, name) == 0)
9300 tr = trace_array_create(name);
9308 mutex_unlock(&trace_types_lock);
9309 mutex_unlock(&event_mutex);
9312 EXPORT_SYMBOL_GPL(trace_array_get_by_name);
9314 static int __remove_instance(struct trace_array *tr)
9318 /* Reference counter for a newly created trace array = 1. */
9319 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
9322 list_del(&tr->list);
9324 /* Disable all the flags that were enabled coming in */
9325 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
9326 if ((1 << i) & ZEROED_TRACE_FLAGS)
9327 set_tracer_flag(tr, 1 << i, 0);
9330 tracing_set_nop(tr);
9331 clear_ftrace_function_probes(tr);
9332 event_trace_del_tracer(tr);
9333 ftrace_clear_pids(tr);
9334 ftrace_destroy_function_files(tr);
9335 tracefs_remove(tr->dir);
9336 free_percpu(tr->last_func_repeats);
9337 free_trace_buffers(tr);
9339 for (i = 0; i < tr->nr_topts; i++) {
9340 kfree(tr->topts[i].topts);
9344 free_cpumask_var(tr->tracing_cpumask);
9351 int trace_array_destroy(struct trace_array *this_tr)
9353 struct trace_array *tr;
9359 mutex_lock(&event_mutex);
9360 mutex_lock(&trace_types_lock);
9364 /* Making sure trace array exists before destroying it. */
9365 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9366 if (tr == this_tr) {
9367 ret = __remove_instance(tr);
9372 mutex_unlock(&trace_types_lock);
9373 mutex_unlock(&event_mutex);
9377 EXPORT_SYMBOL_GPL(trace_array_destroy);
9379 static int instance_rmdir(const char *name)
9381 struct trace_array *tr;
9384 mutex_lock(&event_mutex);
9385 mutex_lock(&trace_types_lock);
9388 tr = trace_array_find(name);
9390 ret = __remove_instance(tr);
9392 mutex_unlock(&trace_types_lock);
9393 mutex_unlock(&event_mutex);
9398 static __init void create_trace_instances(struct dentry *d_tracer)
9400 struct trace_array *tr;
9402 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
9405 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
9408 mutex_lock(&event_mutex);
9409 mutex_lock(&trace_types_lock);
9411 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9414 if (MEM_FAIL(trace_array_create_dir(tr) < 0,
9415 "Failed to create instance directory\n"))
9419 mutex_unlock(&trace_types_lock);
9420 mutex_unlock(&event_mutex);
9424 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
9426 struct trace_event_file *file;
9429 trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer,
9430 tr, &show_traces_fops);
9432 trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer,
9433 tr, &set_tracer_fops);
9435 trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer,
9436 tr, &tracing_cpumask_fops);
9438 trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer,
9439 tr, &tracing_iter_fops);
9441 trace_create_file("trace", TRACE_MODE_WRITE, d_tracer,
9444 trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer,
9445 tr, &tracing_pipe_fops);
9447 trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer,
9448 tr, &tracing_entries_fops);
9450 trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer,
9451 tr, &tracing_total_entries_fops);
9453 trace_create_file("free_buffer", 0200, d_tracer,
9454 tr, &tracing_free_buffer_fops);
9456 trace_create_file("trace_marker", 0220, d_tracer,
9457 tr, &tracing_mark_fops);
9459 file = __find_event_file(tr, "ftrace", "print");
9460 if (file && file->dir)
9461 trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
9462 file, &event_trigger_fops);
9463 tr->trace_marker_file = file;
9465 trace_create_file("trace_marker_raw", 0220, d_tracer,
9466 tr, &tracing_mark_raw_fops);
9468 trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr,
9471 trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer,
9472 tr, &rb_simple_fops);
9474 trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr,
9475 &trace_time_stamp_mode_fops);
9477 tr->buffer_percent = 50;
9479 trace_create_file("buffer_percent", TRACE_MODE_READ, d_tracer,
9480 tr, &buffer_percent_fops);
9482 create_trace_options_dir(tr);
9484 trace_create_maxlat_file(tr, d_tracer);
9486 if (ftrace_create_function_files(tr, d_tracer))
9487 MEM_FAIL(1, "Could not allocate function filter files");
9489 #ifdef CONFIG_TRACER_SNAPSHOT
9490 trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer,
9491 tr, &snapshot_fops);
9494 trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer,
9495 tr, &tracing_err_log_fops);
9497 for_each_tracing_cpu(cpu)
9498 tracing_init_tracefs_percpu(tr, cpu);
9500 ftrace_init_tracefs(tr, d_tracer);
9503 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
9505 struct vfsmount *mnt;
9506 struct file_system_type *type;
9509 * To maintain backward compatibility for tools that mount
9510 * debugfs to get to the tracing facility, tracefs is automatically
9511 * mounted to the debugfs/tracing directory.
9513 type = get_fs_type("tracefs");
9516 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
9517 put_filesystem(type);
9526 * tracing_init_dentry - initialize top level trace array
9528 * This is called when creating files or directories in the tracing
9529 * directory. It is called via fs_initcall() by any of the boot up code
9530 * and expects to return the dentry of the top level tracing directory.
9532 int tracing_init_dentry(void)
9534 struct trace_array *tr = &global_trace;
9536 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9537 pr_warn("Tracing disabled due to lockdown\n");
9541 /* The top level trace array uses NULL as parent */
9545 if (WARN_ON(!tracefs_initialized()))
9549 * As there may still be users that expect the tracing
9550 * files to exist in debugfs/tracing, we must automount
9551 * the tracefs file system there, so older tools still
9552 * work with the newer kernel.
9554 tr->dir = debugfs_create_automount("tracing", NULL,
9555 trace_automount, NULL);
9560 extern struct trace_eval_map *__start_ftrace_eval_maps[];
9561 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9563 static struct workqueue_struct *eval_map_wq __initdata;
9564 static struct work_struct eval_map_work __initdata;
9566 static void __init eval_map_work_func(struct work_struct *work)
9570 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9571 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
9574 static int __init trace_eval_init(void)
9576 INIT_WORK(&eval_map_work, eval_map_work_func);
9578 eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0);
9580 pr_err("Unable to allocate eval_map_wq\n");
9582 eval_map_work_func(&eval_map_work);
9586 queue_work(eval_map_wq, &eval_map_work);
9590 static int __init trace_eval_sync(void)
9592 /* Make sure the eval map updates are finished */
9594 destroy_workqueue(eval_map_wq);
9598 late_initcall_sync(trace_eval_sync);
9601 #ifdef CONFIG_MODULES
9602 static void trace_module_add_evals(struct module *mod)
9604 if (!mod->num_trace_evals)
9608 * Modules with bad taint do not have events created, do
9609 * not bother with enums either.
9611 if (trace_module_has_bad_taint(mod))
9614 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9617 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
9618 static void trace_module_remove_evals(struct module *mod)
9620 union trace_eval_map_item *map;
9621 union trace_eval_map_item **last = &trace_eval_maps;
9623 if (!mod->num_trace_evals)
9626 mutex_lock(&trace_eval_mutex);
9628 map = trace_eval_maps;
9631 if (map->head.mod == mod)
9633 map = trace_eval_jmp_to_tail(map);
9634 last = &map->tail.next;
9635 map = map->tail.next;
9640 *last = trace_eval_jmp_to_tail(map)->tail.next;
9643 mutex_unlock(&trace_eval_mutex);
9646 static inline void trace_module_remove_evals(struct module *mod) { }
9647 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9649 static int trace_module_notify(struct notifier_block *self,
9650 unsigned long val, void *data)
9652 struct module *mod = data;
9655 case MODULE_STATE_COMING:
9656 trace_module_add_evals(mod);
9658 case MODULE_STATE_GOING:
9659 trace_module_remove_evals(mod);
9666 static struct notifier_block trace_module_nb = {
9667 .notifier_call = trace_module_notify,
9670 #endif /* CONFIG_MODULES */
9672 static __init int tracer_init_tracefs(void)
9676 trace_access_lock_init();
9678 ret = tracing_init_dentry();
9684 init_tracer_tracefs(&global_trace, NULL);
9685 ftrace_init_tracefs_toplevel(&global_trace, NULL);
9687 trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL,
9688 &global_trace, &tracing_thresh_fops);
9690 trace_create_file("README", TRACE_MODE_READ, NULL,
9691 NULL, &tracing_readme_fops);
9693 trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL,
9694 NULL, &tracing_saved_cmdlines_fops);
9696 trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
9697 NULL, &tracing_saved_cmdlines_size_fops);
9699 trace_create_file("saved_tgids", TRACE_MODE_READ, NULL,
9700 NULL, &tracing_saved_tgids_fops);
9704 trace_create_eval_file(NULL);
9706 #ifdef CONFIG_MODULES
9707 register_module_notifier(&trace_module_nb);
9710 #ifdef CONFIG_DYNAMIC_FTRACE
9711 trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
9712 NULL, &tracing_dyn_info_fops);
9715 create_trace_instances(NULL);
9717 update_tracer_options(&global_trace);
9722 fs_initcall(tracer_init_tracefs);
9724 static int trace_panic_handler(struct notifier_block *this,
9725 unsigned long event, void *unused)
9727 if (ftrace_dump_on_oops)
9728 ftrace_dump(ftrace_dump_on_oops);
9732 static struct notifier_block trace_panic_notifier = {
9733 .notifier_call = trace_panic_handler,
9735 .priority = 150 /* priority: INT_MAX >= x >= 0 */
9738 static int trace_die_handler(struct notifier_block *self,
9744 if (ftrace_dump_on_oops)
9745 ftrace_dump(ftrace_dump_on_oops);
9753 static struct notifier_block trace_die_notifier = {
9754 .notifier_call = trace_die_handler,
9759 * printk is set to max of 1024, we really don't need it that big.
9760 * Nothing should be printing 1000 characters anyway.
9762 #define TRACE_MAX_PRINT 1000
9765 * Define here KERN_TRACE so that we have one place to modify
9766 * it if we decide to change what log level the ftrace dump
9769 #define KERN_TRACE KERN_EMERG
9772 trace_printk_seq(struct trace_seq *s)
9774 /* Probably should print a warning here. */
9775 if (s->seq.len >= TRACE_MAX_PRINT)
9776 s->seq.len = TRACE_MAX_PRINT;
9779 * More paranoid code. Although the buffer size is set to
9780 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9781 * an extra layer of protection.
9783 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9784 s->seq.len = s->seq.size - 1;
9786 /* should be zero ended, but we are paranoid. */
9787 s->buffer[s->seq.len] = 0;
9789 printk(KERN_TRACE "%s", s->buffer);
9794 void trace_init_global_iter(struct trace_iterator *iter)
9796 iter->tr = &global_trace;
9797 iter->trace = iter->tr->current_trace;
9798 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9799 iter->array_buffer = &global_trace.array_buffer;
9801 if (iter->trace && iter->trace->open)
9802 iter->trace->open(iter);
9804 /* Annotate start of buffers if we had overruns */
9805 if (ring_buffer_overruns(iter->array_buffer->buffer))
9806 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9808 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9809 if (trace_clocks[iter->tr->clock_id].in_ns)
9810 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9813 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9815 /* use static because iter can be a bit big for the stack */
9816 static struct trace_iterator iter;
9817 static atomic_t dump_running;
9818 struct trace_array *tr = &global_trace;
9819 unsigned int old_userobj;
9820 unsigned long flags;
9823 /* Only allow one dump user at a time. */
9824 if (atomic_inc_return(&dump_running) != 1) {
9825 atomic_dec(&dump_running);
9830 * Always turn off tracing when we dump.
9831 * We don't need to show trace output of what happens
9832 * between multiple crashes.
9834 * If the user does a sysrq-z, then they can re-enable
9835 * tracing with echo 1 > tracing_on.
9839 local_irq_save(flags);
9841 /* Simulate the iterator */
9842 trace_init_global_iter(&iter);
9843 /* Can not use kmalloc for iter.temp and iter.fmt */
9844 iter.temp = static_temp_buf;
9845 iter.temp_size = STATIC_TEMP_BUF_SIZE;
9846 iter.fmt = static_fmt_buf;
9847 iter.fmt_size = STATIC_FMT_BUF_SIZE;
9849 for_each_tracing_cpu(cpu) {
9850 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9853 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9855 /* don't look at user memory in panic mode */
9856 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9858 switch (oops_dump_mode) {
9860 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9863 iter.cpu_file = raw_smp_processor_id();
9868 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9869 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9872 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9874 /* Did function tracer already get disabled? */
9875 if (ftrace_is_dead()) {
9876 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9877 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9881 * We need to stop all tracing on all CPUS to read
9882 * the next buffer. This is a bit expensive, but is
9883 * not done often. We fill all what we can read,
9884 * and then release the locks again.
9887 while (!trace_empty(&iter)) {
9890 printk(KERN_TRACE "---------------------------------\n");
9894 trace_iterator_reset(&iter);
9895 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9897 if (trace_find_next_entry_inc(&iter) != NULL) {
9900 ret = print_trace_line(&iter);
9901 if (ret != TRACE_TYPE_NO_CONSUME)
9902 trace_consume(&iter);
9904 touch_nmi_watchdog();
9906 trace_printk_seq(&iter.seq);
9910 printk(KERN_TRACE " (ftrace buffer empty)\n");
9912 printk(KERN_TRACE "---------------------------------\n");
9915 tr->trace_flags |= old_userobj;
9917 for_each_tracing_cpu(cpu) {
9918 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9920 atomic_dec(&dump_running);
9921 local_irq_restore(flags);
9923 EXPORT_SYMBOL_GPL(ftrace_dump);
9925 #define WRITE_BUFSIZE 4096
9927 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9928 size_t count, loff_t *ppos,
9929 int (*createfn)(const char *))
9931 char *kbuf, *buf, *tmp;
9936 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9940 while (done < count) {
9941 size = count - done;
9943 if (size >= WRITE_BUFSIZE)
9944 size = WRITE_BUFSIZE - 1;
9946 if (copy_from_user(kbuf, buffer + done, size)) {
9953 tmp = strchr(buf, '\n');
9956 size = tmp - buf + 1;
9959 if (done + size < count) {
9962 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9963 pr_warn("Line length is too long: Should be less than %d\n",
9971 /* Remove comments */
9972 tmp = strchr(buf, '#');
9977 ret = createfn(buf);
9982 } while (done < count);
9992 __init static int tracer_alloc_buffers(void)
9998 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9999 pr_warn("Tracing disabled due to lockdown\n");
10004 * Make sure we don't accidentally add more trace options
10005 * than we have bits for.
10007 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
10009 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
10012 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
10013 goto out_free_buffer_mask;
10015 /* Only allocate trace_printk buffers if a trace_printk exists */
10016 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
10017 /* Must be called before global_trace.buffer is allocated */
10018 trace_printk_init_buffers();
10020 /* To save memory, keep the ring buffer size to its minimum */
10021 if (ring_buffer_expanded)
10022 ring_buf_size = trace_buf_size;
10026 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
10027 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
10029 raw_spin_lock_init(&global_trace.start_lock);
10032 * The prepare callbacks allocates some memory for the ring buffer. We
10033 * don't free the buffer if the CPU goes down. If we were to free
10034 * the buffer, then the user would lose any trace that was in the
10035 * buffer. The memory will be removed once the "instance" is removed.
10037 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
10038 "trace/RB:preapre", trace_rb_cpu_prepare,
10041 goto out_free_cpumask;
10042 /* Used for event triggers */
10044 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
10046 goto out_rm_hp_state;
10048 if (trace_create_savedcmd() < 0)
10049 goto out_free_temp_buffer;
10051 /* TODO: make the number of buffers hot pluggable with CPUS */
10052 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
10053 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
10054 goto out_free_savedcmd;
10057 if (global_trace.buffer_disabled)
10060 if (trace_boot_clock) {
10061 ret = tracing_set_clock(&global_trace, trace_boot_clock);
10063 pr_warn("Trace clock %s not defined, going back to default\n",
10068 * register_tracer() might reference current_trace, so it
10069 * needs to be set before we register anything. This is
10070 * just a bootstrap of current_trace anyway.
10072 global_trace.current_trace = &nop_trace;
10074 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
10076 ftrace_init_global_array_ops(&global_trace);
10078 init_trace_flags_index(&global_trace);
10080 register_tracer(&nop_trace);
10082 /* Function tracing may start here (via kernel command line) */
10083 init_function_trace();
10085 /* All seems OK, enable tracing */
10086 tracing_disabled = 0;
10088 atomic_notifier_chain_register(&panic_notifier_list,
10089 &trace_panic_notifier);
10091 register_die_notifier(&trace_die_notifier);
10093 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
10095 INIT_LIST_HEAD(&global_trace.systems);
10096 INIT_LIST_HEAD(&global_trace.events);
10097 INIT_LIST_HEAD(&global_trace.hist_vars);
10098 INIT_LIST_HEAD(&global_trace.err_log);
10099 list_add(&global_trace.list, &ftrace_trace_arrays);
10101 apply_trace_boot_options();
10103 register_snapshot_cmd();
10110 free_saved_cmdlines_buffer(savedcmd);
10111 out_free_temp_buffer:
10112 ring_buffer_free(temp_buffer);
10114 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
10116 free_cpumask_var(global_trace.tracing_cpumask);
10117 out_free_buffer_mask:
10118 free_cpumask_var(tracing_buffer_mask);
10123 void __init early_trace_init(void)
10125 if (tracepoint_printk) {
10126 tracepoint_print_iter =
10127 kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
10128 if (MEM_FAIL(!tracepoint_print_iter,
10129 "Failed to allocate trace iterator\n"))
10130 tracepoint_printk = 0;
10132 static_key_enable(&tracepoint_printk_key.key);
10134 tracer_alloc_buffers();
10137 void __init trace_init(void)
10139 trace_event_init();
10142 __init static void clear_boot_tracer(void)
10145 * The default tracer at boot buffer is an init section.
10146 * This function is called in lateinit. If we did not
10147 * find the boot tracer, then clear it out, to prevent
10148 * later registration from accessing the buffer that is
10149 * about to be freed.
10151 if (!default_bootup_tracer)
10154 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10155 default_bootup_tracer);
10156 default_bootup_tracer = NULL;
10159 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
10160 __init static void tracing_set_default_clock(void)
10162 /* sched_clock_stable() is determined in late_initcall */
10163 if (!trace_boot_clock && !sched_clock_stable()) {
10164 if (security_locked_down(LOCKDOWN_TRACEFS)) {
10165 pr_warn("Can not set tracing clock due to lockdown\n");
10169 printk(KERN_WARNING
10170 "Unstable clock detected, switching default tracing clock to \"global\"\n"
10171 "If you want to keep using the local clock, then add:\n"
10172 " \"trace_clock=local\"\n"
10173 "on the kernel command line\n");
10174 tracing_set_clock(&global_trace, "global");
10178 static inline void tracing_set_default_clock(void) { }
10181 __init static int late_trace_init(void)
10183 if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10184 static_key_disable(&tracepoint_printk_key.key);
10185 tracepoint_printk = 0;
10188 tracing_set_default_clock();
10189 clear_boot_tracer();
10193 late_initcall_sync(late_trace_init);