1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer based function tracer
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/security.h>
21 #include <linux/seq_file.h>
22 #include <linux/notifier.h>
23 #include <linux/irqflags.h>
24 #include <linux/debugfs.h>
25 #include <linux/tracefs.h>
26 #include <linux/pagemap.h>
27 #include <linux/hardirq.h>
28 #include <linux/linkage.h>
29 #include <linux/uaccess.h>
30 #include <linux/vmalloc.h>
31 #include <linux/ftrace.h>
32 #include <linux/module.h>
33 #include <linux/percpu.h>
34 #include <linux/splice.h>
35 #include <linux/kdebug.h>
36 #include <linux/string.h>
37 #include <linux/mount.h>
38 #include <linux/rwsem.h>
39 #include <linux/slab.h>
40 #include <linux/ctype.h>
41 #include <linux/init.h>
42 #include <linux/poll.h>
43 #include <linux/nmi.h>
45 #include <linux/trace.h>
46 #include <linux/sched/clock.h>
47 #include <linux/sched/rt.h>
48 #include <linux/fsnotify.h>
49 #include <linux/irq_work.h>
50 #include <linux/workqueue.h>
53 #include "trace_output.h"
56 * On boot up, the ring buffer is set to the minimum size, so that
57 * we do not waste memory on systems that are not using tracing.
59 bool ring_buffer_expanded;
62 * We need to change this state when a selftest is running.
63 * A selftest will lurk into the ring-buffer to count the
64 * entries inserted during the selftest although some concurrent
65 * insertions into the ring-buffer such as trace_printk could occurred
66 * at the same time, giving false positive or negative results.
68 static bool __read_mostly tracing_selftest_running;
71 * If a tracer is running, we do not want to run SELFTEST.
73 bool __read_mostly tracing_selftest_disabled;
75 /* Pipe tracepoints to printk */
76 struct trace_iterator *tracepoint_print_iter;
77 int tracepoint_printk;
78 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
80 /* For tracers that don't implement custom flags */
81 static struct tracer_opt dummy_tracer_opt[] = {
86 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
92 * To prevent the comm cache from being overwritten when no
93 * tracing is active, only save the comm when a trace event
96 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
99 * Kill all tracing for good (never come back).
100 * It is initialized to 1 but will turn to zero if the initialization
101 * of the tracer is successful. But that is the only place that sets
104 static int tracing_disabled = 1;
106 cpumask_var_t __read_mostly tracing_buffer_mask;
109 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
112 * is set, then ftrace_dump is called. This will output the contents
113 * of the ftrace buffers to the console. This is very useful for
114 * capturing traces that lead to crashes and outputing it to a
117 * It is default off, but you can enable it with either specifying
118 * "ftrace_dump_on_oops" in the kernel command line, or setting
119 * /proc/sys/kernel/ftrace_dump_on_oops
120 * Set 1 if you want to dump buffers of all CPUs
121 * Set 2 if you want to dump the buffer of the CPU that triggered oops
124 enum ftrace_dump_mode ftrace_dump_on_oops;
126 /* When set, tracing will stop when a WARN*() is hit */
127 int __disable_trace_on_warning;
129 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
130 /* Map of enums to their values, for "eval_map" file */
131 struct trace_eval_map_head {
133 unsigned long length;
136 union trace_eval_map_item;
138 struct trace_eval_map_tail {
140 * "end" is first and points to NULL as it must be different
141 * than "mod" or "eval_string"
143 union trace_eval_map_item *next;
144 const char *end; /* points to NULL */
147 static DEFINE_MUTEX(trace_eval_mutex);
150 * The trace_eval_maps are saved in an array with two extra elements,
151 * one at the beginning, and one at the end. The beginning item contains
152 * the count of the saved maps (head.length), and the module they
153 * belong to if not built in (head.mod). The ending item contains a
154 * pointer to the next array of saved eval_map items.
156 union trace_eval_map_item {
157 struct trace_eval_map map;
158 struct trace_eval_map_head head;
159 struct trace_eval_map_tail tail;
162 static union trace_eval_map_item *trace_eval_maps;
163 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
165 int tracing_set_tracer(struct trace_array *tr, const char *buf);
166 static void ftrace_trace_userstack(struct trace_buffer *buffer,
167 unsigned long flags, int pc);
169 #define MAX_TRACER_SIZE 100
170 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
171 static char *default_bootup_tracer;
173 static bool allocate_snapshot;
175 static int __init set_cmdline_ftrace(char *str)
177 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
178 default_bootup_tracer = bootup_tracer_buf;
179 /* We are using ftrace early, expand it */
180 ring_buffer_expanded = true;
183 __setup("ftrace=", set_cmdline_ftrace);
185 static int __init set_ftrace_dump_on_oops(char *str)
187 if (*str++ != '=' || !*str) {
188 ftrace_dump_on_oops = DUMP_ALL;
192 if (!strcmp("orig_cpu", str)) {
193 ftrace_dump_on_oops = DUMP_ORIG;
199 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
201 static int __init stop_trace_on_warning(char *str)
203 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
204 __disable_trace_on_warning = 1;
207 __setup("traceoff_on_warning", stop_trace_on_warning);
209 static int __init boot_alloc_snapshot(char *str)
211 allocate_snapshot = true;
212 /* We also need the main ring buffer expanded */
213 ring_buffer_expanded = true;
216 __setup("alloc_snapshot", boot_alloc_snapshot);
219 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
221 static int __init set_trace_boot_options(char *str)
223 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
226 __setup("trace_options=", set_trace_boot_options);
228 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
229 static char *trace_boot_clock __initdata;
231 static int __init set_trace_boot_clock(char *str)
233 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
234 trace_boot_clock = trace_boot_clock_buf;
237 __setup("trace_clock=", set_trace_boot_clock);
239 static int __init set_tracepoint_printk(char *str)
241 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
242 tracepoint_printk = 1;
245 __setup("tp_printk", set_tracepoint_printk);
247 unsigned long long ns2usecs(u64 nsec)
254 /* trace_flags holds trace_options default values */
255 #define TRACE_DEFAULT_FLAGS \
256 (FUNCTION_DEFAULT_FLAGS | \
257 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
258 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
259 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
260 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
262 /* trace_options that are only supported by global_trace */
263 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
264 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
266 /* trace_flags that are default zero for instances */
267 #define ZEROED_TRACE_FLAGS \
268 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
271 * The global_trace is the descriptor that holds the top-level tracing
272 * buffers for the live tracing.
274 static struct trace_array global_trace = {
275 .trace_flags = TRACE_DEFAULT_FLAGS,
278 LIST_HEAD(ftrace_trace_arrays);
280 int trace_array_get(struct trace_array *this_tr)
282 struct trace_array *tr;
285 mutex_lock(&trace_types_lock);
286 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
293 mutex_unlock(&trace_types_lock);
298 static void __trace_array_put(struct trace_array *this_tr)
300 WARN_ON(!this_tr->ref);
305 * trace_array_put - Decrement the reference counter for this trace array.
307 * NOTE: Use this when we no longer need the trace array returned by
308 * trace_array_get_by_name(). This ensures the trace array can be later
312 void trace_array_put(struct trace_array *this_tr)
317 mutex_lock(&trace_types_lock);
318 __trace_array_put(this_tr);
319 mutex_unlock(&trace_types_lock);
321 EXPORT_SYMBOL_GPL(trace_array_put);
323 int tracing_check_open_get_tr(struct trace_array *tr)
327 ret = security_locked_down(LOCKDOWN_TRACEFS);
331 if (tracing_disabled)
334 if (tr && trace_array_get(tr) < 0)
340 int call_filter_check_discard(struct trace_event_call *call, void *rec,
341 struct trace_buffer *buffer,
342 struct ring_buffer_event *event)
344 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
345 !filter_match_preds(call->filter, rec)) {
346 __trace_event_discard_commit(buffer, event);
353 void trace_free_pid_list(struct trace_pid_list *pid_list)
355 vfree(pid_list->pids);
360 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
361 * @filtered_pids: The list of pids to check
362 * @search_pid: The PID to find in @filtered_pids
364 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
367 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
370 * If pid_max changed after filtered_pids was created, we
371 * by default ignore all pids greater than the previous pid_max.
373 if (search_pid >= filtered_pids->pid_max)
376 return test_bit(search_pid, filtered_pids->pids);
380 * trace_ignore_this_task - should a task be ignored for tracing
381 * @filtered_pids: The list of pids to check
382 * @task: The task that should be ignored if not filtered
384 * Checks if @task should be traced or not from @filtered_pids.
385 * Returns true if @task should *NOT* be traced.
386 * Returns false if @task should be traced.
389 trace_ignore_this_task(struct trace_pid_list *filtered_pids,
390 struct trace_pid_list *filtered_no_pids,
391 struct task_struct *task)
394 * If filterd_no_pids is not empty, and the task's pid is listed
395 * in filtered_no_pids, then return true.
396 * Otherwise, if filtered_pids is empty, that means we can
397 * trace all tasks. If it has content, then only trace pids
398 * within filtered_pids.
401 return (filtered_pids &&
402 !trace_find_filtered_pid(filtered_pids, task->pid)) ||
404 trace_find_filtered_pid(filtered_no_pids, task->pid));
408 * trace_filter_add_remove_task - Add or remove a task from a pid_list
409 * @pid_list: The list to modify
410 * @self: The current task for fork or NULL for exit
411 * @task: The task to add or remove
413 * If adding a task, if @self is defined, the task is only added if @self
414 * is also included in @pid_list. This happens on fork and tasks should
415 * only be added when the parent is listed. If @self is NULL, then the
416 * @task pid will be removed from the list, which would happen on exit
419 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
420 struct task_struct *self,
421 struct task_struct *task)
426 /* For forks, we only add if the forking task is listed */
428 if (!trace_find_filtered_pid(pid_list, self->pid))
432 /* Sorry, but we don't support pid_max changing after setting */
433 if (task->pid >= pid_list->pid_max)
436 /* "self" is set for forks, and NULL for exits */
438 set_bit(task->pid, pid_list->pids);
440 clear_bit(task->pid, pid_list->pids);
444 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
445 * @pid_list: The pid list to show
446 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
447 * @pos: The position of the file
449 * This is used by the seq_file "next" operation to iterate the pids
450 * listed in a trace_pid_list structure.
452 * Returns the pid+1 as we want to display pid of zero, but NULL would
453 * stop the iteration.
455 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
457 unsigned long pid = (unsigned long)v;
461 /* pid already is +1 of the actual prevous bit */
462 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
464 /* Return pid + 1 to allow zero to be represented */
465 if (pid < pid_list->pid_max)
466 return (void *)(pid + 1);
472 * trace_pid_start - Used for seq_file to start reading pid lists
473 * @pid_list: The pid list to show
474 * @pos: The position of the file
476 * This is used by seq_file "start" operation to start the iteration
479 * Returns the pid+1 as we want to display pid of zero, but NULL would
480 * stop the iteration.
482 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
487 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
488 if (pid >= pid_list->pid_max)
491 /* Return pid + 1 so that zero can be the exit value */
492 for (pid++; pid && l < *pos;
493 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
499 * trace_pid_show - show the current pid in seq_file processing
500 * @m: The seq_file structure to write into
501 * @v: A void pointer of the pid (+1) value to display
503 * Can be directly used by seq_file operations to display the current
506 int trace_pid_show(struct seq_file *m, void *v)
508 unsigned long pid = (unsigned long)v - 1;
510 seq_printf(m, "%lu\n", pid);
514 /* 128 should be much more than enough */
515 #define PID_BUF_SIZE 127
517 int trace_pid_write(struct trace_pid_list *filtered_pids,
518 struct trace_pid_list **new_pid_list,
519 const char __user *ubuf, size_t cnt)
521 struct trace_pid_list *pid_list;
522 struct trace_parser parser;
530 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
534 * Always recreate a new array. The write is an all or nothing
535 * operation. Always create a new array when adding new pids by
536 * the user. If the operation fails, then the current list is
539 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
541 trace_parser_put(&parser);
545 pid_list->pid_max = READ_ONCE(pid_max);
547 /* Only truncating will shrink pid_max */
548 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
549 pid_list->pid_max = filtered_pids->pid_max;
551 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
552 if (!pid_list->pids) {
553 trace_parser_put(&parser);
559 /* copy the current bits to the new max */
560 for_each_set_bit(pid, filtered_pids->pids,
561 filtered_pids->pid_max) {
562 set_bit(pid, pid_list->pids);
571 ret = trace_get_user(&parser, ubuf, cnt, &pos);
572 if (ret < 0 || !trace_parser_loaded(&parser))
580 if (kstrtoul(parser.buffer, 0, &val))
582 if (val >= pid_list->pid_max)
587 set_bit(pid, pid_list->pids);
590 trace_parser_clear(&parser);
593 trace_parser_put(&parser);
596 trace_free_pid_list(pid_list);
601 /* Cleared the list of pids */
602 trace_free_pid_list(pid_list);
607 *new_pid_list = pid_list;
612 static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
616 /* Early boot up does not have a buffer yet */
618 return trace_clock_local();
620 ts = ring_buffer_time_stamp(buf->buffer, cpu);
621 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
626 u64 ftrace_now(int cpu)
628 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
632 * tracing_is_enabled - Show if global_trace has been disabled
634 * Shows if the global trace has been enabled or not. It uses the
635 * mirror flag "buffer_disabled" to be used in fast paths such as for
636 * the irqsoff tracer. But it may be inaccurate due to races. If you
637 * need to know the accurate state, use tracing_is_on() which is a little
638 * slower, but accurate.
640 int tracing_is_enabled(void)
643 * For quick access (irqsoff uses this in fast path), just
644 * return the mirror variable of the state of the ring buffer.
645 * It's a little racy, but we don't really care.
648 return !global_trace.buffer_disabled;
652 * trace_buf_size is the size in bytes that is allocated
653 * for a buffer. Note, the number of bytes is always rounded
656 * This number is purposely set to a low number of 16384.
657 * If the dump on oops happens, it will be much appreciated
658 * to not have to wait for all that output. Anyway this can be
659 * boot time and run time configurable.
661 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
663 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
665 /* trace_types holds a link list of available tracers. */
666 static struct tracer *trace_types __read_mostly;
669 * trace_types_lock is used to protect the trace_types list.
671 DEFINE_MUTEX(trace_types_lock);
674 * serialize the access of the ring buffer
676 * ring buffer serializes readers, but it is low level protection.
677 * The validity of the events (which returns by ring_buffer_peek() ..etc)
678 * are not protected by ring buffer.
680 * The content of events may become garbage if we allow other process consumes
681 * these events concurrently:
682 * A) the page of the consumed events may become a normal page
683 * (not reader page) in ring buffer, and this page will be rewrited
684 * by events producer.
685 * B) The page of the consumed events may become a page for splice_read,
686 * and this page will be returned to system.
688 * These primitives allow multi process access to different cpu ring buffer
691 * These primitives don't distinguish read-only and read-consume access.
692 * Multi read-only access are also serialized.
696 static DECLARE_RWSEM(all_cpu_access_lock);
697 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
699 static inline void trace_access_lock(int cpu)
701 if (cpu == RING_BUFFER_ALL_CPUS) {
702 /* gain it for accessing the whole ring buffer. */
703 down_write(&all_cpu_access_lock);
705 /* gain it for accessing a cpu ring buffer. */
707 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
708 down_read(&all_cpu_access_lock);
710 /* Secondly block other access to this @cpu ring buffer. */
711 mutex_lock(&per_cpu(cpu_access_lock, cpu));
715 static inline void trace_access_unlock(int cpu)
717 if (cpu == RING_BUFFER_ALL_CPUS) {
718 up_write(&all_cpu_access_lock);
720 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
721 up_read(&all_cpu_access_lock);
725 static inline void trace_access_lock_init(void)
729 for_each_possible_cpu(cpu)
730 mutex_init(&per_cpu(cpu_access_lock, cpu));
735 static DEFINE_MUTEX(access_lock);
737 static inline void trace_access_lock(int cpu)
740 mutex_lock(&access_lock);
743 static inline void trace_access_unlock(int cpu)
746 mutex_unlock(&access_lock);
749 static inline void trace_access_lock_init(void)
755 #ifdef CONFIG_STACKTRACE
756 static void __ftrace_trace_stack(struct trace_buffer *buffer,
758 int skip, int pc, struct pt_regs *regs);
759 static inline void ftrace_trace_stack(struct trace_array *tr,
760 struct trace_buffer *buffer,
762 int skip, int pc, struct pt_regs *regs);
765 static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
767 int skip, int pc, struct pt_regs *regs)
770 static inline void ftrace_trace_stack(struct trace_array *tr,
771 struct trace_buffer *buffer,
773 int skip, int pc, struct pt_regs *regs)
779 static __always_inline void
780 trace_event_setup(struct ring_buffer_event *event,
781 int type, unsigned long flags, int pc)
783 struct trace_entry *ent = ring_buffer_event_data(event);
785 tracing_generic_entry_update(ent, type, flags, pc);
788 static __always_inline struct ring_buffer_event *
789 __trace_buffer_lock_reserve(struct trace_buffer *buffer,
792 unsigned long flags, int pc)
794 struct ring_buffer_event *event;
796 event = ring_buffer_lock_reserve(buffer, len);
798 trace_event_setup(event, type, flags, pc);
803 void tracer_tracing_on(struct trace_array *tr)
805 if (tr->array_buffer.buffer)
806 ring_buffer_record_on(tr->array_buffer.buffer);
808 * This flag is looked at when buffers haven't been allocated
809 * yet, or by some tracers (like irqsoff), that just want to
810 * know if the ring buffer has been disabled, but it can handle
811 * races of where it gets disabled but we still do a record.
812 * As the check is in the fast path of the tracers, it is more
813 * important to be fast than accurate.
815 tr->buffer_disabled = 0;
816 /* Make the flag seen by readers */
821 * tracing_on - enable tracing buffers
823 * This function enables tracing buffers that may have been
824 * disabled with tracing_off.
826 void tracing_on(void)
828 tracer_tracing_on(&global_trace);
830 EXPORT_SYMBOL_GPL(tracing_on);
833 static __always_inline void
834 __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
836 __this_cpu_write(trace_taskinfo_save, true);
838 /* If this is the temp buffer, we need to commit fully */
839 if (this_cpu_read(trace_buffered_event) == event) {
840 /* Length is in event->array[0] */
841 ring_buffer_write(buffer, event->array[0], &event->array[1]);
842 /* Release the temp buffer */
843 this_cpu_dec(trace_buffered_event_cnt);
845 ring_buffer_unlock_commit(buffer, event);
849 * __trace_puts - write a constant string into the trace buffer.
850 * @ip: The address of the caller
851 * @str: The constant string to write
852 * @size: The size of the string.
854 int __trace_puts(unsigned long ip, const char *str, int size)
856 struct ring_buffer_event *event;
857 struct trace_buffer *buffer;
858 struct print_entry *entry;
859 unsigned long irq_flags;
863 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
866 pc = preempt_count();
868 if (unlikely(tracing_selftest_running || tracing_disabled))
871 alloc = sizeof(*entry) + size + 2; /* possible \n added */
873 local_save_flags(irq_flags);
874 buffer = global_trace.array_buffer.buffer;
875 ring_buffer_nest_start(buffer);
876 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
883 entry = ring_buffer_event_data(event);
886 memcpy(&entry->buf, str, size);
888 /* Add a newline if necessary */
889 if (entry->buf[size - 1] != '\n') {
890 entry->buf[size] = '\n';
891 entry->buf[size + 1] = '\0';
893 entry->buf[size] = '\0';
895 __buffer_unlock_commit(buffer, event);
896 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
898 ring_buffer_nest_end(buffer);
901 EXPORT_SYMBOL_GPL(__trace_puts);
904 * __trace_bputs - write the pointer to a constant string into trace buffer
905 * @ip: The address of the caller
906 * @str: The constant string to write to the buffer to
908 int __trace_bputs(unsigned long ip, const char *str)
910 struct ring_buffer_event *event;
911 struct trace_buffer *buffer;
912 struct bputs_entry *entry;
913 unsigned long irq_flags;
914 int size = sizeof(struct bputs_entry);
918 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
921 pc = preempt_count();
923 if (unlikely(tracing_selftest_running || tracing_disabled))
926 local_save_flags(irq_flags);
927 buffer = global_trace.array_buffer.buffer;
929 ring_buffer_nest_start(buffer);
930 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
935 entry = ring_buffer_event_data(event);
939 __buffer_unlock_commit(buffer, event);
940 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
944 ring_buffer_nest_end(buffer);
947 EXPORT_SYMBOL_GPL(__trace_bputs);
949 #ifdef CONFIG_TRACER_SNAPSHOT
950 void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data)
952 struct tracer *tracer = tr->current_trace;
956 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
957 internal_trace_puts("*** snapshot is being ignored ***\n");
961 if (!tr->allocated_snapshot) {
962 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
963 internal_trace_puts("*** stopping trace here! ***\n");
968 /* Note, snapshot can not be used when the tracer uses it */
969 if (tracer->use_max_tr) {
970 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
971 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
975 local_irq_save(flags);
976 update_max_tr(tr, current, smp_processor_id(), cond_data);
977 local_irq_restore(flags);
980 void tracing_snapshot_instance(struct trace_array *tr)
982 tracing_snapshot_instance_cond(tr, NULL);
986 * tracing_snapshot - take a snapshot of the current buffer.
988 * This causes a swap between the snapshot buffer and the current live
989 * tracing buffer. You can use this to take snapshots of the live
990 * trace when some condition is triggered, but continue to trace.
992 * Note, make sure to allocate the snapshot with either
993 * a tracing_snapshot_alloc(), or by doing it manually
994 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
996 * If the snapshot buffer is not allocated, it will stop tracing.
997 * Basically making a permanent snapshot.
999 void tracing_snapshot(void)
1001 struct trace_array *tr = &global_trace;
1003 tracing_snapshot_instance(tr);
1005 EXPORT_SYMBOL_GPL(tracing_snapshot);
1008 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1009 * @tr: The tracing instance to snapshot
1010 * @cond_data: The data to be tested conditionally, and possibly saved
1012 * This is the same as tracing_snapshot() except that the snapshot is
1013 * conditional - the snapshot will only happen if the
1014 * cond_snapshot.update() implementation receiving the cond_data
1015 * returns true, which means that the trace array's cond_snapshot
1016 * update() operation used the cond_data to determine whether the
1017 * snapshot should be taken, and if it was, presumably saved it along
1018 * with the snapshot.
1020 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1022 tracing_snapshot_instance_cond(tr, cond_data);
1024 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1027 * tracing_snapshot_cond_data - get the user data associated with a snapshot
1028 * @tr: The tracing instance
1030 * When the user enables a conditional snapshot using
1031 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1032 * with the snapshot. This accessor is used to retrieve it.
1034 * Should not be called from cond_snapshot.update(), since it takes
1035 * the tr->max_lock lock, which the code calling
1036 * cond_snapshot.update() has already done.
1038 * Returns the cond_data associated with the trace array's snapshot.
1040 void *tracing_cond_snapshot_data(struct trace_array *tr)
1042 void *cond_data = NULL;
1044 arch_spin_lock(&tr->max_lock);
1046 if (tr->cond_snapshot)
1047 cond_data = tr->cond_snapshot->cond_data;
1049 arch_spin_unlock(&tr->max_lock);
1053 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1055 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1056 struct array_buffer *size_buf, int cpu_id);
1057 static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1059 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1063 if (!tr->allocated_snapshot) {
1065 /* allocate spare buffer */
1066 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1067 &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1071 tr->allocated_snapshot = true;
1077 static void free_snapshot(struct trace_array *tr)
1080 * We don't free the ring buffer. instead, resize it because
1081 * The max_tr ring buffer has some state (e.g. ring->clock) and
1082 * we want preserve it.
1084 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1085 set_buffer_entries(&tr->max_buffer, 1);
1086 tracing_reset_online_cpus(&tr->max_buffer);
1087 tr->allocated_snapshot = false;
1091 * tracing_alloc_snapshot - allocate snapshot buffer.
1093 * This only allocates the snapshot buffer if it isn't already
1094 * allocated - it doesn't also take a snapshot.
1096 * This is meant to be used in cases where the snapshot buffer needs
1097 * to be set up for events that can't sleep but need to be able to
1098 * trigger a snapshot.
1100 int tracing_alloc_snapshot(void)
1102 struct trace_array *tr = &global_trace;
1105 ret = tracing_alloc_snapshot_instance(tr);
1110 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1113 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1115 * This is similar to tracing_snapshot(), but it will allocate the
1116 * snapshot buffer if it isn't already allocated. Use this only
1117 * where it is safe to sleep, as the allocation may sleep.
1119 * This causes a swap between the snapshot buffer and the current live
1120 * tracing buffer. You can use this to take snapshots of the live
1121 * trace when some condition is triggered, but continue to trace.
1123 void tracing_snapshot_alloc(void)
1127 ret = tracing_alloc_snapshot();
1133 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1136 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1137 * @tr: The tracing instance
1138 * @cond_data: User data to associate with the snapshot
1139 * @update: Implementation of the cond_snapshot update function
1141 * Check whether the conditional snapshot for the given instance has
1142 * already been enabled, or if the current tracer is already using a
1143 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1144 * save the cond_data and update function inside.
1146 * Returns 0 if successful, error otherwise.
1148 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1149 cond_update_fn_t update)
1151 struct cond_snapshot *cond_snapshot;
1154 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1158 cond_snapshot->cond_data = cond_data;
1159 cond_snapshot->update = update;
1161 mutex_lock(&trace_types_lock);
1163 ret = tracing_alloc_snapshot_instance(tr);
1167 if (tr->current_trace->use_max_tr) {
1173 * The cond_snapshot can only change to NULL without the
1174 * trace_types_lock. We don't care if we race with it going
1175 * to NULL, but we want to make sure that it's not set to
1176 * something other than NULL when we get here, which we can
1177 * do safely with only holding the trace_types_lock and not
1178 * having to take the max_lock.
1180 if (tr->cond_snapshot) {
1185 arch_spin_lock(&tr->max_lock);
1186 tr->cond_snapshot = cond_snapshot;
1187 arch_spin_unlock(&tr->max_lock);
1189 mutex_unlock(&trace_types_lock);
1194 mutex_unlock(&trace_types_lock);
1195 kfree(cond_snapshot);
1198 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1201 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1202 * @tr: The tracing instance
1204 * Check whether the conditional snapshot for the given instance is
1205 * enabled; if so, free the cond_snapshot associated with it,
1206 * otherwise return -EINVAL.
1208 * Returns 0 if successful, error otherwise.
1210 int tracing_snapshot_cond_disable(struct trace_array *tr)
1214 arch_spin_lock(&tr->max_lock);
1216 if (!tr->cond_snapshot)
1219 kfree(tr->cond_snapshot);
1220 tr->cond_snapshot = NULL;
1223 arch_spin_unlock(&tr->max_lock);
1227 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1229 void tracing_snapshot(void)
1231 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1233 EXPORT_SYMBOL_GPL(tracing_snapshot);
1234 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1236 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1238 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1239 int tracing_alloc_snapshot(void)
1241 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1244 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1245 void tracing_snapshot_alloc(void)
1250 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1251 void *tracing_cond_snapshot_data(struct trace_array *tr)
1255 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1256 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1260 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1261 int tracing_snapshot_cond_disable(struct trace_array *tr)
1265 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1266 #endif /* CONFIG_TRACER_SNAPSHOT */
1268 void tracer_tracing_off(struct trace_array *tr)
1270 if (tr->array_buffer.buffer)
1271 ring_buffer_record_off(tr->array_buffer.buffer);
1273 * This flag is looked at when buffers haven't been allocated
1274 * yet, or by some tracers (like irqsoff), that just want to
1275 * know if the ring buffer has been disabled, but it can handle
1276 * races of where it gets disabled but we still do a record.
1277 * As the check is in the fast path of the tracers, it is more
1278 * important to be fast than accurate.
1280 tr->buffer_disabled = 1;
1281 /* Make the flag seen by readers */
1286 * tracing_off - turn off tracing buffers
1288 * This function stops the tracing buffers from recording data.
1289 * It does not disable any overhead the tracers themselves may
1290 * be causing. This function simply causes all recording to
1291 * the ring buffers to fail.
1293 void tracing_off(void)
1295 tracer_tracing_off(&global_trace);
1297 EXPORT_SYMBOL_GPL(tracing_off);
1299 void disable_trace_on_warning(void)
1301 if (__disable_trace_on_warning)
1306 * tracer_tracing_is_on - show real state of ring buffer enabled
1307 * @tr : the trace array to know if ring buffer is enabled
1309 * Shows real state of the ring buffer if it is enabled or not.
1311 bool tracer_tracing_is_on(struct trace_array *tr)
1313 if (tr->array_buffer.buffer)
1314 return ring_buffer_record_is_on(tr->array_buffer.buffer);
1315 return !tr->buffer_disabled;
1319 * tracing_is_on - show state of ring buffers enabled
1321 int tracing_is_on(void)
1323 return tracer_tracing_is_on(&global_trace);
1325 EXPORT_SYMBOL_GPL(tracing_is_on);
1327 static int __init set_buf_size(char *str)
1329 unsigned long buf_size;
1333 buf_size = memparse(str, &str);
1334 /* nr_entries can not be zero */
1337 trace_buf_size = buf_size;
1340 __setup("trace_buf_size=", set_buf_size);
1342 static int __init set_tracing_thresh(char *str)
1344 unsigned long threshold;
1349 ret = kstrtoul(str, 0, &threshold);
1352 tracing_thresh = threshold * 1000;
1355 __setup("tracing_thresh=", set_tracing_thresh);
1357 unsigned long nsecs_to_usecs(unsigned long nsecs)
1359 return nsecs / 1000;
1363 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1364 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1365 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1366 * of strings in the order that the evals (enum) were defined.
1371 /* These must match the bit postions in trace_iterator_flags */
1372 static const char *trace_options[] = {
1380 int in_ns; /* is this clock in nanoseconds? */
1381 } trace_clocks[] = {
1382 { trace_clock_local, "local", 1 },
1383 { trace_clock_global, "global", 1 },
1384 { trace_clock_counter, "counter", 0 },
1385 { trace_clock_jiffies, "uptime", 0 },
1386 { trace_clock, "perf", 1 },
1387 { ktime_get_mono_fast_ns, "mono", 1 },
1388 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1389 { ktime_get_boot_fast_ns, "boot", 1 },
1393 bool trace_clock_in_ns(struct trace_array *tr)
1395 if (trace_clocks[tr->clock_id].in_ns)
1402 * trace_parser_get_init - gets the buffer for trace parser
1404 int trace_parser_get_init(struct trace_parser *parser, int size)
1406 memset(parser, 0, sizeof(*parser));
1408 parser->buffer = kmalloc(size, GFP_KERNEL);
1409 if (!parser->buffer)
1412 parser->size = size;
1417 * trace_parser_put - frees the buffer for trace parser
1419 void trace_parser_put(struct trace_parser *parser)
1421 kfree(parser->buffer);
1422 parser->buffer = NULL;
1426 * trace_get_user - reads the user input string separated by space
1427 * (matched by isspace(ch))
1429 * For each string found the 'struct trace_parser' is updated,
1430 * and the function returns.
1432 * Returns number of bytes read.
1434 * See kernel/trace/trace.h for 'struct trace_parser' details.
1436 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1437 size_t cnt, loff_t *ppos)
1444 trace_parser_clear(parser);
1446 ret = get_user(ch, ubuf++);
1454 * The parser is not finished with the last write,
1455 * continue reading the user input without skipping spaces.
1457 if (!parser->cont) {
1458 /* skip white space */
1459 while (cnt && isspace(ch)) {
1460 ret = get_user(ch, ubuf++);
1469 /* only spaces were written */
1470 if (isspace(ch) || !ch) {
1477 /* read the non-space input */
1478 while (cnt && !isspace(ch) && ch) {
1479 if (parser->idx < parser->size - 1)
1480 parser->buffer[parser->idx++] = ch;
1485 ret = get_user(ch, ubuf++);
1492 /* We either got finished input or we have to wait for another call. */
1493 if (isspace(ch) || !ch) {
1494 parser->buffer[parser->idx] = 0;
1495 parser->cont = false;
1496 } else if (parser->idx < parser->size - 1) {
1497 parser->cont = true;
1498 parser->buffer[parser->idx++] = ch;
1499 /* Make sure the parsed string always terminates with '\0'. */
1500 parser->buffer[parser->idx] = 0;
1513 /* TODO add a seq_buf_to_buffer() */
1514 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1518 if (trace_seq_used(s) <= s->seq.readpos)
1521 len = trace_seq_used(s) - s->seq.readpos;
1524 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1526 s->seq.readpos += cnt;
1530 unsigned long __read_mostly tracing_thresh;
1531 static const struct file_operations tracing_max_lat_fops;
1533 #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1534 defined(CONFIG_FSNOTIFY)
1536 static struct workqueue_struct *fsnotify_wq;
1538 static void latency_fsnotify_workfn(struct work_struct *work)
1540 struct trace_array *tr = container_of(work, struct trace_array,
1542 fsnotify(tr->d_max_latency->d_inode, FS_MODIFY,
1543 tr->d_max_latency->d_inode, FSNOTIFY_EVENT_INODE, NULL, 0);
1546 static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1548 struct trace_array *tr = container_of(iwork, struct trace_array,
1550 queue_work(fsnotify_wq, &tr->fsnotify_work);
1553 static void trace_create_maxlat_file(struct trace_array *tr,
1554 struct dentry *d_tracer)
1556 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1557 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1558 tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
1559 d_tracer, &tr->max_latency,
1560 &tracing_max_lat_fops);
1563 __init static int latency_fsnotify_init(void)
1565 fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1566 WQ_UNBOUND | WQ_HIGHPRI, 0);
1568 pr_err("Unable to allocate tr_max_lat_wq\n");
1574 late_initcall_sync(latency_fsnotify_init);
1576 void latency_fsnotify(struct trace_array *tr)
1581 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1582 * possible that we are called from __schedule() or do_idle(), which
1583 * could cause a deadlock.
1585 irq_work_queue(&tr->fsnotify_irqwork);
1589 * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1590 * defined(CONFIG_FSNOTIFY)
1594 #define trace_create_maxlat_file(tr, d_tracer) \
1595 trace_create_file("tracing_max_latency", 0644, d_tracer, \
1596 &tr->max_latency, &tracing_max_lat_fops)
1600 #ifdef CONFIG_TRACER_MAX_TRACE
1602 * Copy the new maximum trace into the separate maximum-trace
1603 * structure. (this way the maximum trace is permanently saved,
1604 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1607 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1609 struct array_buffer *trace_buf = &tr->array_buffer;
1610 struct array_buffer *max_buf = &tr->max_buffer;
1611 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1612 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1615 max_buf->time_start = data->preempt_timestamp;
1617 max_data->saved_latency = tr->max_latency;
1618 max_data->critical_start = data->critical_start;
1619 max_data->critical_end = data->critical_end;
1621 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1622 max_data->pid = tsk->pid;
1624 * If tsk == current, then use current_uid(), as that does not use
1625 * RCU. The irq tracer can be called out of RCU scope.
1628 max_data->uid = current_uid();
1630 max_data->uid = task_uid(tsk);
1632 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1633 max_data->policy = tsk->policy;
1634 max_data->rt_priority = tsk->rt_priority;
1636 /* record this tasks comm */
1637 tracing_record_cmdline(tsk);
1638 latency_fsnotify(tr);
1642 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1644 * @tsk: the task with the latency
1645 * @cpu: The cpu that initiated the trace.
1646 * @cond_data: User data associated with a conditional snapshot
1648 * Flip the buffers between the @tr and the max_tr and record information
1649 * about which task was the cause of this latency.
1652 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1658 WARN_ON_ONCE(!irqs_disabled());
1660 if (!tr->allocated_snapshot) {
1661 /* Only the nop tracer should hit this when disabling */
1662 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1666 arch_spin_lock(&tr->max_lock);
1668 /* Inherit the recordable setting from array_buffer */
1669 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1670 ring_buffer_record_on(tr->max_buffer.buffer);
1672 ring_buffer_record_off(tr->max_buffer.buffer);
1674 #ifdef CONFIG_TRACER_SNAPSHOT
1675 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1678 swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1680 __update_max_tr(tr, tsk, cpu);
1683 arch_spin_unlock(&tr->max_lock);
1687 * update_max_tr_single - only copy one trace over, and reset the rest
1689 * @tsk: task with the latency
1690 * @cpu: the cpu of the buffer to copy.
1692 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1695 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1702 WARN_ON_ONCE(!irqs_disabled());
1703 if (!tr->allocated_snapshot) {
1704 /* Only the nop tracer should hit this when disabling */
1705 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1709 arch_spin_lock(&tr->max_lock);
1711 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1713 if (ret == -EBUSY) {
1715 * We failed to swap the buffer due to a commit taking
1716 * place on this CPU. We fail to record, but we reset
1717 * the max trace buffer (no one writes directly to it)
1718 * and flag that it failed.
1720 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1721 "Failed to swap buffers due to commit in progress\n");
1724 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1726 __update_max_tr(tr, tsk, cpu);
1727 arch_spin_unlock(&tr->max_lock);
1729 #endif /* CONFIG_TRACER_MAX_TRACE */
1731 static int wait_on_pipe(struct trace_iterator *iter, int full)
1733 /* Iterators are static, they should be filled or empty */
1734 if (trace_buffer_iter(iter, iter->cpu_file))
1737 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1741 #ifdef CONFIG_FTRACE_STARTUP_TEST
1742 static bool selftests_can_run;
1744 struct trace_selftests {
1745 struct list_head list;
1746 struct tracer *type;
1749 static LIST_HEAD(postponed_selftests);
1751 static int save_selftest(struct tracer *type)
1753 struct trace_selftests *selftest;
1755 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1759 selftest->type = type;
1760 list_add(&selftest->list, &postponed_selftests);
1764 static int run_tracer_selftest(struct tracer *type)
1766 struct trace_array *tr = &global_trace;
1767 struct tracer *saved_tracer = tr->current_trace;
1770 if (!type->selftest || tracing_selftest_disabled)
1774 * If a tracer registers early in boot up (before scheduling is
1775 * initialized and such), then do not run its selftests yet.
1776 * Instead, run it a little later in the boot process.
1778 if (!selftests_can_run)
1779 return save_selftest(type);
1782 * Run a selftest on this tracer.
1783 * Here we reset the trace buffer, and set the current
1784 * tracer to be this tracer. The tracer can then run some
1785 * internal tracing to verify that everything is in order.
1786 * If we fail, we do not register this tracer.
1788 tracing_reset_online_cpus(&tr->array_buffer);
1790 tr->current_trace = type;
1792 #ifdef CONFIG_TRACER_MAX_TRACE
1793 if (type->use_max_tr) {
1794 /* If we expanded the buffers, make sure the max is expanded too */
1795 if (ring_buffer_expanded)
1796 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1797 RING_BUFFER_ALL_CPUS);
1798 tr->allocated_snapshot = true;
1802 /* the test is responsible for initializing and enabling */
1803 pr_info("Testing tracer %s: ", type->name);
1804 ret = type->selftest(type, tr);
1805 /* the test is responsible for resetting too */
1806 tr->current_trace = saved_tracer;
1808 printk(KERN_CONT "FAILED!\n");
1809 /* Add the warning after printing 'FAILED' */
1813 /* Only reset on passing, to avoid touching corrupted buffers */
1814 tracing_reset_online_cpus(&tr->array_buffer);
1816 #ifdef CONFIG_TRACER_MAX_TRACE
1817 if (type->use_max_tr) {
1818 tr->allocated_snapshot = false;
1820 /* Shrink the max buffer again */
1821 if (ring_buffer_expanded)
1822 ring_buffer_resize(tr->max_buffer.buffer, 1,
1823 RING_BUFFER_ALL_CPUS);
1827 printk(KERN_CONT "PASSED\n");
1831 static __init int init_trace_selftests(void)
1833 struct trace_selftests *p, *n;
1834 struct tracer *t, **last;
1837 selftests_can_run = true;
1839 mutex_lock(&trace_types_lock);
1841 if (list_empty(&postponed_selftests))
1844 pr_info("Running postponed tracer tests:\n");
1846 tracing_selftest_running = true;
1847 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1848 /* This loop can take minutes when sanitizers are enabled, so
1849 * lets make sure we allow RCU processing.
1852 ret = run_tracer_selftest(p->type);
1853 /* If the test fails, then warn and remove from available_tracers */
1855 WARN(1, "tracer: %s failed selftest, disabling\n",
1857 last = &trace_types;
1858 for (t = trace_types; t; t = t->next) {
1869 tracing_selftest_running = false;
1872 mutex_unlock(&trace_types_lock);
1876 core_initcall(init_trace_selftests);
1878 static inline int run_tracer_selftest(struct tracer *type)
1882 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1884 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1886 static void __init apply_trace_boot_options(void);
1889 * register_tracer - register a tracer with the ftrace system.
1890 * @type: the plugin for the tracer
1892 * Register a new plugin tracer.
1894 int __init register_tracer(struct tracer *type)
1900 pr_info("Tracer must have a name\n");
1904 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1905 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1909 if (security_locked_down(LOCKDOWN_TRACEFS)) {
1910 pr_warn("Can not register tracer %s due to lockdown\n",
1915 mutex_lock(&trace_types_lock);
1917 tracing_selftest_running = true;
1919 for (t = trace_types; t; t = t->next) {
1920 if (strcmp(type->name, t->name) == 0) {
1922 pr_info("Tracer %s already registered\n",
1929 if (!type->set_flag)
1930 type->set_flag = &dummy_set_flag;
1932 /*allocate a dummy tracer_flags*/
1933 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1938 type->flags->val = 0;
1939 type->flags->opts = dummy_tracer_opt;
1941 if (!type->flags->opts)
1942 type->flags->opts = dummy_tracer_opt;
1944 /* store the tracer for __set_tracer_option */
1945 type->flags->trace = type;
1947 ret = run_tracer_selftest(type);
1951 type->next = trace_types;
1953 add_tracer_options(&global_trace, type);
1956 tracing_selftest_running = false;
1957 mutex_unlock(&trace_types_lock);
1959 if (ret || !default_bootup_tracer)
1962 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1965 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1966 /* Do we want this tracer to start on bootup? */
1967 tracing_set_tracer(&global_trace, type->name);
1968 default_bootup_tracer = NULL;
1970 apply_trace_boot_options();
1972 /* disable other selftests, since this will break it. */
1973 tracing_selftest_disabled = true;
1974 #ifdef CONFIG_FTRACE_STARTUP_TEST
1975 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1983 static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
1985 struct trace_buffer *buffer = buf->buffer;
1990 ring_buffer_record_disable(buffer);
1992 /* Make sure all commits have finished */
1994 ring_buffer_reset_cpu(buffer, cpu);
1996 ring_buffer_record_enable(buffer);
1999 void tracing_reset_online_cpus(struct array_buffer *buf)
2001 struct trace_buffer *buffer = buf->buffer;
2007 ring_buffer_record_disable(buffer);
2009 /* Make sure all commits have finished */
2012 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2014 for_each_online_cpu(cpu)
2015 ring_buffer_reset_cpu(buffer, cpu);
2017 ring_buffer_record_enable(buffer);
2020 /* Must have trace_types_lock held */
2021 void tracing_reset_all_online_cpus(void)
2023 struct trace_array *tr;
2025 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2026 if (!tr->clear_trace)
2028 tr->clear_trace = false;
2029 tracing_reset_online_cpus(&tr->array_buffer);
2030 #ifdef CONFIG_TRACER_MAX_TRACE
2031 tracing_reset_online_cpus(&tr->max_buffer);
2036 static int *tgid_map;
2038 #define SAVED_CMDLINES_DEFAULT 128
2039 #define NO_CMDLINE_MAP UINT_MAX
2040 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2041 struct saved_cmdlines_buffer {
2042 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2043 unsigned *map_cmdline_to_pid;
2044 unsigned cmdline_num;
2046 char *saved_cmdlines;
2048 static struct saved_cmdlines_buffer *savedcmd;
2050 /* temporary disable recording */
2051 static atomic_t trace_record_taskinfo_disabled __read_mostly;
2053 static inline char *get_saved_cmdlines(int idx)
2055 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2058 static inline void set_cmdline(int idx, const char *cmdline)
2060 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2063 static int allocate_cmdlines_buffer(unsigned int val,
2064 struct saved_cmdlines_buffer *s)
2066 s->map_cmdline_to_pid = kmalloc_array(val,
2067 sizeof(*s->map_cmdline_to_pid),
2069 if (!s->map_cmdline_to_pid)
2072 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2073 if (!s->saved_cmdlines) {
2074 kfree(s->map_cmdline_to_pid);
2079 s->cmdline_num = val;
2080 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2081 sizeof(s->map_pid_to_cmdline));
2082 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2083 val * sizeof(*s->map_cmdline_to_pid));
2088 static int trace_create_savedcmd(void)
2092 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2096 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2106 int is_tracing_stopped(void)
2108 return global_trace.stop_count;
2112 * tracing_start - quick start of the tracer
2114 * If tracing is enabled but was stopped by tracing_stop,
2115 * this will start the tracer back up.
2117 void tracing_start(void)
2119 struct trace_buffer *buffer;
2120 unsigned long flags;
2122 if (tracing_disabled)
2125 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2126 if (--global_trace.stop_count) {
2127 if (global_trace.stop_count < 0) {
2128 /* Someone screwed up their debugging */
2130 global_trace.stop_count = 0;
2135 /* Prevent the buffers from switching */
2136 arch_spin_lock(&global_trace.max_lock);
2138 buffer = global_trace.array_buffer.buffer;
2140 ring_buffer_record_enable(buffer);
2142 #ifdef CONFIG_TRACER_MAX_TRACE
2143 buffer = global_trace.max_buffer.buffer;
2145 ring_buffer_record_enable(buffer);
2148 arch_spin_unlock(&global_trace.max_lock);
2151 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2154 static void tracing_start_tr(struct trace_array *tr)
2156 struct trace_buffer *buffer;
2157 unsigned long flags;
2159 if (tracing_disabled)
2162 /* If global, we need to also start the max tracer */
2163 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2164 return tracing_start();
2166 raw_spin_lock_irqsave(&tr->start_lock, flags);
2168 if (--tr->stop_count) {
2169 if (tr->stop_count < 0) {
2170 /* Someone screwed up their debugging */
2177 buffer = tr->array_buffer.buffer;
2179 ring_buffer_record_enable(buffer);
2182 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2186 * tracing_stop - quick stop of the tracer
2188 * Light weight way to stop tracing. Use in conjunction with
2191 void tracing_stop(void)
2193 struct trace_buffer *buffer;
2194 unsigned long flags;
2196 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2197 if (global_trace.stop_count++)
2200 /* Prevent the buffers from switching */
2201 arch_spin_lock(&global_trace.max_lock);
2203 buffer = global_trace.array_buffer.buffer;
2205 ring_buffer_record_disable(buffer);
2207 #ifdef CONFIG_TRACER_MAX_TRACE
2208 buffer = global_trace.max_buffer.buffer;
2210 ring_buffer_record_disable(buffer);
2213 arch_spin_unlock(&global_trace.max_lock);
2216 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2219 static void tracing_stop_tr(struct trace_array *tr)
2221 struct trace_buffer *buffer;
2222 unsigned long flags;
2224 /* If global, we need to also stop the max tracer */
2225 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2226 return tracing_stop();
2228 raw_spin_lock_irqsave(&tr->start_lock, flags);
2229 if (tr->stop_count++)
2232 buffer = tr->array_buffer.buffer;
2234 ring_buffer_record_disable(buffer);
2237 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2240 static int trace_save_cmdline(struct task_struct *tsk)
2244 /* treat recording of idle task as a success */
2248 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
2252 * It's not the end of the world if we don't get
2253 * the lock, but we also don't want to spin
2254 * nor do we want to disable interrupts,
2255 * so if we miss here, then better luck next time.
2257 if (!arch_spin_trylock(&trace_cmdline_lock))
2260 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2261 if (idx == NO_CMDLINE_MAP) {
2262 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2265 * Check whether the cmdline buffer at idx has a pid
2266 * mapped. We are going to overwrite that entry so we
2267 * need to clear the map_pid_to_cmdline. Otherwise we
2268 * would read the new comm for the old pid.
2270 pid = savedcmd->map_cmdline_to_pid[idx];
2271 if (pid != NO_CMDLINE_MAP)
2272 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
2274 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2275 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
2277 savedcmd->cmdline_idx = idx;
2280 set_cmdline(idx, tsk->comm);
2282 arch_spin_unlock(&trace_cmdline_lock);
2287 static void __trace_find_cmdline(int pid, char comm[])
2292 strcpy(comm, "<idle>");
2296 if (WARN_ON_ONCE(pid < 0)) {
2297 strcpy(comm, "<XXX>");
2301 if (pid > PID_MAX_DEFAULT) {
2302 strcpy(comm, "<...>");
2306 map = savedcmd->map_pid_to_cmdline[pid];
2307 if (map != NO_CMDLINE_MAP)
2308 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2310 strcpy(comm, "<...>");
2313 void trace_find_cmdline(int pid, char comm[])
2316 arch_spin_lock(&trace_cmdline_lock);
2318 __trace_find_cmdline(pid, comm);
2320 arch_spin_unlock(&trace_cmdline_lock);
2324 int trace_find_tgid(int pid)
2326 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2329 return tgid_map[pid];
2332 static int trace_save_tgid(struct task_struct *tsk)
2334 /* treat recording of idle task as a success */
2338 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2341 tgid_map[tsk->pid] = tsk->tgid;
2345 static bool tracing_record_taskinfo_skip(int flags)
2347 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2349 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2351 if (!__this_cpu_read(trace_taskinfo_save))
2357 * tracing_record_taskinfo - record the task info of a task
2359 * @task: task to record
2360 * @flags: TRACE_RECORD_CMDLINE for recording comm
2361 * TRACE_RECORD_TGID for recording tgid
2363 void tracing_record_taskinfo(struct task_struct *task, int flags)
2367 if (tracing_record_taskinfo_skip(flags))
2371 * Record as much task information as possible. If some fail, continue
2372 * to try to record the others.
2374 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2375 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2377 /* If recording any information failed, retry again soon. */
2381 __this_cpu_write(trace_taskinfo_save, false);
2385 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2387 * @prev: previous task during sched_switch
2388 * @next: next task during sched_switch
2389 * @flags: TRACE_RECORD_CMDLINE for recording comm
2390 * TRACE_RECORD_TGID for recording tgid
2392 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2393 struct task_struct *next, int flags)
2397 if (tracing_record_taskinfo_skip(flags))
2401 * Record as much task information as possible. If some fail, continue
2402 * to try to record the others.
2404 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2405 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2406 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2407 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2409 /* If recording any information failed, retry again soon. */
2413 __this_cpu_write(trace_taskinfo_save, false);
2416 /* Helpers to record a specific task information */
2417 void tracing_record_cmdline(struct task_struct *task)
2419 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2422 void tracing_record_tgid(struct task_struct *task)
2424 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2428 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2429 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2430 * simplifies those functions and keeps them in sync.
2432 enum print_line_t trace_handle_return(struct trace_seq *s)
2434 return trace_seq_has_overflowed(s) ?
2435 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2437 EXPORT_SYMBOL_GPL(trace_handle_return);
2440 tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
2441 unsigned long flags, int pc)
2443 struct task_struct *tsk = current;
2445 entry->preempt_count = pc & 0xff;
2446 entry->pid = (tsk) ? tsk->pid : 0;
2449 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2450 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2452 TRACE_FLAG_IRQS_NOSUPPORT |
2454 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
2455 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2456 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2457 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2458 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2460 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2462 struct ring_buffer_event *
2463 trace_buffer_lock_reserve(struct trace_buffer *buffer,
2466 unsigned long flags, int pc)
2468 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2471 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2472 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2473 static int trace_buffered_event_ref;
2476 * trace_buffered_event_enable - enable buffering events
2478 * When events are being filtered, it is quicker to use a temporary
2479 * buffer to write the event data into if there's a likely chance
2480 * that it will not be committed. The discard of the ring buffer
2481 * is not as fast as committing, and is much slower than copying
2484 * When an event is to be filtered, allocate per cpu buffers to
2485 * write the event data into, and if the event is filtered and discarded
2486 * it is simply dropped, otherwise, the entire data is to be committed
2489 void trace_buffered_event_enable(void)
2491 struct ring_buffer_event *event;
2495 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2497 if (trace_buffered_event_ref++)
2500 for_each_tracing_cpu(cpu) {
2501 page = alloc_pages_node(cpu_to_node(cpu),
2502 GFP_KERNEL | __GFP_NORETRY, 0);
2506 event = page_address(page);
2507 memset(event, 0, sizeof(*event));
2509 per_cpu(trace_buffered_event, cpu) = event;
2512 if (cpu == smp_processor_id() &&
2513 this_cpu_read(trace_buffered_event) !=
2514 per_cpu(trace_buffered_event, cpu))
2521 trace_buffered_event_disable();
2524 static void enable_trace_buffered_event(void *data)
2526 /* Probably not needed, but do it anyway */
2528 this_cpu_dec(trace_buffered_event_cnt);
2531 static void disable_trace_buffered_event(void *data)
2533 this_cpu_inc(trace_buffered_event_cnt);
2537 * trace_buffered_event_disable - disable buffering events
2539 * When a filter is removed, it is faster to not use the buffered
2540 * events, and to commit directly into the ring buffer. Free up
2541 * the temp buffers when there are no more users. This requires
2542 * special synchronization with current events.
2544 void trace_buffered_event_disable(void)
2548 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2550 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2553 if (--trace_buffered_event_ref)
2557 /* For each CPU, set the buffer as used. */
2558 smp_call_function_many(tracing_buffer_mask,
2559 disable_trace_buffered_event, NULL, 1);
2562 /* Wait for all current users to finish */
2565 for_each_tracing_cpu(cpu) {
2566 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2567 per_cpu(trace_buffered_event, cpu) = NULL;
2570 * Make sure trace_buffered_event is NULL before clearing
2571 * trace_buffered_event_cnt.
2576 /* Do the work on each cpu */
2577 smp_call_function_many(tracing_buffer_mask,
2578 enable_trace_buffered_event, NULL, 1);
2582 static struct trace_buffer *temp_buffer;
2584 struct ring_buffer_event *
2585 trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2586 struct trace_event_file *trace_file,
2587 int type, unsigned long len,
2588 unsigned long flags, int pc)
2590 struct ring_buffer_event *entry;
2593 *current_rb = trace_file->tr->array_buffer.buffer;
2595 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2596 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2597 (entry = this_cpu_read(trace_buffered_event))) {
2598 /* Try to use the per cpu buffer first */
2599 val = this_cpu_inc_return(trace_buffered_event_cnt);
2601 trace_event_setup(entry, type, flags, pc);
2602 entry->array[0] = len;
2605 this_cpu_dec(trace_buffered_event_cnt);
2608 entry = __trace_buffer_lock_reserve(*current_rb,
2609 type, len, flags, pc);
2611 * If tracing is off, but we have triggers enabled
2612 * we still need to look at the event data. Use the temp_buffer
2613 * to store the trace event for the tigger to use. It's recusive
2614 * safe and will not be recorded anywhere.
2616 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2617 *current_rb = temp_buffer;
2618 entry = __trace_buffer_lock_reserve(*current_rb,
2619 type, len, flags, pc);
2623 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2625 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2626 static DEFINE_MUTEX(tracepoint_printk_mutex);
2628 static void output_printk(struct trace_event_buffer *fbuffer)
2630 struct trace_event_call *event_call;
2631 struct trace_event_file *file;
2632 struct trace_event *event;
2633 unsigned long flags;
2634 struct trace_iterator *iter = tracepoint_print_iter;
2636 /* We should never get here if iter is NULL */
2637 if (WARN_ON_ONCE(!iter))
2640 event_call = fbuffer->trace_file->event_call;
2641 if (!event_call || !event_call->event.funcs ||
2642 !event_call->event.funcs->trace)
2645 file = fbuffer->trace_file;
2646 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2647 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2648 !filter_match_preds(file->filter, fbuffer->entry)))
2651 event = &fbuffer->trace_file->event_call->event;
2653 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2654 trace_seq_init(&iter->seq);
2655 iter->ent = fbuffer->entry;
2656 event_call->event.funcs->trace(iter, 0, event);
2657 trace_seq_putc(&iter->seq, 0);
2658 printk("%s", iter->seq.buffer);
2660 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2663 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2664 void __user *buffer, size_t *lenp,
2667 int save_tracepoint_printk;
2670 mutex_lock(&tracepoint_printk_mutex);
2671 save_tracepoint_printk = tracepoint_printk;
2673 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2676 * This will force exiting early, as tracepoint_printk
2677 * is always zero when tracepoint_printk_iter is not allocated
2679 if (!tracepoint_print_iter)
2680 tracepoint_printk = 0;
2682 if (save_tracepoint_printk == tracepoint_printk)
2685 if (tracepoint_printk)
2686 static_key_enable(&tracepoint_printk_key.key);
2688 static_key_disable(&tracepoint_printk_key.key);
2691 mutex_unlock(&tracepoint_printk_mutex);
2696 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2698 if (static_key_false(&tracepoint_printk_key.key))
2699 output_printk(fbuffer);
2701 event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer,
2702 fbuffer->event, fbuffer->entry,
2703 fbuffer->flags, fbuffer->pc, fbuffer->regs);
2705 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2710 * trace_buffer_unlock_commit_regs()
2711 * trace_event_buffer_commit()
2712 * trace_event_raw_event_xxx()
2714 # define STACK_SKIP 3
2716 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2717 struct trace_buffer *buffer,
2718 struct ring_buffer_event *event,
2719 unsigned long flags, int pc,
2720 struct pt_regs *regs)
2722 __buffer_unlock_commit(buffer, event);
2725 * If regs is not set, then skip the necessary functions.
2726 * Note, we can still get here via blktrace, wakeup tracer
2727 * and mmiotrace, but that's ok if they lose a function or
2728 * two. They are not that meaningful.
2730 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2731 ftrace_trace_userstack(buffer, flags, pc);
2735 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2738 trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2739 struct ring_buffer_event *event)
2741 __buffer_unlock_commit(buffer, event);
2745 trace_process_export(struct trace_export *export,
2746 struct ring_buffer_event *event)
2748 struct trace_entry *entry;
2749 unsigned int size = 0;
2751 entry = ring_buffer_event_data(event);
2752 size = ring_buffer_event_length(event);
2753 export->write(export, entry, size);
2756 static DEFINE_MUTEX(ftrace_export_lock);
2758 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2760 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2762 static inline void ftrace_exports_enable(void)
2764 static_branch_enable(&ftrace_exports_enabled);
2767 static inline void ftrace_exports_disable(void)
2769 static_branch_disable(&ftrace_exports_enabled);
2772 static void ftrace_exports(struct ring_buffer_event *event)
2774 struct trace_export *export;
2776 preempt_disable_notrace();
2778 export = rcu_dereference_raw_check(ftrace_exports_list);
2780 trace_process_export(export, event);
2781 export = rcu_dereference_raw_check(export->next);
2784 preempt_enable_notrace();
2788 add_trace_export(struct trace_export **list, struct trace_export *export)
2790 rcu_assign_pointer(export->next, *list);
2792 * We are entering export into the list but another
2793 * CPU might be walking that list. We need to make sure
2794 * the export->next pointer is valid before another CPU sees
2795 * the export pointer included into the list.
2797 rcu_assign_pointer(*list, export);
2801 rm_trace_export(struct trace_export **list, struct trace_export *export)
2803 struct trace_export **p;
2805 for (p = list; *p != NULL; p = &(*p)->next)
2812 rcu_assign_pointer(*p, (*p)->next);
2818 add_ftrace_export(struct trace_export **list, struct trace_export *export)
2821 ftrace_exports_enable();
2823 add_trace_export(list, export);
2827 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2831 ret = rm_trace_export(list, export);
2833 ftrace_exports_disable();
2838 int register_ftrace_export(struct trace_export *export)
2840 if (WARN_ON_ONCE(!export->write))
2843 mutex_lock(&ftrace_export_lock);
2845 add_ftrace_export(&ftrace_exports_list, export);
2847 mutex_unlock(&ftrace_export_lock);
2851 EXPORT_SYMBOL_GPL(register_ftrace_export);
2853 int unregister_ftrace_export(struct trace_export *export)
2857 mutex_lock(&ftrace_export_lock);
2859 ret = rm_ftrace_export(&ftrace_exports_list, export);
2861 mutex_unlock(&ftrace_export_lock);
2865 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2868 trace_function(struct trace_array *tr,
2869 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2872 struct trace_event_call *call = &event_function;
2873 struct trace_buffer *buffer = tr->array_buffer.buffer;
2874 struct ring_buffer_event *event;
2875 struct ftrace_entry *entry;
2877 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2881 entry = ring_buffer_event_data(event);
2883 entry->parent_ip = parent_ip;
2885 if (!call_filter_check_discard(call, entry, buffer, event)) {
2886 if (static_branch_unlikely(&ftrace_exports_enabled))
2887 ftrace_exports(event);
2888 __buffer_unlock_commit(buffer, event);
2892 #ifdef CONFIG_STACKTRACE
2894 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2895 #define FTRACE_KSTACK_NESTING 4
2897 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2899 struct ftrace_stack {
2900 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2904 struct ftrace_stacks {
2905 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2908 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2909 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2911 static void __ftrace_trace_stack(struct trace_buffer *buffer,
2912 unsigned long flags,
2913 int skip, int pc, struct pt_regs *regs)
2915 struct trace_event_call *call = &event_kernel_stack;
2916 struct ring_buffer_event *event;
2917 unsigned int size, nr_entries;
2918 struct ftrace_stack *fstack;
2919 struct stack_entry *entry;
2923 * Add one, for this function and the call to save_stack_trace()
2924 * If regs is set, then these functions will not be in the way.
2926 #ifndef CONFIG_UNWINDER_ORC
2932 * Since events can happen in NMIs there's no safe way to
2933 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2934 * or NMI comes in, it will just have to use the default
2935 * FTRACE_STACK_SIZE.
2937 preempt_disable_notrace();
2939 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2941 /* This should never happen. If it does, yell once and skip */
2942 if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING))
2946 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2947 * interrupt will either see the value pre increment or post
2948 * increment. If the interrupt happens pre increment it will have
2949 * restored the counter when it returns. We just need a barrier to
2950 * keep gcc from moving things around.
2954 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2955 size = ARRAY_SIZE(fstack->calls);
2958 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2961 nr_entries = stack_trace_save(fstack->calls, size, skip);
2964 size = nr_entries * sizeof(unsigned long);
2965 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2966 sizeof(*entry) + size, flags, pc);
2969 entry = ring_buffer_event_data(event);
2971 memcpy(&entry->caller, fstack->calls, size);
2972 entry->size = nr_entries;
2974 if (!call_filter_check_discard(call, entry, buffer, event))
2975 __buffer_unlock_commit(buffer, event);
2978 /* Again, don't let gcc optimize things here */
2980 __this_cpu_dec(ftrace_stack_reserve);
2981 preempt_enable_notrace();
2985 static inline void ftrace_trace_stack(struct trace_array *tr,
2986 struct trace_buffer *buffer,
2987 unsigned long flags,
2988 int skip, int pc, struct pt_regs *regs)
2990 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2993 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2996 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2999 struct trace_buffer *buffer = tr->array_buffer.buffer;
3001 if (rcu_is_watching()) {
3002 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
3007 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
3008 * but if the above rcu_is_watching() failed, then the NMI
3009 * triggered someplace critical, and rcu_irq_enter() should
3010 * not be called from NMI.
3012 if (unlikely(in_nmi()))
3015 rcu_irq_enter_irqson();
3016 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
3017 rcu_irq_exit_irqson();
3021 * trace_dump_stack - record a stack back trace in the trace buffer
3022 * @skip: Number of functions to skip (helper handlers)
3024 void trace_dump_stack(int skip)
3026 unsigned long flags;
3028 if (tracing_disabled || tracing_selftest_running)
3031 local_save_flags(flags);
3033 #ifndef CONFIG_UNWINDER_ORC
3034 /* Skip 1 to skip this function. */
3037 __ftrace_trace_stack(global_trace.array_buffer.buffer,
3038 flags, skip, preempt_count(), NULL);
3040 EXPORT_SYMBOL_GPL(trace_dump_stack);
3042 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
3043 static DEFINE_PER_CPU(int, user_stack_count);
3046 ftrace_trace_userstack(struct trace_buffer *buffer, unsigned long flags, int pc)
3048 struct trace_event_call *call = &event_user_stack;
3049 struct ring_buffer_event *event;
3050 struct userstack_entry *entry;
3052 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
3056 * NMIs can not handle page faults, even with fix ups.
3057 * The save user stack can (and often does) fault.
3059 if (unlikely(in_nmi()))
3063 * prevent recursion, since the user stack tracing may
3064 * trigger other kernel events.
3067 if (__this_cpu_read(user_stack_count))
3070 __this_cpu_inc(user_stack_count);
3072 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3073 sizeof(*entry), flags, pc);
3075 goto out_drop_count;
3076 entry = ring_buffer_event_data(event);
3078 entry->tgid = current->tgid;
3079 memset(&entry->caller, 0, sizeof(entry->caller));
3081 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3082 if (!call_filter_check_discard(call, entry, buffer, event))
3083 __buffer_unlock_commit(buffer, event);
3086 __this_cpu_dec(user_stack_count);
3090 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
3091 static void ftrace_trace_userstack(struct trace_buffer *buffer,
3092 unsigned long flags, int pc)
3095 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3097 #endif /* CONFIG_STACKTRACE */
3099 /* created for use with alloc_percpu */
3100 struct trace_buffer_struct {
3102 char buffer[4][TRACE_BUF_SIZE];
3105 static struct trace_buffer_struct *trace_percpu_buffer;
3108 * Thise allows for lockless recording. If we're nested too deeply, then
3109 * this returns NULL.
3111 static char *get_trace_buf(void)
3113 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3115 if (!buffer || buffer->nesting >= 4)
3120 /* Interrupts must see nesting incremented before we use the buffer */
3122 return &buffer->buffer[buffer->nesting][0];
3125 static void put_trace_buf(void)
3127 /* Don't let the decrement of nesting leak before this */
3129 this_cpu_dec(trace_percpu_buffer->nesting);
3132 static int alloc_percpu_trace_buffer(void)
3134 struct trace_buffer_struct *buffers;
3136 buffers = alloc_percpu(struct trace_buffer_struct);
3137 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3140 trace_percpu_buffer = buffers;
3144 static int buffers_allocated;
3146 void trace_printk_init_buffers(void)
3148 if (buffers_allocated)
3151 if (alloc_percpu_trace_buffer())
3154 /* trace_printk() is for debug use only. Don't use it in production. */
3157 pr_warn("**********************************************************\n");
3158 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3160 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3162 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3163 pr_warn("** unsafe for production use. **\n");
3165 pr_warn("** If you see this message and you are not debugging **\n");
3166 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3168 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3169 pr_warn("**********************************************************\n");
3171 /* Expand the buffers to set size */
3172 tracing_update_buffers();
3174 buffers_allocated = 1;
3177 * trace_printk_init_buffers() can be called by modules.
3178 * If that happens, then we need to start cmdline recording
3179 * directly here. If the global_trace.buffer is already
3180 * allocated here, then this was called by module code.
3182 if (global_trace.array_buffer.buffer)
3183 tracing_start_cmdline_record();
3185 EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3187 void trace_printk_start_comm(void)
3189 /* Start tracing comms if trace printk is set */
3190 if (!buffers_allocated)
3192 tracing_start_cmdline_record();
3195 static void trace_printk_start_stop_comm(int enabled)
3197 if (!buffers_allocated)
3201 tracing_start_cmdline_record();
3203 tracing_stop_cmdline_record();
3207 * trace_vbprintk - write binary msg to tracing buffer
3208 * @ip: The address of the caller
3209 * @fmt: The string format to write to the buffer
3210 * @args: Arguments for @fmt
3212 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3214 struct trace_event_call *call = &event_bprint;
3215 struct ring_buffer_event *event;
3216 struct trace_buffer *buffer;
3217 struct trace_array *tr = &global_trace;
3218 struct bprint_entry *entry;
3219 unsigned long flags;
3221 int len = 0, size, pc;
3223 if (unlikely(tracing_selftest_running || tracing_disabled))
3226 /* Don't pollute graph traces with trace_vprintk internals */
3227 pause_graph_tracing();
3229 pc = preempt_count();
3230 preempt_disable_notrace();
3232 tbuffer = get_trace_buf();
3238 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3240 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3243 local_save_flags(flags);
3244 size = sizeof(*entry) + sizeof(u32) * len;
3245 buffer = tr->array_buffer.buffer;
3246 ring_buffer_nest_start(buffer);
3247 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3251 entry = ring_buffer_event_data(event);
3255 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3256 if (!call_filter_check_discard(call, entry, buffer, event)) {
3257 __buffer_unlock_commit(buffer, event);
3258 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
3262 ring_buffer_nest_end(buffer);
3267 preempt_enable_notrace();
3268 unpause_graph_tracing();
3272 EXPORT_SYMBOL_GPL(trace_vbprintk);
3276 __trace_array_vprintk(struct trace_buffer *buffer,
3277 unsigned long ip, const char *fmt, va_list args)
3279 struct trace_event_call *call = &event_print;
3280 struct ring_buffer_event *event;
3281 int len = 0, size, pc;
3282 struct print_entry *entry;
3283 unsigned long flags;
3286 if (tracing_disabled || tracing_selftest_running)
3289 /* Don't pollute graph traces with trace_vprintk internals */
3290 pause_graph_tracing();
3292 pc = preempt_count();
3293 preempt_disable_notrace();
3296 tbuffer = get_trace_buf();
3302 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3304 local_save_flags(flags);
3305 size = sizeof(*entry) + len + 1;
3306 ring_buffer_nest_start(buffer);
3307 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3311 entry = ring_buffer_event_data(event);
3314 memcpy(&entry->buf, tbuffer, len + 1);
3315 if (!call_filter_check_discard(call, entry, buffer, event)) {
3316 __buffer_unlock_commit(buffer, event);
3317 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3321 ring_buffer_nest_end(buffer);
3325 preempt_enable_notrace();
3326 unpause_graph_tracing();
3332 int trace_array_vprintk(struct trace_array *tr,
3333 unsigned long ip, const char *fmt, va_list args)
3335 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3339 int trace_array_printk(struct trace_array *tr,
3340 unsigned long ip, const char *fmt, ...)
3345 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3352 ret = trace_array_vprintk(tr, ip, fmt, ap);
3356 EXPORT_SYMBOL_GPL(trace_array_printk);
3359 int trace_array_printk_buf(struct trace_buffer *buffer,
3360 unsigned long ip, const char *fmt, ...)
3365 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3369 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3375 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3377 return trace_array_vprintk(&global_trace, ip, fmt, args);
3379 EXPORT_SYMBOL_GPL(trace_vprintk);
3381 static void trace_iterator_increment(struct trace_iterator *iter)
3383 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3387 ring_buffer_iter_advance(buf_iter);
3390 static struct trace_entry *
3391 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3392 unsigned long *lost_events)
3394 struct ring_buffer_event *event;
3395 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3398 event = ring_buffer_iter_peek(buf_iter, ts);
3400 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
3401 (unsigned long)-1 : 0;
3403 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3408 iter->ent_size = ring_buffer_event_length(event);
3409 return ring_buffer_event_data(event);
3415 static struct trace_entry *
3416 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3417 unsigned long *missing_events, u64 *ent_ts)
3419 struct trace_buffer *buffer = iter->array_buffer->buffer;
3420 struct trace_entry *ent, *next = NULL;
3421 unsigned long lost_events = 0, next_lost = 0;
3422 int cpu_file = iter->cpu_file;
3423 u64 next_ts = 0, ts;
3429 * If we are in a per_cpu trace file, don't bother by iterating over
3430 * all cpu and peek directly.
3432 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3433 if (ring_buffer_empty_cpu(buffer, cpu_file))
3435 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3437 *ent_cpu = cpu_file;
3442 for_each_tracing_cpu(cpu) {
3444 if (ring_buffer_empty_cpu(buffer, cpu))
3447 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3450 * Pick the entry with the smallest timestamp:
3452 if (ent && (!next || ts < next_ts)) {
3456 next_lost = lost_events;
3457 next_size = iter->ent_size;
3461 iter->ent_size = next_size;
3464 *ent_cpu = next_cpu;
3470 *missing_events = next_lost;
3475 #define STATIC_TEMP_BUF_SIZE 128
3476 static char static_temp_buf[STATIC_TEMP_BUF_SIZE];
3478 /* Find the next real entry, without updating the iterator itself */
3479 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3480 int *ent_cpu, u64 *ent_ts)
3482 /* __find_next_entry will reset ent_size */
3483 int ent_size = iter->ent_size;
3484 struct trace_entry *entry;
3487 * If called from ftrace_dump(), then the iter->temp buffer
3488 * will be the static_temp_buf and not created from kmalloc.
3489 * If the entry size is greater than the buffer, we can
3490 * not save it. Just return NULL in that case. This is only
3491 * used to add markers when two consecutive events' time
3492 * stamps have a large delta. See trace_print_lat_context()
3494 if (iter->temp == static_temp_buf &&
3495 STATIC_TEMP_BUF_SIZE < ent_size)
3499 * The __find_next_entry() may call peek_next_entry(), which may
3500 * call ring_buffer_peek() that may make the contents of iter->ent
3501 * undefined. Need to copy iter->ent now.
3503 if (iter->ent && iter->ent != iter->temp) {
3504 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
3505 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3507 iter->temp = kmalloc(iter->ent_size, GFP_KERNEL);
3511 memcpy(iter->temp, iter->ent, iter->ent_size);
3512 iter->temp_size = iter->ent_size;
3513 iter->ent = iter->temp;
3515 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3516 /* Put back the original ent_size */
3517 iter->ent_size = ent_size;
3522 /* Find the next real entry, and increment the iterator to the next entry */
3523 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3525 iter->ent = __find_next_entry(iter, &iter->cpu,
3526 &iter->lost_events, &iter->ts);
3529 trace_iterator_increment(iter);
3531 return iter->ent ? iter : NULL;
3534 static void trace_consume(struct trace_iterator *iter)
3536 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
3537 &iter->lost_events);
3540 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3542 struct trace_iterator *iter = m->private;
3546 WARN_ON_ONCE(iter->leftover);
3550 /* can't go backwards */
3555 ent = trace_find_next_entry_inc(iter);
3559 while (ent && iter->idx < i)
3560 ent = trace_find_next_entry_inc(iter);
3567 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3569 struct ring_buffer_event *event;
3570 struct ring_buffer_iter *buf_iter;
3571 unsigned long entries = 0;
3574 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
3576 buf_iter = trace_buffer_iter(iter, cpu);
3580 ring_buffer_iter_reset(buf_iter);
3583 * We could have the case with the max latency tracers
3584 * that a reset never took place on a cpu. This is evident
3585 * by the timestamp being before the start of the buffer.
3587 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3588 if (ts >= iter->array_buffer->time_start)
3591 ring_buffer_iter_advance(buf_iter);
3594 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
3598 * The current tracer is copied to avoid a global locking
3601 static void *s_start(struct seq_file *m, loff_t *pos)
3603 struct trace_iterator *iter = m->private;
3604 struct trace_array *tr = iter->tr;
3605 int cpu_file = iter->cpu_file;
3611 * copy the tracer to avoid using a global lock all around.
3612 * iter->trace is a copy of current_trace, the pointer to the
3613 * name may be used instead of a strcmp(), as iter->trace->name
3614 * will point to the same string as current_trace->name.
3616 mutex_lock(&trace_types_lock);
3617 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3618 *iter->trace = *tr->current_trace;
3619 mutex_unlock(&trace_types_lock);
3621 #ifdef CONFIG_TRACER_MAX_TRACE
3622 if (iter->snapshot && iter->trace->use_max_tr)
3623 return ERR_PTR(-EBUSY);
3626 if (!iter->snapshot)
3627 atomic_inc(&trace_record_taskinfo_disabled);
3629 if (*pos != iter->pos) {
3634 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3635 for_each_tracing_cpu(cpu)
3636 tracing_iter_reset(iter, cpu);
3638 tracing_iter_reset(iter, cpu_file);
3641 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3646 * If we overflowed the seq_file before, then we want
3647 * to just reuse the trace_seq buffer again.
3653 p = s_next(m, p, &l);
3657 trace_event_read_lock();
3658 trace_access_lock(cpu_file);
3662 static void s_stop(struct seq_file *m, void *p)
3664 struct trace_iterator *iter = m->private;
3666 #ifdef CONFIG_TRACER_MAX_TRACE
3667 if (iter->snapshot && iter->trace->use_max_tr)
3671 if (!iter->snapshot)
3672 atomic_dec(&trace_record_taskinfo_disabled);
3674 trace_access_unlock(iter->cpu_file);
3675 trace_event_read_unlock();
3679 get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
3680 unsigned long *entries, int cpu)
3682 unsigned long count;
3684 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3686 * If this buffer has skipped entries, then we hold all
3687 * entries for the trace and we need to ignore the
3688 * ones before the time stamp.
3690 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3691 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3692 /* total is the same as the entries */
3696 ring_buffer_overrun_cpu(buf->buffer, cpu);
3701 get_total_entries(struct array_buffer *buf,
3702 unsigned long *total, unsigned long *entries)
3710 for_each_tracing_cpu(cpu) {
3711 get_total_entries_cpu(buf, &t, &e, cpu);
3717 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
3719 unsigned long total, entries;
3724 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
3729 unsigned long trace_total_entries(struct trace_array *tr)
3731 unsigned long total, entries;
3736 get_total_entries(&tr->array_buffer, &total, &entries);
3741 static void print_lat_help_header(struct seq_file *m)
3743 seq_puts(m, "# _------=> CPU# \n"
3744 "# / _-----=> irqs-off \n"
3745 "# | / _----=> need-resched \n"
3746 "# || / _---=> hardirq/softirq \n"
3747 "# ||| / _--=> preempt-depth \n"
3749 "# cmd pid ||||| time | caller \n"
3750 "# \\ / ||||| \\ | / \n");
3753 static void print_event_info(struct array_buffer *buf, struct seq_file *m)
3755 unsigned long total;
3756 unsigned long entries;
3758 get_total_entries(buf, &total, &entries);
3759 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3760 entries, total, num_online_cpus());
3764 static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
3767 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3769 print_event_info(buf, m);
3771 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3772 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
3775 static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
3778 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3779 const char *space = " ";
3780 int prec = tgid ? 10 : 2;
3782 print_event_info(buf, m);
3784 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space);
3785 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
3786 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
3787 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
3788 seq_printf(m, "# %.*s||| / delay\n", prec, space);
3789 seq_printf(m, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID ");
3790 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | ");
3794 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3796 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3797 struct array_buffer *buf = iter->array_buffer;
3798 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3799 struct tracer *type = iter->trace;
3800 unsigned long entries;
3801 unsigned long total;
3802 const char *name = "preemption";
3806 get_total_entries(buf, &total, &entries);
3808 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3810 seq_puts(m, "# -----------------------------------"
3811 "---------------------------------\n");
3812 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3813 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3814 nsecs_to_usecs(data->saved_latency),
3818 #if defined(CONFIG_PREEMPT_NONE)
3820 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3822 #elif defined(CONFIG_PREEMPT)
3824 #elif defined(CONFIG_PREEMPT_RT)
3829 /* These are reserved for later use */
3832 seq_printf(m, " #P:%d)\n", num_online_cpus());
3836 seq_puts(m, "# -----------------\n");
3837 seq_printf(m, "# | task: %.16s-%d "
3838 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3839 data->comm, data->pid,
3840 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3841 data->policy, data->rt_priority);
3842 seq_puts(m, "# -----------------\n");
3844 if (data->critical_start) {
3845 seq_puts(m, "# => started at: ");
3846 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3847 trace_print_seq(m, &iter->seq);
3848 seq_puts(m, "\n# => ended at: ");
3849 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3850 trace_print_seq(m, &iter->seq);
3851 seq_puts(m, "\n#\n");
3857 static void test_cpu_buff_start(struct trace_iterator *iter)
3859 struct trace_seq *s = &iter->seq;
3860 struct trace_array *tr = iter->tr;
3862 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3865 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3868 if (cpumask_available(iter->started) &&
3869 cpumask_test_cpu(iter->cpu, iter->started))
3872 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
3875 if (cpumask_available(iter->started))
3876 cpumask_set_cpu(iter->cpu, iter->started);
3878 /* Don't print started cpu buffer for the first entry of the trace */
3880 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3884 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3886 struct trace_array *tr = iter->tr;
3887 struct trace_seq *s = &iter->seq;
3888 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3889 struct trace_entry *entry;
3890 struct trace_event *event;
3894 test_cpu_buff_start(iter);
3896 event = ftrace_find_event(entry->type);
3898 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3899 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3900 trace_print_lat_context(iter);
3902 trace_print_context(iter);
3905 if (trace_seq_has_overflowed(s))
3906 return TRACE_TYPE_PARTIAL_LINE;
3909 return event->funcs->trace(iter, sym_flags, event);
3911 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3913 return trace_handle_return(s);
3916 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3918 struct trace_array *tr = iter->tr;
3919 struct trace_seq *s = &iter->seq;
3920 struct trace_entry *entry;
3921 struct trace_event *event;
3925 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3926 trace_seq_printf(s, "%d %d %llu ",
3927 entry->pid, iter->cpu, iter->ts);
3929 if (trace_seq_has_overflowed(s))
3930 return TRACE_TYPE_PARTIAL_LINE;
3932 event = ftrace_find_event(entry->type);
3934 return event->funcs->raw(iter, 0, event);
3936 trace_seq_printf(s, "%d ?\n", entry->type);
3938 return trace_handle_return(s);
3941 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3943 struct trace_array *tr = iter->tr;
3944 struct trace_seq *s = &iter->seq;
3945 unsigned char newline = '\n';
3946 struct trace_entry *entry;
3947 struct trace_event *event;
3951 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3952 SEQ_PUT_HEX_FIELD(s, entry->pid);
3953 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3954 SEQ_PUT_HEX_FIELD(s, iter->ts);
3955 if (trace_seq_has_overflowed(s))
3956 return TRACE_TYPE_PARTIAL_LINE;
3959 event = ftrace_find_event(entry->type);
3961 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3962 if (ret != TRACE_TYPE_HANDLED)
3966 SEQ_PUT_FIELD(s, newline);
3968 return trace_handle_return(s);
3971 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3973 struct trace_array *tr = iter->tr;
3974 struct trace_seq *s = &iter->seq;
3975 struct trace_entry *entry;
3976 struct trace_event *event;
3980 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3981 SEQ_PUT_FIELD(s, entry->pid);
3982 SEQ_PUT_FIELD(s, iter->cpu);
3983 SEQ_PUT_FIELD(s, iter->ts);
3984 if (trace_seq_has_overflowed(s))
3985 return TRACE_TYPE_PARTIAL_LINE;
3988 event = ftrace_find_event(entry->type);
3989 return event ? event->funcs->binary(iter, 0, event) :
3993 int trace_empty(struct trace_iterator *iter)
3995 struct ring_buffer_iter *buf_iter;
3998 /* If we are looking at one CPU buffer, only check that one */
3999 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4000 cpu = iter->cpu_file;
4001 buf_iter = trace_buffer_iter(iter, cpu);
4003 if (!ring_buffer_iter_empty(buf_iter))
4006 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4012 for_each_tracing_cpu(cpu) {
4013 buf_iter = trace_buffer_iter(iter, cpu);
4015 if (!ring_buffer_iter_empty(buf_iter))
4018 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4026 /* Called with trace_event_read_lock() held. */
4027 enum print_line_t print_trace_line(struct trace_iterator *iter)
4029 struct trace_array *tr = iter->tr;
4030 unsigned long trace_flags = tr->trace_flags;
4031 enum print_line_t ret;
4033 if (iter->lost_events) {
4034 if (iter->lost_events == (unsigned long)-1)
4035 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4038 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4039 iter->cpu, iter->lost_events);
4040 if (trace_seq_has_overflowed(&iter->seq))
4041 return TRACE_TYPE_PARTIAL_LINE;
4044 if (iter->trace && iter->trace->print_line) {
4045 ret = iter->trace->print_line(iter);
4046 if (ret != TRACE_TYPE_UNHANDLED)
4050 if (iter->ent->type == TRACE_BPUTS &&
4051 trace_flags & TRACE_ITER_PRINTK &&
4052 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4053 return trace_print_bputs_msg_only(iter);
4055 if (iter->ent->type == TRACE_BPRINT &&
4056 trace_flags & TRACE_ITER_PRINTK &&
4057 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4058 return trace_print_bprintk_msg_only(iter);
4060 if (iter->ent->type == TRACE_PRINT &&
4061 trace_flags & TRACE_ITER_PRINTK &&
4062 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4063 return trace_print_printk_msg_only(iter);
4065 if (trace_flags & TRACE_ITER_BIN)
4066 return print_bin_fmt(iter);
4068 if (trace_flags & TRACE_ITER_HEX)
4069 return print_hex_fmt(iter);
4071 if (trace_flags & TRACE_ITER_RAW)
4072 return print_raw_fmt(iter);
4074 return print_trace_fmt(iter);
4077 void trace_latency_header(struct seq_file *m)
4079 struct trace_iterator *iter = m->private;
4080 struct trace_array *tr = iter->tr;
4082 /* print nothing if the buffers are empty */
4083 if (trace_empty(iter))
4086 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4087 print_trace_header(m, iter);
4089 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4090 print_lat_help_header(m);
4093 void trace_default_header(struct seq_file *m)
4095 struct trace_iterator *iter = m->private;
4096 struct trace_array *tr = iter->tr;
4097 unsigned long trace_flags = tr->trace_flags;
4099 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4102 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4103 /* print nothing if the buffers are empty */
4104 if (trace_empty(iter))
4106 print_trace_header(m, iter);
4107 if (!(trace_flags & TRACE_ITER_VERBOSE))
4108 print_lat_help_header(m);
4110 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4111 if (trace_flags & TRACE_ITER_IRQ_INFO)
4112 print_func_help_header_irq(iter->array_buffer,
4115 print_func_help_header(iter->array_buffer, m,
4121 static void test_ftrace_alive(struct seq_file *m)
4123 if (!ftrace_is_dead())
4125 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4126 "# MAY BE MISSING FUNCTION EVENTS\n");
4129 #ifdef CONFIG_TRACER_MAX_TRACE
4130 static void show_snapshot_main_help(struct seq_file *m)
4132 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4133 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4134 "# Takes a snapshot of the main buffer.\n"
4135 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4136 "# (Doesn't have to be '2' works with any number that\n"
4137 "# is not a '0' or '1')\n");
4140 static void show_snapshot_percpu_help(struct seq_file *m)
4142 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4143 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4144 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4145 "# Takes a snapshot of the main buffer for this cpu.\n");
4147 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4148 "# Must use main snapshot file to allocate.\n");
4150 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4151 "# (Doesn't have to be '2' works with any number that\n"
4152 "# is not a '0' or '1')\n");
4155 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4157 if (iter->tr->allocated_snapshot)
4158 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4160 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4162 seq_puts(m, "# Snapshot commands:\n");
4163 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4164 show_snapshot_main_help(m);
4166 show_snapshot_percpu_help(m);
4169 /* Should never be called */
4170 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4173 static int s_show(struct seq_file *m, void *v)
4175 struct trace_iterator *iter = v;
4178 if (iter->ent == NULL) {
4180 seq_printf(m, "# tracer: %s\n", iter->trace->name);
4182 test_ftrace_alive(m);
4184 if (iter->snapshot && trace_empty(iter))
4185 print_snapshot_help(m, iter);
4186 else if (iter->trace && iter->trace->print_header)
4187 iter->trace->print_header(m);
4189 trace_default_header(m);
4191 } else if (iter->leftover) {
4193 * If we filled the seq_file buffer earlier, we
4194 * want to just show it now.
4196 ret = trace_print_seq(m, &iter->seq);
4198 /* ret should this time be zero, but you never know */
4199 iter->leftover = ret;
4202 print_trace_line(iter);
4203 ret = trace_print_seq(m, &iter->seq);
4205 * If we overflow the seq_file buffer, then it will
4206 * ask us for this data again at start up.
4208 * ret is 0 if seq_file write succeeded.
4211 iter->leftover = ret;
4218 * Should be used after trace_array_get(), trace_types_lock
4219 * ensures that i_cdev was already initialized.
4221 static inline int tracing_get_cpu(struct inode *inode)
4223 if (inode->i_cdev) /* See trace_create_cpu_file() */
4224 return (long)inode->i_cdev - 1;
4225 return RING_BUFFER_ALL_CPUS;
4228 static const struct seq_operations tracer_seq_ops = {
4235 static struct trace_iterator *
4236 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4238 struct trace_array *tr = inode->i_private;
4239 struct trace_iterator *iter;
4242 if (tracing_disabled)
4243 return ERR_PTR(-ENODEV);
4245 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4247 return ERR_PTR(-ENOMEM);
4249 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4251 if (!iter->buffer_iter)
4255 * trace_find_next_entry() may need to save off iter->ent.
4256 * It will place it into the iter->temp buffer. As most
4257 * events are less than 128, allocate a buffer of that size.
4258 * If one is greater, then trace_find_next_entry() will
4259 * allocate a new buffer to adjust for the bigger iter->ent.
4260 * It's not critical if it fails to get allocated here.
4262 iter->temp = kmalloc(128, GFP_KERNEL);
4264 iter->temp_size = 128;
4267 * We make a copy of the current tracer to avoid concurrent
4268 * changes on it while we are reading.
4270 mutex_lock(&trace_types_lock);
4271 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4275 *iter->trace = *tr->current_trace;
4277 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4282 #ifdef CONFIG_TRACER_MAX_TRACE
4283 /* Currently only the top directory has a snapshot */
4284 if (tr->current_trace->print_max || snapshot)
4285 iter->array_buffer = &tr->max_buffer;
4288 iter->array_buffer = &tr->array_buffer;
4289 iter->snapshot = snapshot;
4291 iter->cpu_file = tracing_get_cpu(inode);
4292 mutex_init(&iter->mutex);
4294 /* Notify the tracer early; before we stop tracing. */
4295 if (iter->trace->open)
4296 iter->trace->open(iter);
4298 /* Annotate start of buffers if we had overruns */
4299 if (ring_buffer_overruns(iter->array_buffer->buffer))
4300 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4302 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4303 if (trace_clocks[tr->clock_id].in_ns)
4304 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4307 * If pause-on-trace is enabled, then stop the trace while
4308 * dumping, unless this is the "snapshot" file
4310 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4311 tracing_stop_tr(tr);
4313 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4314 for_each_tracing_cpu(cpu) {
4315 iter->buffer_iter[cpu] =
4316 ring_buffer_read_prepare(iter->array_buffer->buffer,
4319 ring_buffer_read_prepare_sync();
4320 for_each_tracing_cpu(cpu) {
4321 ring_buffer_read_start(iter->buffer_iter[cpu]);
4322 tracing_iter_reset(iter, cpu);
4325 cpu = iter->cpu_file;
4326 iter->buffer_iter[cpu] =
4327 ring_buffer_read_prepare(iter->array_buffer->buffer,
4329 ring_buffer_read_prepare_sync();
4330 ring_buffer_read_start(iter->buffer_iter[cpu]);
4331 tracing_iter_reset(iter, cpu);
4334 mutex_unlock(&trace_types_lock);
4339 mutex_unlock(&trace_types_lock);
4342 kfree(iter->buffer_iter);
4344 seq_release_private(inode, file);
4345 return ERR_PTR(-ENOMEM);
4348 int tracing_open_generic(struct inode *inode, struct file *filp)
4352 ret = tracing_check_open_get_tr(NULL);
4356 filp->private_data = inode->i_private;
4360 bool tracing_is_disabled(void)
4362 return (tracing_disabled) ? true: false;
4366 * Open and update trace_array ref count.
4367 * Must have the current trace_array passed to it.
4369 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4371 struct trace_array *tr = inode->i_private;
4374 ret = tracing_check_open_get_tr(tr);
4378 filp->private_data = inode->i_private;
4383 static int tracing_release(struct inode *inode, struct file *file)
4385 struct trace_array *tr = inode->i_private;
4386 struct seq_file *m = file->private_data;
4387 struct trace_iterator *iter;
4390 if (!(file->f_mode & FMODE_READ)) {
4391 trace_array_put(tr);
4395 /* Writes do not use seq_file */
4397 mutex_lock(&trace_types_lock);
4399 for_each_tracing_cpu(cpu) {
4400 if (iter->buffer_iter[cpu])
4401 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4404 if (iter->trace && iter->trace->close)
4405 iter->trace->close(iter);
4407 if (!iter->snapshot && tr->stop_count)
4408 /* reenable tracing if it was previously enabled */
4409 tracing_start_tr(tr);
4411 __trace_array_put(tr);
4413 mutex_unlock(&trace_types_lock);
4415 mutex_destroy(&iter->mutex);
4416 free_cpumask_var(iter->started);
4419 kfree(iter->buffer_iter);
4420 seq_release_private(inode, file);
4425 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4427 struct trace_array *tr = inode->i_private;
4429 trace_array_put(tr);
4433 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4435 struct trace_array *tr = inode->i_private;
4437 trace_array_put(tr);
4439 return single_release(inode, file);
4442 static int tracing_open(struct inode *inode, struct file *file)
4444 struct trace_array *tr = inode->i_private;
4445 struct trace_iterator *iter;
4448 ret = tracing_check_open_get_tr(tr);
4452 /* If this file was open for write, then erase contents */
4453 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4454 int cpu = tracing_get_cpu(inode);
4455 struct array_buffer *trace_buf = &tr->array_buffer;
4457 #ifdef CONFIG_TRACER_MAX_TRACE
4458 if (tr->current_trace->print_max)
4459 trace_buf = &tr->max_buffer;
4462 if (cpu == RING_BUFFER_ALL_CPUS)
4463 tracing_reset_online_cpus(trace_buf);
4465 tracing_reset_cpu(trace_buf, cpu);
4468 if (file->f_mode & FMODE_READ) {
4469 iter = __tracing_open(inode, file, false);
4471 ret = PTR_ERR(iter);
4472 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4473 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4477 trace_array_put(tr);
4483 * Some tracers are not suitable for instance buffers.
4484 * A tracer is always available for the global array (toplevel)
4485 * or if it explicitly states that it is.
4488 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4490 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4493 /* Find the next tracer that this trace array may use */
4494 static struct tracer *
4495 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4497 while (t && !trace_ok_for_array(t, tr))
4504 t_next(struct seq_file *m, void *v, loff_t *pos)
4506 struct trace_array *tr = m->private;
4507 struct tracer *t = v;
4512 t = get_tracer_for_array(tr, t->next);
4517 static void *t_start(struct seq_file *m, loff_t *pos)
4519 struct trace_array *tr = m->private;
4523 mutex_lock(&trace_types_lock);
4525 t = get_tracer_for_array(tr, trace_types);
4526 for (; t && l < *pos; t = t_next(m, t, &l))
4532 static void t_stop(struct seq_file *m, void *p)
4534 mutex_unlock(&trace_types_lock);
4537 static int t_show(struct seq_file *m, void *v)
4539 struct tracer *t = v;
4544 seq_puts(m, t->name);
4553 static const struct seq_operations show_traces_seq_ops = {
4560 static int show_traces_open(struct inode *inode, struct file *file)
4562 struct trace_array *tr = inode->i_private;
4566 ret = tracing_check_open_get_tr(tr);
4570 ret = seq_open(file, &show_traces_seq_ops);
4572 trace_array_put(tr);
4576 m = file->private_data;
4582 static int show_traces_release(struct inode *inode, struct file *file)
4584 struct trace_array *tr = inode->i_private;
4586 trace_array_put(tr);
4587 return seq_release(inode, file);
4591 tracing_write_stub(struct file *filp, const char __user *ubuf,
4592 size_t count, loff_t *ppos)
4597 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4601 if (file->f_mode & FMODE_READ)
4602 ret = seq_lseek(file, offset, whence);
4604 file->f_pos = ret = 0;
4609 static const struct file_operations tracing_fops = {
4610 .open = tracing_open,
4612 .write = tracing_write_stub,
4613 .llseek = tracing_lseek,
4614 .release = tracing_release,
4617 static const struct file_operations show_traces_fops = {
4618 .open = show_traces_open,
4620 .llseek = seq_lseek,
4621 .release = show_traces_release,
4625 tracing_cpumask_read(struct file *filp, char __user *ubuf,
4626 size_t count, loff_t *ppos)
4628 struct trace_array *tr = file_inode(filp)->i_private;
4632 len = snprintf(NULL, 0, "%*pb\n",
4633 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4634 mask_str = kmalloc(len, GFP_KERNEL);
4638 len = snprintf(mask_str, len, "%*pb\n",
4639 cpumask_pr_args(tr->tracing_cpumask));
4644 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4652 int tracing_set_cpumask(struct trace_array *tr,
4653 cpumask_var_t tracing_cpumask_new)
4660 local_irq_disable();
4661 arch_spin_lock(&tr->max_lock);
4662 for_each_tracing_cpu(cpu) {
4664 * Increase/decrease the disabled counter if we are
4665 * about to flip a bit in the cpumask:
4667 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4668 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4669 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4670 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
4672 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4673 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4674 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4675 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
4678 arch_spin_unlock(&tr->max_lock);
4681 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4687 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4688 size_t count, loff_t *ppos)
4690 struct trace_array *tr = file_inode(filp)->i_private;
4691 cpumask_var_t tracing_cpumask_new;
4694 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4697 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4701 err = tracing_set_cpumask(tr, tracing_cpumask_new);
4705 free_cpumask_var(tracing_cpumask_new);
4710 free_cpumask_var(tracing_cpumask_new);
4715 static const struct file_operations tracing_cpumask_fops = {
4716 .open = tracing_open_generic_tr,
4717 .read = tracing_cpumask_read,
4718 .write = tracing_cpumask_write,
4719 .release = tracing_release_generic_tr,
4720 .llseek = generic_file_llseek,
4723 static int tracing_trace_options_show(struct seq_file *m, void *v)
4725 struct tracer_opt *trace_opts;
4726 struct trace_array *tr = m->private;
4730 mutex_lock(&trace_types_lock);
4731 tracer_flags = tr->current_trace->flags->val;
4732 trace_opts = tr->current_trace->flags->opts;
4734 for (i = 0; trace_options[i]; i++) {
4735 if (tr->trace_flags & (1 << i))
4736 seq_printf(m, "%s\n", trace_options[i]);
4738 seq_printf(m, "no%s\n", trace_options[i]);
4741 for (i = 0; trace_opts[i].name; i++) {
4742 if (tracer_flags & trace_opts[i].bit)
4743 seq_printf(m, "%s\n", trace_opts[i].name);
4745 seq_printf(m, "no%s\n", trace_opts[i].name);
4747 mutex_unlock(&trace_types_lock);
4752 static int __set_tracer_option(struct trace_array *tr,
4753 struct tracer_flags *tracer_flags,
4754 struct tracer_opt *opts, int neg)
4756 struct tracer *trace = tracer_flags->trace;
4759 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4764 tracer_flags->val &= ~opts->bit;
4766 tracer_flags->val |= opts->bit;
4770 /* Try to assign a tracer specific option */
4771 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4773 struct tracer *trace = tr->current_trace;
4774 struct tracer_flags *tracer_flags = trace->flags;
4775 struct tracer_opt *opts = NULL;
4778 for (i = 0; tracer_flags->opts[i].name; i++) {
4779 opts = &tracer_flags->opts[i];
4781 if (strcmp(cmp, opts->name) == 0)
4782 return __set_tracer_option(tr, trace->flags, opts, neg);
4788 /* Some tracers require overwrite to stay enabled */
4789 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4791 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4797 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4799 if ((mask == TRACE_ITER_RECORD_TGID) ||
4800 (mask == TRACE_ITER_RECORD_CMD))
4801 lockdep_assert_held(&event_mutex);
4803 /* do nothing if flag is already set */
4804 if (!!(tr->trace_flags & mask) == !!enabled)
4807 /* Give the tracer a chance to approve the change */
4808 if (tr->current_trace->flag_changed)
4809 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4813 tr->trace_flags |= mask;
4815 tr->trace_flags &= ~mask;
4817 if (mask == TRACE_ITER_RECORD_CMD)
4818 trace_event_enable_cmd_record(enabled);
4820 if (mask == TRACE_ITER_RECORD_TGID) {
4822 tgid_map = kvcalloc(PID_MAX_DEFAULT + 1,
4826 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4830 trace_event_enable_tgid_record(enabled);
4833 if (mask == TRACE_ITER_EVENT_FORK)
4834 trace_event_follow_fork(tr, enabled);
4836 if (mask == TRACE_ITER_FUNC_FORK)
4837 ftrace_pid_follow_fork(tr, enabled);
4839 if (mask == TRACE_ITER_OVERWRITE) {
4840 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
4841 #ifdef CONFIG_TRACER_MAX_TRACE
4842 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4846 if (mask == TRACE_ITER_PRINTK) {
4847 trace_printk_start_stop_comm(enabled);
4848 trace_printk_control(enabled);
4854 int trace_set_options(struct trace_array *tr, char *option)
4859 size_t orig_len = strlen(option);
4862 cmp = strstrip(option);
4864 len = str_has_prefix(cmp, "no");
4870 mutex_lock(&event_mutex);
4871 mutex_lock(&trace_types_lock);
4873 ret = match_string(trace_options, -1, cmp);
4874 /* If no option could be set, test the specific tracer options */
4876 ret = set_tracer_option(tr, cmp, neg);
4878 ret = set_tracer_flag(tr, 1 << ret, !neg);
4880 mutex_unlock(&trace_types_lock);
4881 mutex_unlock(&event_mutex);
4884 * If the first trailing whitespace is replaced with '\0' by strstrip,
4885 * turn it back into a space.
4887 if (orig_len > strlen(option))
4888 option[strlen(option)] = ' ';
4893 static void __init apply_trace_boot_options(void)
4895 char *buf = trace_boot_options_buf;
4899 option = strsep(&buf, ",");
4905 trace_set_options(&global_trace, option);
4907 /* Put back the comma to allow this to be called again */
4914 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4915 size_t cnt, loff_t *ppos)
4917 struct seq_file *m = filp->private_data;
4918 struct trace_array *tr = m->private;
4922 if (cnt >= sizeof(buf))
4925 if (copy_from_user(buf, ubuf, cnt))
4930 ret = trace_set_options(tr, buf);
4939 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4941 struct trace_array *tr = inode->i_private;
4944 ret = tracing_check_open_get_tr(tr);
4948 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4950 trace_array_put(tr);
4955 static const struct file_operations tracing_iter_fops = {
4956 .open = tracing_trace_options_open,
4958 .llseek = seq_lseek,
4959 .release = tracing_single_release_tr,
4960 .write = tracing_trace_options_write,
4963 static const char readme_msg[] =
4964 "tracing mini-HOWTO:\n\n"
4965 "# echo 0 > tracing_on : quick way to disable tracing\n"
4966 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4967 " Important files:\n"
4968 " trace\t\t\t- The static contents of the buffer\n"
4969 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4970 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4971 " current_tracer\t- function and latency tracers\n"
4972 " available_tracers\t- list of configured tracers for current_tracer\n"
4973 " error_log\t- error log for failed commands (that support it)\n"
4974 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4975 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4976 " trace_clock\t\t-change the clock used to order events\n"
4977 " local: Per cpu clock but may not be synced across CPUs\n"
4978 " global: Synced across CPUs but slows tracing down.\n"
4979 " counter: Not a clock, but just an increment\n"
4980 " uptime: Jiffy counter from time of boot\n"
4981 " perf: Same clock that perf events use\n"
4982 #ifdef CONFIG_X86_64
4983 " x86-tsc: TSC cycle counter\n"
4985 "\n timestamp_mode\t-view the mode used to timestamp events\n"
4986 " delta: Delta difference against a buffer-wide timestamp\n"
4987 " absolute: Absolute (standalone) timestamp\n"
4988 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4989 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4990 " tracing_cpumask\t- Limit which CPUs to trace\n"
4991 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4992 "\t\t\t Remove sub-buffer with rmdir\n"
4993 " trace_options\t\t- Set format or modify how tracing happens\n"
4994 "\t\t\t Disable an option by prefixing 'no' to the\n"
4995 "\t\t\t option name\n"
4996 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4997 #ifdef CONFIG_DYNAMIC_FTRACE
4998 "\n available_filter_functions - list of functions that can be filtered on\n"
4999 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5000 "\t\t\t functions\n"
5001 "\t accepts: func_full_name or glob-matching-pattern\n"
5002 "\t modules: Can select a group via module\n"
5003 "\t Format: :mod:<module-name>\n"
5004 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
5005 "\t triggers: a command to perform when function is hit\n"
5006 "\t Format: <function>:<trigger>[:count]\n"
5007 "\t trigger: traceon, traceoff\n"
5008 "\t\t enable_event:<system>:<event>\n"
5009 "\t\t disable_event:<system>:<event>\n"
5010 #ifdef CONFIG_STACKTRACE
5013 #ifdef CONFIG_TRACER_SNAPSHOT
5018 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5019 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5020 "\t The first one will disable tracing every time do_fault is hit\n"
5021 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
5022 "\t The first time do trap is hit and it disables tracing, the\n"
5023 "\t counter will decrement to 2. If tracing is already disabled,\n"
5024 "\t the counter will not decrement. It only decrements when the\n"
5025 "\t trigger did work\n"
5026 "\t To remove trigger without count:\n"
5027 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5028 "\t To remove trigger with a count:\n"
5029 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5030 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5031 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5032 "\t modules: Can select a group via module command :mod:\n"
5033 "\t Does not accept triggers\n"
5034 #endif /* CONFIG_DYNAMIC_FTRACE */
5035 #ifdef CONFIG_FUNCTION_TRACER
5036 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5038 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5041 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5042 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5043 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5044 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5046 #ifdef CONFIG_TRACER_SNAPSHOT
5047 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5048 "\t\t\t snapshot buffer. Read the contents for more\n"
5049 "\t\t\t information\n"
5051 #ifdef CONFIG_STACK_TRACER
5052 " stack_trace\t\t- Shows the max stack trace when active\n"
5053 " stack_max_size\t- Shows current max stack size that was traced\n"
5054 "\t\t\t Write into this file to reset the max size (trigger a\n"
5055 "\t\t\t new trace)\n"
5056 #ifdef CONFIG_DYNAMIC_FTRACE
5057 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5060 #endif /* CONFIG_STACK_TRACER */
5061 #ifdef CONFIG_DYNAMIC_EVENTS
5062 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5063 "\t\t\t Write into this file to define/undefine new trace events.\n"
5065 #ifdef CONFIG_KPROBE_EVENTS
5066 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5067 "\t\t\t Write into this file to define/undefine new trace events.\n"
5069 #ifdef CONFIG_UPROBE_EVENTS
5070 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5071 "\t\t\t Write into this file to define/undefine new trace events.\n"
5073 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5074 "\t accepts: event-definitions (one definition per line)\n"
5075 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
5076 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
5077 #ifdef CONFIG_HIST_TRIGGERS
5078 "\t s:[synthetic/]<event> <field> [<field>]\n"
5080 "\t -:[<group>/]<event>\n"
5081 #ifdef CONFIG_KPROBE_EVENTS
5082 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5083 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5085 #ifdef CONFIG_UPROBE_EVENTS
5086 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
5088 "\t args: <name>=fetcharg[:type]\n"
5089 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
5090 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5091 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5093 "\t $stack<index>, $stack, $retval, $comm,\n"
5095 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5096 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
5097 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5098 "\t <type>\\[<array-size>\\]\n"
5099 #ifdef CONFIG_HIST_TRIGGERS
5100 "\t field: <stype> <name>;\n"
5101 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5102 "\t [unsigned] char/int/long\n"
5105 " events/\t\t- Directory containing all trace event subsystems:\n"
5106 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5107 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5108 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5110 " filter\t\t- If set, only events passing filter are traced\n"
5111 " events/<system>/<event>/\t- Directory containing control files for\n"
5113 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5114 " filter\t\t- If set, only events passing filter are traced\n"
5115 " trigger\t\t- If set, a command to perform when event is hit\n"
5116 "\t Format: <trigger>[:count][if <filter>]\n"
5117 "\t trigger: traceon, traceoff\n"
5118 "\t enable_event:<system>:<event>\n"
5119 "\t disable_event:<system>:<event>\n"
5120 #ifdef CONFIG_HIST_TRIGGERS
5121 "\t enable_hist:<system>:<event>\n"
5122 "\t disable_hist:<system>:<event>\n"
5124 #ifdef CONFIG_STACKTRACE
5127 #ifdef CONFIG_TRACER_SNAPSHOT
5130 #ifdef CONFIG_HIST_TRIGGERS
5131 "\t\t hist (see below)\n"
5133 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
5134 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5135 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5136 "\t events/block/block_unplug/trigger\n"
5137 "\t The first disables tracing every time block_unplug is hit.\n"
5138 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
5139 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
5140 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5141 "\t Like function triggers, the counter is only decremented if it\n"
5142 "\t enabled or disabled tracing.\n"
5143 "\t To remove a trigger without a count:\n"
5144 "\t echo '!<trigger> > <system>/<event>/trigger\n"
5145 "\t To remove a trigger with a count:\n"
5146 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5147 "\t Filters can be ignored when removing a trigger.\n"
5148 #ifdef CONFIG_HIST_TRIGGERS
5149 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5150 "\t Format: hist:keys=<field1[,field2,...]>\n"
5151 "\t [:values=<field1[,field2,...]>]\n"
5152 "\t [:sort=<field1[,field2,...]>]\n"
5153 "\t [:size=#entries]\n"
5154 "\t [:pause][:continue][:clear]\n"
5155 "\t [:name=histname1]\n"
5156 "\t [:<handler>.<action>]\n"
5157 "\t [if <filter>]\n\n"
5158 "\t When a matching event is hit, an entry is added to a hash\n"
5159 "\t table using the key(s) and value(s) named, and the value of a\n"
5160 "\t sum called 'hitcount' is incremented. Keys and values\n"
5161 "\t correspond to fields in the event's format description. Keys\n"
5162 "\t can be any field, or the special string 'stacktrace'.\n"
5163 "\t Compound keys consisting of up to two fields can be specified\n"
5164 "\t by the 'keys' keyword. Values must correspond to numeric\n"
5165 "\t fields. Sort keys consisting of up to two fields can be\n"
5166 "\t specified using the 'sort' keyword. The sort direction can\n"
5167 "\t be modified by appending '.descending' or '.ascending' to a\n"
5168 "\t sort field. The 'size' parameter can be used to specify more\n"
5169 "\t or fewer than the default 2048 entries for the hashtable size.\n"
5170 "\t If a hist trigger is given a name using the 'name' parameter,\n"
5171 "\t its histogram data will be shared with other triggers of the\n"
5172 "\t same name, and trigger hits will update this common data.\n\n"
5173 "\t Reading the 'hist' file for the event will dump the hash\n"
5174 "\t table in its entirety to stdout. If there are multiple hist\n"
5175 "\t triggers attached to an event, there will be a table for each\n"
5176 "\t trigger in the output. The table displayed for a named\n"
5177 "\t trigger will be the same as any other instance having the\n"
5178 "\t same name. The default format used to display a given field\n"
5179 "\t can be modified by appending any of the following modifiers\n"
5180 "\t to the field name, as applicable:\n\n"
5181 "\t .hex display a number as a hex value\n"
5182 "\t .sym display an address as a symbol\n"
5183 "\t .sym-offset display an address as a symbol and offset\n"
5184 "\t .execname display a common_pid as a program name\n"
5185 "\t .syscall display a syscall id as a syscall name\n"
5186 "\t .log2 display log2 value rather than raw number\n"
5187 "\t .usecs display a common_timestamp in microseconds\n\n"
5188 "\t The 'pause' parameter can be used to pause an existing hist\n"
5189 "\t trigger or to start a hist trigger but not log any events\n"
5190 "\t until told to do so. 'continue' can be used to start or\n"
5191 "\t restart a paused hist trigger.\n\n"
5192 "\t The 'clear' parameter will clear the contents of a running\n"
5193 "\t hist trigger and leave its current paused/active state\n"
5195 "\t The enable_hist and disable_hist triggers can be used to\n"
5196 "\t have one event conditionally start and stop another event's\n"
5197 "\t already-attached hist trigger. The syntax is analogous to\n"
5198 "\t the enable_event and disable_event triggers.\n\n"
5199 "\t Hist trigger handlers and actions are executed whenever a\n"
5200 "\t a histogram entry is added or updated. They take the form:\n\n"
5201 "\t <handler>.<action>\n\n"
5202 "\t The available handlers are:\n\n"
5203 "\t onmatch(matching.event) - invoke on addition or update\n"
5204 "\t onmax(var) - invoke if var exceeds current max\n"
5205 "\t onchange(var) - invoke action if var changes\n\n"
5206 "\t The available actions are:\n\n"
5207 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5208 "\t save(field,...) - save current event fields\n"
5209 #ifdef CONFIG_TRACER_SNAPSHOT
5210 "\t snapshot() - snapshot the trace buffer\n"
5216 tracing_readme_read(struct file *filp, char __user *ubuf,
5217 size_t cnt, loff_t *ppos)
5219 return simple_read_from_buffer(ubuf, cnt, ppos,
5220 readme_msg, strlen(readme_msg));
5223 static const struct file_operations tracing_readme_fops = {
5224 .open = tracing_open_generic,
5225 .read = tracing_readme_read,
5226 .llseek = generic_file_llseek,
5229 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5233 if (*pos || m->count)
5238 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
5239 if (trace_find_tgid(*ptr))
5246 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5256 v = saved_tgids_next(m, v, &l);
5264 static void saved_tgids_stop(struct seq_file *m, void *v)
5268 static int saved_tgids_show(struct seq_file *m, void *v)
5270 int pid = (int *)v - tgid_map;
5272 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5276 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5277 .start = saved_tgids_start,
5278 .stop = saved_tgids_stop,
5279 .next = saved_tgids_next,
5280 .show = saved_tgids_show,
5283 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5287 ret = tracing_check_open_get_tr(NULL);
5291 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5295 static const struct file_operations tracing_saved_tgids_fops = {
5296 .open = tracing_saved_tgids_open,
5298 .llseek = seq_lseek,
5299 .release = seq_release,
5302 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5304 unsigned int *ptr = v;
5306 if (*pos || m->count)
5311 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5313 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5322 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5328 arch_spin_lock(&trace_cmdline_lock);
5330 v = &savedcmd->map_cmdline_to_pid[0];
5332 v = saved_cmdlines_next(m, v, &l);
5340 static void saved_cmdlines_stop(struct seq_file *m, void *v)
5342 arch_spin_unlock(&trace_cmdline_lock);
5346 static int saved_cmdlines_show(struct seq_file *m, void *v)
5348 char buf[TASK_COMM_LEN];
5349 unsigned int *pid = v;
5351 __trace_find_cmdline(*pid, buf);
5352 seq_printf(m, "%d %s\n", *pid, buf);
5356 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5357 .start = saved_cmdlines_start,
5358 .next = saved_cmdlines_next,
5359 .stop = saved_cmdlines_stop,
5360 .show = saved_cmdlines_show,
5363 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5367 ret = tracing_check_open_get_tr(NULL);
5371 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5374 static const struct file_operations tracing_saved_cmdlines_fops = {
5375 .open = tracing_saved_cmdlines_open,
5377 .llseek = seq_lseek,
5378 .release = seq_release,
5382 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5383 size_t cnt, loff_t *ppos)
5388 arch_spin_lock(&trace_cmdline_lock);
5389 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5390 arch_spin_unlock(&trace_cmdline_lock);
5392 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5395 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5397 kfree(s->saved_cmdlines);
5398 kfree(s->map_cmdline_to_pid);
5402 static int tracing_resize_saved_cmdlines(unsigned int val)
5404 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5406 s = kmalloc(sizeof(*s), GFP_KERNEL);
5410 if (allocate_cmdlines_buffer(val, s) < 0) {
5415 arch_spin_lock(&trace_cmdline_lock);
5416 savedcmd_temp = savedcmd;
5418 arch_spin_unlock(&trace_cmdline_lock);
5419 free_saved_cmdlines_buffer(savedcmd_temp);
5425 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5426 size_t cnt, loff_t *ppos)
5431 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5435 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5436 if (!val || val > PID_MAX_DEFAULT)
5439 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5448 static const struct file_operations tracing_saved_cmdlines_size_fops = {
5449 .open = tracing_open_generic,
5450 .read = tracing_saved_cmdlines_size_read,
5451 .write = tracing_saved_cmdlines_size_write,
5454 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5455 static union trace_eval_map_item *
5456 update_eval_map(union trace_eval_map_item *ptr)
5458 if (!ptr->map.eval_string) {
5459 if (ptr->tail.next) {
5460 ptr = ptr->tail.next;
5461 /* Set ptr to the next real item (skip head) */
5469 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5471 union trace_eval_map_item *ptr = v;
5474 * Paranoid! If ptr points to end, we don't want to increment past it.
5475 * This really should never happen.
5478 ptr = update_eval_map(ptr);
5479 if (WARN_ON_ONCE(!ptr))
5483 ptr = update_eval_map(ptr);
5488 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5490 union trace_eval_map_item *v;
5493 mutex_lock(&trace_eval_mutex);
5495 v = trace_eval_maps;
5499 while (v && l < *pos) {
5500 v = eval_map_next(m, v, &l);
5506 static void eval_map_stop(struct seq_file *m, void *v)
5508 mutex_unlock(&trace_eval_mutex);
5511 static int eval_map_show(struct seq_file *m, void *v)
5513 union trace_eval_map_item *ptr = v;
5515 seq_printf(m, "%s %ld (%s)\n",
5516 ptr->map.eval_string, ptr->map.eval_value,
5522 static const struct seq_operations tracing_eval_map_seq_ops = {
5523 .start = eval_map_start,
5524 .next = eval_map_next,
5525 .stop = eval_map_stop,
5526 .show = eval_map_show,
5529 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5533 ret = tracing_check_open_get_tr(NULL);
5537 return seq_open(filp, &tracing_eval_map_seq_ops);
5540 static const struct file_operations tracing_eval_map_fops = {
5541 .open = tracing_eval_map_open,
5543 .llseek = seq_lseek,
5544 .release = seq_release,
5547 static inline union trace_eval_map_item *
5548 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5550 /* Return tail of array given the head */
5551 return ptr + ptr->head.length + 1;
5555 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5558 struct trace_eval_map **stop;
5559 struct trace_eval_map **map;
5560 union trace_eval_map_item *map_array;
5561 union trace_eval_map_item *ptr;
5566 * The trace_eval_maps contains the map plus a head and tail item,
5567 * where the head holds the module and length of array, and the
5568 * tail holds a pointer to the next list.
5570 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5572 pr_warn("Unable to allocate trace eval mapping\n");
5576 mutex_lock(&trace_eval_mutex);
5578 if (!trace_eval_maps)
5579 trace_eval_maps = map_array;
5581 ptr = trace_eval_maps;
5583 ptr = trace_eval_jmp_to_tail(ptr);
5584 if (!ptr->tail.next)
5586 ptr = ptr->tail.next;
5589 ptr->tail.next = map_array;
5591 map_array->head.mod = mod;
5592 map_array->head.length = len;
5595 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5596 map_array->map = **map;
5599 memset(map_array, 0, sizeof(*map_array));
5601 mutex_unlock(&trace_eval_mutex);
5604 static void trace_create_eval_file(struct dentry *d_tracer)
5606 trace_create_file("eval_map", 0444, d_tracer,
5607 NULL, &tracing_eval_map_fops);
5610 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5611 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5612 static inline void trace_insert_eval_map_file(struct module *mod,
5613 struct trace_eval_map **start, int len) { }
5614 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5616 static void trace_insert_eval_map(struct module *mod,
5617 struct trace_eval_map **start, int len)
5619 struct trace_eval_map **map;
5626 trace_event_eval_update(map, len);
5628 trace_insert_eval_map_file(mod, start, len);
5632 tracing_set_trace_read(struct file *filp, char __user *ubuf,
5633 size_t cnt, loff_t *ppos)
5635 struct trace_array *tr = filp->private_data;
5636 char buf[MAX_TRACER_SIZE+2];
5639 mutex_lock(&trace_types_lock);
5640 r = sprintf(buf, "%s\n", tr->current_trace->name);
5641 mutex_unlock(&trace_types_lock);
5643 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5646 int tracer_init(struct tracer *t, struct trace_array *tr)
5648 tracing_reset_online_cpus(&tr->array_buffer);
5652 static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
5656 for_each_tracing_cpu(cpu)
5657 per_cpu_ptr(buf->data, cpu)->entries = val;
5660 #ifdef CONFIG_TRACER_MAX_TRACE
5661 /* resize @tr's buffer to the size of @size_tr's entries */
5662 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
5663 struct array_buffer *size_buf, int cpu_id)
5667 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5668 for_each_tracing_cpu(cpu) {
5669 ret = ring_buffer_resize(trace_buf->buffer,
5670 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5673 per_cpu_ptr(trace_buf->data, cpu)->entries =
5674 per_cpu_ptr(size_buf->data, cpu)->entries;
5677 ret = ring_buffer_resize(trace_buf->buffer,
5678 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5680 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5681 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5686 #endif /* CONFIG_TRACER_MAX_TRACE */
5688 static int __tracing_resize_ring_buffer(struct trace_array *tr,
5689 unsigned long size, int cpu)
5694 * If kernel or user changes the size of the ring buffer
5695 * we use the size that was given, and we can forget about
5696 * expanding it later.
5698 ring_buffer_expanded = true;
5700 /* May be called before buffers are initialized */
5701 if (!tr->array_buffer.buffer)
5704 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
5708 #ifdef CONFIG_TRACER_MAX_TRACE
5709 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5710 !tr->current_trace->use_max_tr)
5713 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5715 int r = resize_buffer_duplicate_size(&tr->array_buffer,
5716 &tr->array_buffer, cpu);
5719 * AARGH! We are left with different
5720 * size max buffer!!!!
5721 * The max buffer is our "snapshot" buffer.
5722 * When a tracer needs a snapshot (one of the
5723 * latency tracers), it swaps the max buffer
5724 * with the saved snap shot. We succeeded to
5725 * update the size of the main buffer, but failed to
5726 * update the size of the max buffer. But when we tried
5727 * to reset the main buffer to the original size, we
5728 * failed there too. This is very unlikely to
5729 * happen, but if it does, warn and kill all
5733 tracing_disabled = 1;
5738 if (cpu == RING_BUFFER_ALL_CPUS)
5739 set_buffer_entries(&tr->max_buffer, size);
5741 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5744 #endif /* CONFIG_TRACER_MAX_TRACE */
5746 if (cpu == RING_BUFFER_ALL_CPUS)
5747 set_buffer_entries(&tr->array_buffer, size);
5749 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
5754 ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5755 unsigned long size, int cpu_id)
5759 mutex_lock(&trace_types_lock);
5761 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5762 /* make sure, this cpu is enabled in the mask */
5763 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5769 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5774 mutex_unlock(&trace_types_lock);
5781 * tracing_update_buffers - used by tracing facility to expand ring buffers
5783 * To save on memory when the tracing is never used on a system with it
5784 * configured in. The ring buffers are set to a minimum size. But once
5785 * a user starts to use the tracing facility, then they need to grow
5786 * to their default size.
5788 * This function is to be called when a tracer is about to be used.
5790 int tracing_update_buffers(void)
5794 mutex_lock(&trace_types_lock);
5795 if (!ring_buffer_expanded)
5796 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5797 RING_BUFFER_ALL_CPUS);
5798 mutex_unlock(&trace_types_lock);
5803 struct trace_option_dentry;
5806 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5809 * Used to clear out the tracer before deletion of an instance.
5810 * Must have trace_types_lock held.
5812 static void tracing_set_nop(struct trace_array *tr)
5814 if (tr->current_trace == &nop_trace)
5817 tr->current_trace->enabled--;
5819 if (tr->current_trace->reset)
5820 tr->current_trace->reset(tr);
5822 tr->current_trace = &nop_trace;
5825 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5827 /* Only enable if the directory has been created already. */
5831 create_trace_option_files(tr, t);
5834 int tracing_set_tracer(struct trace_array *tr, const char *buf)
5837 #ifdef CONFIG_TRACER_MAX_TRACE
5842 mutex_lock(&trace_types_lock);
5844 if (!ring_buffer_expanded) {
5845 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5846 RING_BUFFER_ALL_CPUS);
5852 for (t = trace_types; t; t = t->next) {
5853 if (strcmp(t->name, buf) == 0)
5860 if (t == tr->current_trace)
5863 #ifdef CONFIG_TRACER_SNAPSHOT
5864 if (t->use_max_tr) {
5865 arch_spin_lock(&tr->max_lock);
5866 if (tr->cond_snapshot)
5868 arch_spin_unlock(&tr->max_lock);
5873 /* Some tracers won't work on kernel command line */
5874 if (system_state < SYSTEM_RUNNING && t->noboot) {
5875 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5880 /* Some tracers are only allowed for the top level buffer */
5881 if (!trace_ok_for_array(t, tr)) {
5886 /* If trace pipe files are being read, we can't change the tracer */
5887 if (tr->current_trace->ref) {
5892 trace_branch_disable();
5894 tr->current_trace->enabled--;
5896 if (tr->current_trace->reset)
5897 tr->current_trace->reset(tr);
5899 /* Current trace needs to be nop_trace before synchronize_rcu */
5900 tr->current_trace = &nop_trace;
5902 #ifdef CONFIG_TRACER_MAX_TRACE
5903 had_max_tr = tr->allocated_snapshot;
5905 if (had_max_tr && !t->use_max_tr) {
5907 * We need to make sure that the update_max_tr sees that
5908 * current_trace changed to nop_trace to keep it from
5909 * swapping the buffers after we resize it.
5910 * The update_max_tr is called from interrupts disabled
5911 * so a synchronized_sched() is sufficient.
5918 #ifdef CONFIG_TRACER_MAX_TRACE
5919 if (t->use_max_tr && !had_max_tr) {
5920 ret = tracing_alloc_snapshot_instance(tr);
5927 ret = tracer_init(t, tr);
5932 tr->current_trace = t;
5933 tr->current_trace->enabled++;
5934 trace_branch_enable(tr);
5936 mutex_unlock(&trace_types_lock);
5942 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5943 size_t cnt, loff_t *ppos)
5945 struct trace_array *tr = filp->private_data;
5946 char buf[MAX_TRACER_SIZE+1];
5953 if (cnt > MAX_TRACER_SIZE)
5954 cnt = MAX_TRACER_SIZE;
5956 if (copy_from_user(buf, ubuf, cnt))
5961 /* strip ending whitespace. */
5962 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5965 err = tracing_set_tracer(tr, buf);
5975 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5976 size_t cnt, loff_t *ppos)
5981 r = snprintf(buf, sizeof(buf), "%ld\n",
5982 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5983 if (r > sizeof(buf))
5985 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5989 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5990 size_t cnt, loff_t *ppos)
5995 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6005 tracing_thresh_read(struct file *filp, char __user *ubuf,
6006 size_t cnt, loff_t *ppos)
6008 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6012 tracing_thresh_write(struct file *filp, const char __user *ubuf,
6013 size_t cnt, loff_t *ppos)
6015 struct trace_array *tr = filp->private_data;
6018 mutex_lock(&trace_types_lock);
6019 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6023 if (tr->current_trace->update_thresh) {
6024 ret = tr->current_trace->update_thresh(tr);
6031 mutex_unlock(&trace_types_lock);
6036 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6039 tracing_max_lat_read(struct file *filp, char __user *ubuf,
6040 size_t cnt, loff_t *ppos)
6042 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
6046 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6047 size_t cnt, loff_t *ppos)
6049 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
6054 static int tracing_open_pipe(struct inode *inode, struct file *filp)
6056 struct trace_array *tr = inode->i_private;
6057 struct trace_iterator *iter;
6060 ret = tracing_check_open_get_tr(tr);
6064 mutex_lock(&trace_types_lock);
6066 /* create a buffer to store the information to pass to userspace */
6067 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6070 __trace_array_put(tr);
6074 trace_seq_init(&iter->seq);
6075 iter->trace = tr->current_trace;
6077 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6082 /* trace pipe does not show start of buffer */
6083 cpumask_setall(iter->started);
6085 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6086 iter->iter_flags |= TRACE_FILE_LAT_FMT;
6088 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6089 if (trace_clocks[tr->clock_id].in_ns)
6090 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6093 iter->array_buffer = &tr->array_buffer;
6094 iter->cpu_file = tracing_get_cpu(inode);
6095 mutex_init(&iter->mutex);
6096 filp->private_data = iter;
6098 if (iter->trace->pipe_open)
6099 iter->trace->pipe_open(iter);
6101 nonseekable_open(inode, filp);
6103 tr->current_trace->ref++;
6105 mutex_unlock(&trace_types_lock);
6110 __trace_array_put(tr);
6111 mutex_unlock(&trace_types_lock);
6115 static int tracing_release_pipe(struct inode *inode, struct file *file)
6117 struct trace_iterator *iter = file->private_data;
6118 struct trace_array *tr = inode->i_private;
6120 mutex_lock(&trace_types_lock);
6122 tr->current_trace->ref--;
6124 if (iter->trace->pipe_close)
6125 iter->trace->pipe_close(iter);
6127 mutex_unlock(&trace_types_lock);
6129 free_cpumask_var(iter->started);
6130 mutex_destroy(&iter->mutex);
6133 trace_array_put(tr);
6139 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6141 struct trace_array *tr = iter->tr;
6143 /* Iterators are static, they should be filled or empty */
6144 if (trace_buffer_iter(iter, iter->cpu_file))
6145 return EPOLLIN | EPOLLRDNORM;
6147 if (tr->trace_flags & TRACE_ITER_BLOCK)
6149 * Always select as readable when in blocking mode
6151 return EPOLLIN | EPOLLRDNORM;
6153 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6158 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6160 struct trace_iterator *iter = filp->private_data;
6162 return trace_poll(iter, filp, poll_table);
6165 /* Must be called with iter->mutex held. */
6166 static int tracing_wait_pipe(struct file *filp)
6168 struct trace_iterator *iter = filp->private_data;
6171 while (trace_empty(iter)) {
6173 if ((filp->f_flags & O_NONBLOCK)) {
6178 * We block until we read something and tracing is disabled.
6179 * We still block if tracing is disabled, but we have never
6180 * read anything. This allows a user to cat this file, and
6181 * then enable tracing. But after we have read something,
6182 * we give an EOF when tracing is again disabled.
6184 * iter->pos will be 0 if we haven't read anything.
6186 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6189 mutex_unlock(&iter->mutex);
6191 ret = wait_on_pipe(iter, 0);
6193 mutex_lock(&iter->mutex);
6206 tracing_read_pipe(struct file *filp, char __user *ubuf,
6207 size_t cnt, loff_t *ppos)
6209 struct trace_iterator *iter = filp->private_data;
6213 * Avoid more than one consumer on a single file descriptor
6214 * This is just a matter of traces coherency, the ring buffer itself
6217 mutex_lock(&iter->mutex);
6219 /* return any leftover data */
6220 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6224 trace_seq_init(&iter->seq);
6226 if (iter->trace->read) {
6227 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6233 sret = tracing_wait_pipe(filp);
6237 /* stop when tracing is finished */
6238 if (trace_empty(iter)) {
6243 if (cnt >= PAGE_SIZE)
6244 cnt = PAGE_SIZE - 1;
6246 /* reset all but tr, trace, and overruns */
6247 memset(&iter->seq, 0,
6248 sizeof(struct trace_iterator) -
6249 offsetof(struct trace_iterator, seq));
6250 cpumask_clear(iter->started);
6251 trace_seq_init(&iter->seq);
6254 trace_event_read_lock();
6255 trace_access_lock(iter->cpu_file);
6256 while (trace_find_next_entry_inc(iter) != NULL) {
6257 enum print_line_t ret;
6258 int save_len = iter->seq.seq.len;
6260 ret = print_trace_line(iter);
6261 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6262 /* don't print partial lines */
6263 iter->seq.seq.len = save_len;
6266 if (ret != TRACE_TYPE_NO_CONSUME)
6267 trace_consume(iter);
6269 if (trace_seq_used(&iter->seq) >= cnt)
6273 * Setting the full flag means we reached the trace_seq buffer
6274 * size and we should leave by partial output condition above.
6275 * One of the trace_seq_* functions is not used properly.
6277 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6280 trace_access_unlock(iter->cpu_file);
6281 trace_event_read_unlock();
6283 /* Now copy what we have to the user */
6284 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6285 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6286 trace_seq_init(&iter->seq);
6289 * If there was nothing to send to user, in spite of consuming trace
6290 * entries, go back to wait for more entries.
6296 mutex_unlock(&iter->mutex);
6301 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6304 __free_page(spd->pages[idx]);
6308 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6314 /* Seq buffer is page-sized, exactly what we need. */
6316 save_len = iter->seq.seq.len;
6317 ret = print_trace_line(iter);
6319 if (trace_seq_has_overflowed(&iter->seq)) {
6320 iter->seq.seq.len = save_len;
6325 * This should not be hit, because it should only
6326 * be set if the iter->seq overflowed. But check it
6327 * anyway to be safe.
6329 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6330 iter->seq.seq.len = save_len;
6334 count = trace_seq_used(&iter->seq) - save_len;
6337 iter->seq.seq.len = save_len;
6341 if (ret != TRACE_TYPE_NO_CONSUME)
6342 trace_consume(iter);
6344 if (!trace_find_next_entry_inc(iter)) {
6354 static ssize_t tracing_splice_read_pipe(struct file *filp,
6356 struct pipe_inode_info *pipe,
6360 struct page *pages_def[PIPE_DEF_BUFFERS];
6361 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6362 struct trace_iterator *iter = filp->private_data;
6363 struct splice_pipe_desc spd = {
6365 .partial = partial_def,
6366 .nr_pages = 0, /* This gets updated below. */
6367 .nr_pages_max = PIPE_DEF_BUFFERS,
6368 .ops = &default_pipe_buf_ops,
6369 .spd_release = tracing_spd_release_pipe,
6375 if (splice_grow_spd(pipe, &spd))
6378 mutex_lock(&iter->mutex);
6380 if (iter->trace->splice_read) {
6381 ret = iter->trace->splice_read(iter, filp,
6382 ppos, pipe, len, flags);
6387 ret = tracing_wait_pipe(filp);
6391 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6396 trace_event_read_lock();
6397 trace_access_lock(iter->cpu_file);
6399 /* Fill as many pages as possible. */
6400 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6401 spd.pages[i] = alloc_page(GFP_KERNEL);
6405 rem = tracing_fill_pipe_page(rem, iter);
6407 /* Copy the data into the page, so we can start over. */
6408 ret = trace_seq_to_buffer(&iter->seq,
6409 page_address(spd.pages[i]),
6410 trace_seq_used(&iter->seq));
6412 __free_page(spd.pages[i]);
6415 spd.partial[i].offset = 0;
6416 spd.partial[i].len = trace_seq_used(&iter->seq);
6418 trace_seq_init(&iter->seq);
6421 trace_access_unlock(iter->cpu_file);
6422 trace_event_read_unlock();
6423 mutex_unlock(&iter->mutex);
6428 ret = splice_to_pipe(pipe, &spd);
6432 splice_shrink_spd(&spd);
6436 mutex_unlock(&iter->mutex);
6441 tracing_entries_read(struct file *filp, char __user *ubuf,
6442 size_t cnt, loff_t *ppos)
6444 struct inode *inode = file_inode(filp);
6445 struct trace_array *tr = inode->i_private;
6446 int cpu = tracing_get_cpu(inode);
6451 mutex_lock(&trace_types_lock);
6453 if (cpu == RING_BUFFER_ALL_CPUS) {
6454 int cpu, buf_size_same;
6459 /* check if all cpu sizes are same */
6460 for_each_tracing_cpu(cpu) {
6461 /* fill in the size from first enabled cpu */
6463 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6464 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6470 if (buf_size_same) {
6471 if (!ring_buffer_expanded)
6472 r = sprintf(buf, "%lu (expanded: %lu)\n",
6474 trace_buf_size >> 10);
6476 r = sprintf(buf, "%lu\n", size >> 10);
6478 r = sprintf(buf, "X\n");
6480 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6482 mutex_unlock(&trace_types_lock);
6484 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6489 tracing_entries_write(struct file *filp, const char __user *ubuf,
6490 size_t cnt, loff_t *ppos)
6492 struct inode *inode = file_inode(filp);
6493 struct trace_array *tr = inode->i_private;
6497 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6501 /* must have at least 1 entry */
6505 /* value is in KB */
6507 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6517 tracing_total_entries_read(struct file *filp, char __user *ubuf,
6518 size_t cnt, loff_t *ppos)
6520 struct trace_array *tr = filp->private_data;
6523 unsigned long size = 0, expanded_size = 0;
6525 mutex_lock(&trace_types_lock);
6526 for_each_tracing_cpu(cpu) {
6527 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
6528 if (!ring_buffer_expanded)
6529 expanded_size += trace_buf_size >> 10;
6531 if (ring_buffer_expanded)
6532 r = sprintf(buf, "%lu\n", size);
6534 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6535 mutex_unlock(&trace_types_lock);
6537 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6541 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6542 size_t cnt, loff_t *ppos)
6545 * There is no need to read what the user has written, this function
6546 * is just to make sure that there is no error when "echo" is used
6555 tracing_free_buffer_release(struct inode *inode, struct file *filp)
6557 struct trace_array *tr = inode->i_private;
6559 /* disable tracing ? */
6560 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6561 tracer_tracing_off(tr);
6562 /* resize the ring buffer to 0 */
6563 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6565 trace_array_put(tr);
6571 tracing_mark_write(struct file *filp, const char __user *ubuf,
6572 size_t cnt, loff_t *fpos)
6574 struct trace_array *tr = filp->private_data;
6575 struct ring_buffer_event *event;
6576 enum event_trigger_type tt = ETT_NONE;
6577 struct trace_buffer *buffer;
6578 struct print_entry *entry;
6579 unsigned long irq_flags;
6584 /* Used in tracing_mark_raw_write() as well */
6585 #define FAULTED_STR "<faulted>"
6586 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6588 if (tracing_disabled)
6591 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6594 if (cnt > TRACE_BUF_SIZE)
6595 cnt = TRACE_BUF_SIZE;
6597 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6599 local_save_flags(irq_flags);
6600 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6602 /* If less than "<faulted>", then make sure we can still add that */
6603 if (cnt < FAULTED_SIZE)
6604 size += FAULTED_SIZE - cnt;
6606 buffer = tr->array_buffer.buffer;
6607 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6608 irq_flags, preempt_count());
6609 if (unlikely(!event))
6610 /* Ring buffer disabled, return as if not open for write */
6613 entry = ring_buffer_event_data(event);
6614 entry->ip = _THIS_IP_;
6616 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6618 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6625 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6626 /* do not add \n before testing triggers, but add \0 */
6627 entry->buf[cnt] = '\0';
6628 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6631 if (entry->buf[cnt - 1] != '\n') {
6632 entry->buf[cnt] = '\n';
6633 entry->buf[cnt + 1] = '\0';
6635 entry->buf[cnt] = '\0';
6637 __buffer_unlock_commit(buffer, event);
6640 event_triggers_post_call(tr->trace_marker_file, tt);
6648 /* Limit it for now to 3K (including tag) */
6649 #define RAW_DATA_MAX_SIZE (1024*3)
6652 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6653 size_t cnt, loff_t *fpos)
6655 struct trace_array *tr = filp->private_data;
6656 struct ring_buffer_event *event;
6657 struct trace_buffer *buffer;
6658 struct raw_data_entry *entry;
6659 unsigned long irq_flags;
6664 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6666 if (tracing_disabled)
6669 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6672 /* The marker must at least have a tag id */
6673 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6676 if (cnt > TRACE_BUF_SIZE)
6677 cnt = TRACE_BUF_SIZE;
6679 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6681 local_save_flags(irq_flags);
6682 size = sizeof(*entry) + cnt;
6683 if (cnt < FAULT_SIZE_ID)
6684 size += FAULT_SIZE_ID - cnt;
6686 buffer = tr->array_buffer.buffer;
6687 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6688 irq_flags, preempt_count());
6690 /* Ring buffer disabled, return as if not open for write */
6693 entry = ring_buffer_event_data(event);
6695 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6698 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6703 __buffer_unlock_commit(buffer, event);
6711 static int tracing_clock_show(struct seq_file *m, void *v)
6713 struct trace_array *tr = m->private;
6716 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6718 "%s%s%s%s", i ? " " : "",
6719 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6720 i == tr->clock_id ? "]" : "");
6726 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6730 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6731 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6734 if (i == ARRAY_SIZE(trace_clocks))
6737 mutex_lock(&trace_types_lock);
6741 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
6744 * New clock may not be consistent with the previous clock.
6745 * Reset the buffer so that it doesn't have incomparable timestamps.
6747 tracing_reset_online_cpus(&tr->array_buffer);
6749 #ifdef CONFIG_TRACER_MAX_TRACE
6750 if (tr->max_buffer.buffer)
6751 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6752 tracing_reset_online_cpus(&tr->max_buffer);
6755 mutex_unlock(&trace_types_lock);
6760 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6761 size_t cnt, loff_t *fpos)
6763 struct seq_file *m = filp->private_data;
6764 struct trace_array *tr = m->private;
6766 const char *clockstr;
6769 if (cnt >= sizeof(buf))
6772 if (copy_from_user(buf, ubuf, cnt))
6777 clockstr = strstrip(buf);
6779 ret = tracing_set_clock(tr, clockstr);
6788 static int tracing_clock_open(struct inode *inode, struct file *file)
6790 struct trace_array *tr = inode->i_private;
6793 ret = tracing_check_open_get_tr(tr);
6797 ret = single_open(file, tracing_clock_show, inode->i_private);
6799 trace_array_put(tr);
6804 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6806 struct trace_array *tr = m->private;
6808 mutex_lock(&trace_types_lock);
6810 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
6811 seq_puts(m, "delta [absolute]\n");
6813 seq_puts(m, "[delta] absolute\n");
6815 mutex_unlock(&trace_types_lock);
6820 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6822 struct trace_array *tr = inode->i_private;
6825 ret = tracing_check_open_get_tr(tr);
6829 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6831 trace_array_put(tr);
6836 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6840 mutex_lock(&trace_types_lock);
6842 if (abs && tr->time_stamp_abs_ref++)
6846 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6851 if (--tr->time_stamp_abs_ref)
6855 ring_buffer_set_time_stamp_abs(tr->array_buffer.buffer, abs);
6857 #ifdef CONFIG_TRACER_MAX_TRACE
6858 if (tr->max_buffer.buffer)
6859 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6862 mutex_unlock(&trace_types_lock);
6867 struct ftrace_buffer_info {
6868 struct trace_iterator iter;
6870 unsigned int spare_cpu;
6874 #ifdef CONFIG_TRACER_SNAPSHOT
6875 static int tracing_snapshot_open(struct inode *inode, struct file *file)
6877 struct trace_array *tr = inode->i_private;
6878 struct trace_iterator *iter;
6882 ret = tracing_check_open_get_tr(tr);
6886 if (file->f_mode & FMODE_READ) {
6887 iter = __tracing_open(inode, file, true);
6889 ret = PTR_ERR(iter);
6891 /* Writes still need the seq_file to hold the private data */
6893 m = kzalloc(sizeof(*m), GFP_KERNEL);
6896 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6904 iter->array_buffer = &tr->max_buffer;
6905 iter->cpu_file = tracing_get_cpu(inode);
6907 file->private_data = m;
6911 trace_array_put(tr);
6917 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6920 struct seq_file *m = filp->private_data;
6921 struct trace_iterator *iter = m->private;
6922 struct trace_array *tr = iter->tr;
6926 ret = tracing_update_buffers();
6930 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6934 mutex_lock(&trace_types_lock);
6936 if (tr->current_trace->use_max_tr) {
6941 arch_spin_lock(&tr->max_lock);
6942 if (tr->cond_snapshot)
6944 arch_spin_unlock(&tr->max_lock);
6950 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6954 if (tr->allocated_snapshot)
6958 /* Only allow per-cpu swap if the ring buffer supports it */
6959 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6960 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6965 if (tr->allocated_snapshot)
6966 ret = resize_buffer_duplicate_size(&tr->max_buffer,
6967 &tr->array_buffer, iter->cpu_file);
6969 ret = tracing_alloc_snapshot_instance(tr);
6972 local_irq_disable();
6973 /* Now, we're going to swap */
6974 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6975 update_max_tr(tr, current, smp_processor_id(), NULL);
6977 update_max_tr_single(tr, current, iter->cpu_file);
6981 if (tr->allocated_snapshot) {
6982 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6983 tracing_reset_online_cpus(&tr->max_buffer);
6985 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
6995 mutex_unlock(&trace_types_lock);
6999 static int tracing_snapshot_release(struct inode *inode, struct file *file)
7001 struct seq_file *m = file->private_data;
7004 ret = tracing_release(inode, file);
7006 if (file->f_mode & FMODE_READ)
7009 /* If write only, the seq_file is just a stub */
7017 static int tracing_buffers_open(struct inode *inode, struct file *filp);
7018 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7019 size_t count, loff_t *ppos);
7020 static int tracing_buffers_release(struct inode *inode, struct file *file);
7021 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7022 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7024 static int snapshot_raw_open(struct inode *inode, struct file *filp)
7026 struct ftrace_buffer_info *info;
7029 /* The following checks for tracefs lockdown */
7030 ret = tracing_buffers_open(inode, filp);
7034 info = filp->private_data;
7036 if (info->iter.trace->use_max_tr) {
7037 tracing_buffers_release(inode, filp);
7041 info->iter.snapshot = true;
7042 info->iter.array_buffer = &info->iter.tr->max_buffer;
7047 #endif /* CONFIG_TRACER_SNAPSHOT */
7050 static const struct file_operations tracing_thresh_fops = {
7051 .open = tracing_open_generic,
7052 .read = tracing_thresh_read,
7053 .write = tracing_thresh_write,
7054 .llseek = generic_file_llseek,
7057 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7058 static const struct file_operations tracing_max_lat_fops = {
7059 .open = tracing_open_generic,
7060 .read = tracing_max_lat_read,
7061 .write = tracing_max_lat_write,
7062 .llseek = generic_file_llseek,
7066 static const struct file_operations set_tracer_fops = {
7067 .open = tracing_open_generic,
7068 .read = tracing_set_trace_read,
7069 .write = tracing_set_trace_write,
7070 .llseek = generic_file_llseek,
7073 static const struct file_operations tracing_pipe_fops = {
7074 .open = tracing_open_pipe,
7075 .poll = tracing_poll_pipe,
7076 .read = tracing_read_pipe,
7077 .splice_read = tracing_splice_read_pipe,
7078 .release = tracing_release_pipe,
7079 .llseek = no_llseek,
7082 static const struct file_operations tracing_entries_fops = {
7083 .open = tracing_open_generic_tr,
7084 .read = tracing_entries_read,
7085 .write = tracing_entries_write,
7086 .llseek = generic_file_llseek,
7087 .release = tracing_release_generic_tr,
7090 static const struct file_operations tracing_total_entries_fops = {
7091 .open = tracing_open_generic_tr,
7092 .read = tracing_total_entries_read,
7093 .llseek = generic_file_llseek,
7094 .release = tracing_release_generic_tr,
7097 static const struct file_operations tracing_free_buffer_fops = {
7098 .open = tracing_open_generic_tr,
7099 .write = tracing_free_buffer_write,
7100 .release = tracing_free_buffer_release,
7103 static const struct file_operations tracing_mark_fops = {
7104 .open = tracing_open_generic_tr,
7105 .write = tracing_mark_write,
7106 .llseek = generic_file_llseek,
7107 .release = tracing_release_generic_tr,
7110 static const struct file_operations tracing_mark_raw_fops = {
7111 .open = tracing_open_generic_tr,
7112 .write = tracing_mark_raw_write,
7113 .llseek = generic_file_llseek,
7114 .release = tracing_release_generic_tr,
7117 static const struct file_operations trace_clock_fops = {
7118 .open = tracing_clock_open,
7120 .llseek = seq_lseek,
7121 .release = tracing_single_release_tr,
7122 .write = tracing_clock_write,
7125 static const struct file_operations trace_time_stamp_mode_fops = {
7126 .open = tracing_time_stamp_mode_open,
7128 .llseek = seq_lseek,
7129 .release = tracing_single_release_tr,
7132 #ifdef CONFIG_TRACER_SNAPSHOT
7133 static const struct file_operations snapshot_fops = {
7134 .open = tracing_snapshot_open,
7136 .write = tracing_snapshot_write,
7137 .llseek = tracing_lseek,
7138 .release = tracing_snapshot_release,
7141 static const struct file_operations snapshot_raw_fops = {
7142 .open = snapshot_raw_open,
7143 .read = tracing_buffers_read,
7144 .release = tracing_buffers_release,
7145 .splice_read = tracing_buffers_splice_read,
7146 .llseek = no_llseek,
7149 #endif /* CONFIG_TRACER_SNAPSHOT */
7151 #define TRACING_LOG_ERRS_MAX 8
7152 #define TRACING_LOG_LOC_MAX 128
7154 #define CMD_PREFIX " Command: "
7157 const char **errs; /* ptr to loc-specific array of err strings */
7158 u8 type; /* index into errs -> specific err string */
7159 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
7163 struct tracing_log_err {
7164 struct list_head list;
7165 struct err_info info;
7166 char loc[TRACING_LOG_LOC_MAX]; /* err location */
7167 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
7170 static DEFINE_MUTEX(tracing_err_log_lock);
7172 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
7174 struct tracing_log_err *err;
7176 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7177 err = kzalloc(sizeof(*err), GFP_KERNEL);
7179 err = ERR_PTR(-ENOMEM);
7180 tr->n_err_log_entries++;
7185 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7186 list_del(&err->list);
7192 * err_pos - find the position of a string within a command for error careting
7193 * @cmd: The tracing command that caused the error
7194 * @str: The string to position the caret at within @cmd
7196 * Finds the position of the first occurence of @str within @cmd. The
7197 * return value can be passed to tracing_log_err() for caret placement
7200 * Returns the index within @cmd of the first occurence of @str or 0
7201 * if @str was not found.
7203 unsigned int err_pos(char *cmd, const char *str)
7207 if (WARN_ON(!strlen(cmd)))
7210 found = strstr(cmd, str);
7218 * tracing_log_err - write an error to the tracing error log
7219 * @tr: The associated trace array for the error (NULL for top level array)
7220 * @loc: A string describing where the error occurred
7221 * @cmd: The tracing command that caused the error
7222 * @errs: The array of loc-specific static error strings
7223 * @type: The index into errs[], which produces the specific static err string
7224 * @pos: The position the caret should be placed in the cmd
7226 * Writes an error into tracing/error_log of the form:
7228 * <loc>: error: <text>
7232 * tracing/error_log is a small log file containing the last
7233 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7234 * unless there has been a tracing error, and the error log can be
7235 * cleared and have its memory freed by writing the empty string in
7236 * truncation mode to it i.e. echo > tracing/error_log.
7238 * NOTE: the @errs array along with the @type param are used to
7239 * produce a static error string - this string is not copied and saved
7240 * when the error is logged - only a pointer to it is saved. See
7241 * existing callers for examples of how static strings are typically
7242 * defined for use with tracing_log_err().
7244 void tracing_log_err(struct trace_array *tr,
7245 const char *loc, const char *cmd,
7246 const char **errs, u8 type, u8 pos)
7248 struct tracing_log_err *err;
7253 mutex_lock(&tracing_err_log_lock);
7254 err = get_tracing_log_err(tr);
7255 if (PTR_ERR(err) == -ENOMEM) {
7256 mutex_unlock(&tracing_err_log_lock);
7260 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7261 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7263 err->info.errs = errs;
7264 err->info.type = type;
7265 err->info.pos = pos;
7266 err->info.ts = local_clock();
7268 list_add_tail(&err->list, &tr->err_log);
7269 mutex_unlock(&tracing_err_log_lock);
7272 static void clear_tracing_err_log(struct trace_array *tr)
7274 struct tracing_log_err *err, *next;
7276 mutex_lock(&tracing_err_log_lock);
7277 list_for_each_entry_safe(err, next, &tr->err_log, list) {
7278 list_del(&err->list);
7282 tr->n_err_log_entries = 0;
7283 mutex_unlock(&tracing_err_log_lock);
7286 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7288 struct trace_array *tr = m->private;
7290 mutex_lock(&tracing_err_log_lock);
7292 return seq_list_start(&tr->err_log, *pos);
7295 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7297 struct trace_array *tr = m->private;
7299 return seq_list_next(v, &tr->err_log, pos);
7302 static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7304 mutex_unlock(&tracing_err_log_lock);
7307 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7311 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7313 for (i = 0; i < pos; i++)
7318 static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7320 struct tracing_log_err *err = v;
7323 const char *err_text = err->info.errs[err->info.type];
7324 u64 sec = err->info.ts;
7327 nsec = do_div(sec, NSEC_PER_SEC);
7328 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7329 err->loc, err_text);
7330 seq_printf(m, "%s", err->cmd);
7331 tracing_err_log_show_pos(m, err->info.pos);
7337 static const struct seq_operations tracing_err_log_seq_ops = {
7338 .start = tracing_err_log_seq_start,
7339 .next = tracing_err_log_seq_next,
7340 .stop = tracing_err_log_seq_stop,
7341 .show = tracing_err_log_seq_show
7344 static int tracing_err_log_open(struct inode *inode, struct file *file)
7346 struct trace_array *tr = inode->i_private;
7349 ret = tracing_check_open_get_tr(tr);
7353 /* If this file was opened for write, then erase contents */
7354 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7355 clear_tracing_err_log(tr);
7357 if (file->f_mode & FMODE_READ) {
7358 ret = seq_open(file, &tracing_err_log_seq_ops);
7360 struct seq_file *m = file->private_data;
7363 trace_array_put(tr);
7369 static ssize_t tracing_err_log_write(struct file *file,
7370 const char __user *buffer,
7371 size_t count, loff_t *ppos)
7376 static int tracing_err_log_release(struct inode *inode, struct file *file)
7378 struct trace_array *tr = inode->i_private;
7380 trace_array_put(tr);
7382 if (file->f_mode & FMODE_READ)
7383 seq_release(inode, file);
7388 static const struct file_operations tracing_err_log_fops = {
7389 .open = tracing_err_log_open,
7390 .write = tracing_err_log_write,
7392 .llseek = seq_lseek,
7393 .release = tracing_err_log_release,
7396 static int tracing_buffers_open(struct inode *inode, struct file *filp)
7398 struct trace_array *tr = inode->i_private;
7399 struct ftrace_buffer_info *info;
7402 ret = tracing_check_open_get_tr(tr);
7406 info = kzalloc(sizeof(*info), GFP_KERNEL);
7408 trace_array_put(tr);
7412 mutex_lock(&trace_types_lock);
7415 info->iter.cpu_file = tracing_get_cpu(inode);
7416 info->iter.trace = tr->current_trace;
7417 info->iter.array_buffer = &tr->array_buffer;
7419 /* Force reading ring buffer for first read */
7420 info->read = (unsigned int)-1;
7422 filp->private_data = info;
7424 tr->current_trace->ref++;
7426 mutex_unlock(&trace_types_lock);
7428 ret = nonseekable_open(inode, filp);
7430 trace_array_put(tr);
7436 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7438 struct ftrace_buffer_info *info = filp->private_data;
7439 struct trace_iterator *iter = &info->iter;
7441 return trace_poll(iter, filp, poll_table);
7445 tracing_buffers_read(struct file *filp, char __user *ubuf,
7446 size_t count, loff_t *ppos)
7448 struct ftrace_buffer_info *info = filp->private_data;
7449 struct trace_iterator *iter = &info->iter;
7456 #ifdef CONFIG_TRACER_MAX_TRACE
7457 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7462 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
7464 if (IS_ERR(info->spare)) {
7465 ret = PTR_ERR(info->spare);
7468 info->spare_cpu = iter->cpu_file;
7474 /* Do we have previous read data to read? */
7475 if (info->read < PAGE_SIZE)
7479 trace_access_lock(iter->cpu_file);
7480 ret = ring_buffer_read_page(iter->array_buffer->buffer,
7484 trace_access_unlock(iter->cpu_file);
7487 if (trace_empty(iter)) {
7488 if ((filp->f_flags & O_NONBLOCK))
7491 ret = wait_on_pipe(iter, 0);
7502 size = PAGE_SIZE - info->read;
7506 ret = copy_to_user(ubuf, info->spare + info->read, size);
7518 static int tracing_buffers_release(struct inode *inode, struct file *file)
7520 struct ftrace_buffer_info *info = file->private_data;
7521 struct trace_iterator *iter = &info->iter;
7523 mutex_lock(&trace_types_lock);
7525 iter->tr->current_trace->ref--;
7527 __trace_array_put(iter->tr);
7530 ring_buffer_free_read_page(iter->array_buffer->buffer,
7531 info->spare_cpu, info->spare);
7534 mutex_unlock(&trace_types_lock);
7540 struct trace_buffer *buffer;
7543 refcount_t refcount;
7546 static void buffer_ref_release(struct buffer_ref *ref)
7548 if (!refcount_dec_and_test(&ref->refcount))
7550 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7554 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7555 struct pipe_buffer *buf)
7557 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7559 buffer_ref_release(ref);
7563 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7564 struct pipe_buffer *buf)
7566 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7568 if (refcount_read(&ref->refcount) > INT_MAX/2)
7571 refcount_inc(&ref->refcount);
7575 /* Pipe buffer operations for a buffer. */
7576 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7577 .confirm = generic_pipe_buf_confirm,
7578 .release = buffer_pipe_buf_release,
7579 .steal = generic_pipe_buf_nosteal,
7580 .get = buffer_pipe_buf_get,
7584 * Callback from splice_to_pipe(), if we need to release some pages
7585 * at the end of the spd in case we error'ed out in filling the pipe.
7587 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7589 struct buffer_ref *ref =
7590 (struct buffer_ref *)spd->partial[i].private;
7592 buffer_ref_release(ref);
7593 spd->partial[i].private = 0;
7597 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7598 struct pipe_inode_info *pipe, size_t len,
7601 struct ftrace_buffer_info *info = file->private_data;
7602 struct trace_iterator *iter = &info->iter;
7603 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7604 struct page *pages_def[PIPE_DEF_BUFFERS];
7605 struct splice_pipe_desc spd = {
7607 .partial = partial_def,
7608 .nr_pages_max = PIPE_DEF_BUFFERS,
7609 .ops = &buffer_pipe_buf_ops,
7610 .spd_release = buffer_spd_release,
7612 struct buffer_ref *ref;
7616 #ifdef CONFIG_TRACER_MAX_TRACE
7617 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7621 if (*ppos & (PAGE_SIZE - 1))
7624 if (len & (PAGE_SIZE - 1)) {
7625 if (len < PAGE_SIZE)
7630 if (splice_grow_spd(pipe, &spd))
7634 trace_access_lock(iter->cpu_file);
7635 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7637 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
7641 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
7647 refcount_set(&ref->refcount, 1);
7648 ref->buffer = iter->array_buffer->buffer;
7649 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
7650 if (IS_ERR(ref->page)) {
7651 ret = PTR_ERR(ref->page);
7656 ref->cpu = iter->cpu_file;
7658 r = ring_buffer_read_page(ref->buffer, &ref->page,
7659 len, iter->cpu_file, 1);
7661 ring_buffer_free_read_page(ref->buffer, ref->cpu,
7667 page = virt_to_page(ref->page);
7669 spd.pages[i] = page;
7670 spd.partial[i].len = PAGE_SIZE;
7671 spd.partial[i].offset = 0;
7672 spd.partial[i].private = (unsigned long)ref;
7676 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7679 trace_access_unlock(iter->cpu_file);
7682 /* did we read anything? */
7683 if (!spd.nr_pages) {
7688 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7691 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
7698 ret = splice_to_pipe(pipe, &spd);
7700 splice_shrink_spd(&spd);
7705 static const struct file_operations tracing_buffers_fops = {
7706 .open = tracing_buffers_open,
7707 .read = tracing_buffers_read,
7708 .poll = tracing_buffers_poll,
7709 .release = tracing_buffers_release,
7710 .splice_read = tracing_buffers_splice_read,
7711 .llseek = no_llseek,
7715 tracing_stats_read(struct file *filp, char __user *ubuf,
7716 size_t count, loff_t *ppos)
7718 struct inode *inode = file_inode(filp);
7719 struct trace_array *tr = inode->i_private;
7720 struct array_buffer *trace_buf = &tr->array_buffer;
7721 int cpu = tracing_get_cpu(inode);
7722 struct trace_seq *s;
7724 unsigned long long t;
7725 unsigned long usec_rem;
7727 s = kmalloc(sizeof(*s), GFP_KERNEL);
7733 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7734 trace_seq_printf(s, "entries: %ld\n", cnt);
7736 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7737 trace_seq_printf(s, "overrun: %ld\n", cnt);
7739 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7740 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7742 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7743 trace_seq_printf(s, "bytes: %ld\n", cnt);
7745 if (trace_clocks[tr->clock_id].in_ns) {
7746 /* local or global for trace_clock */
7747 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7748 usec_rem = do_div(t, USEC_PER_SEC);
7749 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7752 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
7753 usec_rem = do_div(t, USEC_PER_SEC);
7754 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7756 /* counter or tsc mode for trace_clock */
7757 trace_seq_printf(s, "oldest event ts: %llu\n",
7758 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7760 trace_seq_printf(s, "now ts: %llu\n",
7761 ring_buffer_time_stamp(trace_buf->buffer, cpu));
7764 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7765 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7767 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7768 trace_seq_printf(s, "read events: %ld\n", cnt);
7770 count = simple_read_from_buffer(ubuf, count, ppos,
7771 s->buffer, trace_seq_used(s));
7778 static const struct file_operations tracing_stats_fops = {
7779 .open = tracing_open_generic_tr,
7780 .read = tracing_stats_read,
7781 .llseek = generic_file_llseek,
7782 .release = tracing_release_generic_tr,
7785 #ifdef CONFIG_DYNAMIC_FTRACE
7788 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7789 size_t cnt, loff_t *ppos)
7795 /* 256 should be plenty to hold the amount needed */
7796 buf = kmalloc(256, GFP_KERNEL);
7800 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
7801 ftrace_update_tot_cnt,
7802 ftrace_number_of_pages,
7803 ftrace_number_of_groups);
7805 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7810 static const struct file_operations tracing_dyn_info_fops = {
7811 .open = tracing_open_generic,
7812 .read = tracing_read_dyn_info,
7813 .llseek = generic_file_llseek,
7815 #endif /* CONFIG_DYNAMIC_FTRACE */
7817 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7819 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7820 struct trace_array *tr, struct ftrace_probe_ops *ops,
7823 tracing_snapshot_instance(tr);
7827 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7828 struct trace_array *tr, struct ftrace_probe_ops *ops,
7831 struct ftrace_func_mapper *mapper = data;
7835 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7845 tracing_snapshot_instance(tr);
7849 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7850 struct ftrace_probe_ops *ops, void *data)
7852 struct ftrace_func_mapper *mapper = data;
7855 seq_printf(m, "%ps:", (void *)ip);
7857 seq_puts(m, "snapshot");
7860 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7863 seq_printf(m, ":count=%ld\n", *count);
7865 seq_puts(m, ":unlimited\n");
7871 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7872 unsigned long ip, void *init_data, void **data)
7874 struct ftrace_func_mapper *mapper = *data;
7877 mapper = allocate_ftrace_func_mapper();
7883 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7887 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7888 unsigned long ip, void *data)
7890 struct ftrace_func_mapper *mapper = data;
7895 free_ftrace_func_mapper(mapper, NULL);
7899 ftrace_func_mapper_remove_ip(mapper, ip);
7902 static struct ftrace_probe_ops snapshot_probe_ops = {
7903 .func = ftrace_snapshot,
7904 .print = ftrace_snapshot_print,
7907 static struct ftrace_probe_ops snapshot_count_probe_ops = {
7908 .func = ftrace_count_snapshot,
7909 .print = ftrace_snapshot_print,
7910 .init = ftrace_snapshot_init,
7911 .free = ftrace_snapshot_free,
7915 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7916 char *glob, char *cmd, char *param, int enable)
7918 struct ftrace_probe_ops *ops;
7919 void *count = (void *)-1;
7926 /* hash funcs only work with set_ftrace_filter */
7930 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7933 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
7938 number = strsep(¶m, ":");
7940 if (!strlen(number))
7944 * We use the callback data field (which is a pointer)
7947 ret = kstrtoul(number, 0, (unsigned long *)&count);
7952 ret = tracing_alloc_snapshot_instance(tr);
7956 ret = register_ftrace_function_probe(glob, tr, ops, count);
7959 return ret < 0 ? ret : 0;
7962 static struct ftrace_func_command ftrace_snapshot_cmd = {
7964 .func = ftrace_trace_snapshot_callback,
7967 static __init int register_snapshot_cmd(void)
7969 return register_ftrace_command(&ftrace_snapshot_cmd);
7972 static inline __init int register_snapshot_cmd(void) { return 0; }
7973 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7975 static struct dentry *tracing_get_dentry(struct trace_array *tr)
7977 if (WARN_ON(!tr->dir))
7978 return ERR_PTR(-ENODEV);
7980 /* Top directory uses NULL as the parent */
7981 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7984 /* All sub buffers have a descriptor */
7988 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7990 struct dentry *d_tracer;
7993 return tr->percpu_dir;
7995 d_tracer = tracing_get_dentry(tr);
7996 if (IS_ERR(d_tracer))
7999 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8001 MEM_FAIL(!tr->percpu_dir,
8002 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8004 return tr->percpu_dir;
8007 static struct dentry *
8008 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8009 void *data, long cpu, const struct file_operations *fops)
8011 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8013 if (ret) /* See tracing_get_cpu() */
8014 d_inode(ret)->i_cdev = (void *)(cpu + 1);
8019 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8021 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8022 struct dentry *d_cpu;
8023 char cpu_dir[30]; /* 30 characters should be more than enough */
8028 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8029 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8031 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8035 /* per cpu trace_pipe */
8036 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
8037 tr, cpu, &tracing_pipe_fops);
8040 trace_create_cpu_file("trace", 0644, d_cpu,
8041 tr, cpu, &tracing_fops);
8043 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
8044 tr, cpu, &tracing_buffers_fops);
8046 trace_create_cpu_file("stats", 0444, d_cpu,
8047 tr, cpu, &tracing_stats_fops);
8049 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
8050 tr, cpu, &tracing_entries_fops);
8052 #ifdef CONFIG_TRACER_SNAPSHOT
8053 trace_create_cpu_file("snapshot", 0644, d_cpu,
8054 tr, cpu, &snapshot_fops);
8056 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
8057 tr, cpu, &snapshot_raw_fops);
8061 #ifdef CONFIG_FTRACE_SELFTEST
8062 /* Let selftest have access to static functions in this file */
8063 #include "trace_selftest.c"
8067 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8070 struct trace_option_dentry *topt = filp->private_data;
8073 if (topt->flags->val & topt->opt->bit)
8078 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8082 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8085 struct trace_option_dentry *topt = filp->private_data;
8089 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8093 if (val != 0 && val != 1)
8096 if (!!(topt->flags->val & topt->opt->bit) != val) {
8097 mutex_lock(&trace_types_lock);
8098 ret = __set_tracer_option(topt->tr, topt->flags,
8100 mutex_unlock(&trace_types_lock);
8111 static const struct file_operations trace_options_fops = {
8112 .open = tracing_open_generic,
8113 .read = trace_options_read,
8114 .write = trace_options_write,
8115 .llseek = generic_file_llseek,
8119 * In order to pass in both the trace_array descriptor as well as the index
8120 * to the flag that the trace option file represents, the trace_array
8121 * has a character array of trace_flags_index[], which holds the index
8122 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8123 * The address of this character array is passed to the flag option file
8124 * read/write callbacks.
8126 * In order to extract both the index and the trace_array descriptor,
8127 * get_tr_index() uses the following algorithm.
8131 * As the pointer itself contains the address of the index (remember
8134 * Then to get the trace_array descriptor, by subtracting that index
8135 * from the ptr, we get to the start of the index itself.
8137 * ptr - idx == &index[0]
8139 * Then a simple container_of() from that pointer gets us to the
8140 * trace_array descriptor.
8142 static void get_tr_index(void *data, struct trace_array **ptr,
8143 unsigned int *pindex)
8145 *pindex = *(unsigned char *)data;
8147 *ptr = container_of(data - *pindex, struct trace_array,
8152 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8155 void *tr_index = filp->private_data;
8156 struct trace_array *tr;
8160 get_tr_index(tr_index, &tr, &index);
8162 if (tr->trace_flags & (1 << index))
8167 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8171 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8174 void *tr_index = filp->private_data;
8175 struct trace_array *tr;
8180 get_tr_index(tr_index, &tr, &index);
8182 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8186 if (val != 0 && val != 1)
8189 mutex_lock(&event_mutex);
8190 mutex_lock(&trace_types_lock);
8191 ret = set_tracer_flag(tr, 1 << index, val);
8192 mutex_unlock(&trace_types_lock);
8193 mutex_unlock(&event_mutex);
8203 static const struct file_operations trace_options_core_fops = {
8204 .open = tracing_open_generic,
8205 .read = trace_options_core_read,
8206 .write = trace_options_core_write,
8207 .llseek = generic_file_llseek,
8210 struct dentry *trace_create_file(const char *name,
8212 struct dentry *parent,
8214 const struct file_operations *fops)
8218 ret = tracefs_create_file(name, mode, parent, data, fops);
8220 pr_warn("Could not create tracefs '%s' entry\n", name);
8226 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8228 struct dentry *d_tracer;
8233 d_tracer = tracing_get_dentry(tr);
8234 if (IS_ERR(d_tracer))
8237 tr->options = tracefs_create_dir("options", d_tracer);
8239 pr_warn("Could not create tracefs directory 'options'\n");
8247 create_trace_option_file(struct trace_array *tr,
8248 struct trace_option_dentry *topt,
8249 struct tracer_flags *flags,
8250 struct tracer_opt *opt)
8252 struct dentry *t_options;
8254 t_options = trace_options_init_dentry(tr);
8258 topt->flags = flags;
8262 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
8263 &trace_options_fops);
8268 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8270 struct trace_option_dentry *topts;
8271 struct trace_options *tr_topts;
8272 struct tracer_flags *flags;
8273 struct tracer_opt *opts;
8280 flags = tracer->flags;
8282 if (!flags || !flags->opts)
8286 * If this is an instance, only create flags for tracers
8287 * the instance may have.
8289 if (!trace_ok_for_array(tracer, tr))
8292 for (i = 0; i < tr->nr_topts; i++) {
8293 /* Make sure there's no duplicate flags. */
8294 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8300 for (cnt = 0; opts[cnt].name; cnt++)
8303 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8307 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8314 tr->topts = tr_topts;
8315 tr->topts[tr->nr_topts].tracer = tracer;
8316 tr->topts[tr->nr_topts].topts = topts;
8319 for (cnt = 0; opts[cnt].name; cnt++) {
8320 create_trace_option_file(tr, &topts[cnt], flags,
8322 MEM_FAIL(topts[cnt].entry == NULL,
8323 "Failed to create trace option: %s",
8328 static struct dentry *
8329 create_trace_option_core_file(struct trace_array *tr,
8330 const char *option, long index)
8332 struct dentry *t_options;
8334 t_options = trace_options_init_dentry(tr);
8338 return trace_create_file(option, 0644, t_options,
8339 (void *)&tr->trace_flags_index[index],
8340 &trace_options_core_fops);
8343 static void create_trace_options_dir(struct trace_array *tr)
8345 struct dentry *t_options;
8346 bool top_level = tr == &global_trace;
8349 t_options = trace_options_init_dentry(tr);
8353 for (i = 0; trace_options[i]; i++) {
8355 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8356 create_trace_option_core_file(tr, trace_options[i], i);
8361 rb_simple_read(struct file *filp, char __user *ubuf,
8362 size_t cnt, loff_t *ppos)
8364 struct trace_array *tr = filp->private_data;
8368 r = tracer_tracing_is_on(tr);
8369 r = sprintf(buf, "%d\n", r);
8371 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8375 rb_simple_write(struct file *filp, const char __user *ubuf,
8376 size_t cnt, loff_t *ppos)
8378 struct trace_array *tr = filp->private_data;
8379 struct trace_buffer *buffer = tr->array_buffer.buffer;
8383 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8388 mutex_lock(&trace_types_lock);
8389 if (!!val == tracer_tracing_is_on(tr)) {
8390 val = 0; /* do nothing */
8392 tracer_tracing_on(tr);
8393 if (tr->current_trace->start)
8394 tr->current_trace->start(tr);
8396 tracer_tracing_off(tr);
8397 if (tr->current_trace->stop)
8398 tr->current_trace->stop(tr);
8400 mutex_unlock(&trace_types_lock);
8408 static const struct file_operations rb_simple_fops = {
8409 .open = tracing_open_generic_tr,
8410 .read = rb_simple_read,
8411 .write = rb_simple_write,
8412 .release = tracing_release_generic_tr,
8413 .llseek = default_llseek,
8417 buffer_percent_read(struct file *filp, char __user *ubuf,
8418 size_t cnt, loff_t *ppos)
8420 struct trace_array *tr = filp->private_data;
8424 r = tr->buffer_percent;
8425 r = sprintf(buf, "%d\n", r);
8427 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8431 buffer_percent_write(struct file *filp, const char __user *ubuf,
8432 size_t cnt, loff_t *ppos)
8434 struct trace_array *tr = filp->private_data;
8438 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8448 tr->buffer_percent = val;
8455 static const struct file_operations buffer_percent_fops = {
8456 .open = tracing_open_generic_tr,
8457 .read = buffer_percent_read,
8458 .write = buffer_percent_write,
8459 .release = tracing_release_generic_tr,
8460 .llseek = default_llseek,
8463 static struct dentry *trace_instance_dir;
8466 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8469 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
8471 enum ring_buffer_flags rb_flags;
8473 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
8477 buf->buffer = ring_buffer_alloc(size, rb_flags);
8481 buf->data = alloc_percpu(struct trace_array_cpu);
8483 ring_buffer_free(buf->buffer);
8488 /* Allocate the first page for all buffers */
8489 set_buffer_entries(&tr->array_buffer,
8490 ring_buffer_size(tr->array_buffer.buffer, 0));
8495 static int allocate_trace_buffers(struct trace_array *tr, int size)
8499 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
8503 #ifdef CONFIG_TRACER_MAX_TRACE
8504 ret = allocate_trace_buffer(tr, &tr->max_buffer,
8505 allocate_snapshot ? size : 1);
8506 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
8507 ring_buffer_free(tr->array_buffer.buffer);
8508 tr->array_buffer.buffer = NULL;
8509 free_percpu(tr->array_buffer.data);
8510 tr->array_buffer.data = NULL;
8513 tr->allocated_snapshot = allocate_snapshot;
8516 * Only the top level trace array gets its snapshot allocated
8517 * from the kernel command line.
8519 allocate_snapshot = false;
8524 static void free_trace_buffer(struct array_buffer *buf)
8527 ring_buffer_free(buf->buffer);
8529 free_percpu(buf->data);
8534 static void free_trace_buffers(struct trace_array *tr)
8539 free_trace_buffer(&tr->array_buffer);
8541 #ifdef CONFIG_TRACER_MAX_TRACE
8542 free_trace_buffer(&tr->max_buffer);
8546 static void init_trace_flags_index(struct trace_array *tr)
8550 /* Used by the trace options files */
8551 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8552 tr->trace_flags_index[i] = i;
8555 static void __update_tracer_options(struct trace_array *tr)
8559 for (t = trace_types; t; t = t->next)
8560 add_tracer_options(tr, t);
8563 static void update_tracer_options(struct trace_array *tr)
8565 mutex_lock(&trace_types_lock);
8566 __update_tracer_options(tr);
8567 mutex_unlock(&trace_types_lock);
8570 /* Must have trace_types_lock held */
8571 struct trace_array *trace_array_find(const char *instance)
8573 struct trace_array *tr, *found = NULL;
8575 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8576 if (tr->name && strcmp(tr->name, instance) == 0) {
8585 struct trace_array *trace_array_find_get(const char *instance)
8587 struct trace_array *tr;
8589 mutex_lock(&trace_types_lock);
8590 tr = trace_array_find(instance);
8593 mutex_unlock(&trace_types_lock);
8598 static struct trace_array *trace_array_create(const char *name)
8600 struct trace_array *tr;
8604 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8606 return ERR_PTR(ret);
8608 tr->name = kstrdup(name, GFP_KERNEL);
8612 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8615 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8617 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8619 raw_spin_lock_init(&tr->start_lock);
8621 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8623 tr->current_trace = &nop_trace;
8625 INIT_LIST_HEAD(&tr->systems);
8626 INIT_LIST_HEAD(&tr->events);
8627 INIT_LIST_HEAD(&tr->hist_vars);
8628 INIT_LIST_HEAD(&tr->err_log);
8630 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8633 tr->dir = tracefs_create_dir(name, trace_instance_dir);
8637 ret = event_trace_add_tracer(tr->dir, tr);
8639 tracefs_remove(tr->dir);
8643 ftrace_init_trace_array(tr);
8645 init_tracer_tracefs(tr, tr->dir);
8646 init_trace_flags_index(tr);
8647 __update_tracer_options(tr);
8649 list_add(&tr->list, &ftrace_trace_arrays);
8657 free_trace_buffers(tr);
8658 free_cpumask_var(tr->tracing_cpumask);
8662 return ERR_PTR(ret);
8665 static int instance_mkdir(const char *name)
8667 struct trace_array *tr;
8670 mutex_lock(&event_mutex);
8671 mutex_lock(&trace_types_lock);
8674 if (trace_array_find(name))
8677 tr = trace_array_create(name);
8679 ret = PTR_ERR_OR_ZERO(tr);
8682 mutex_unlock(&trace_types_lock);
8683 mutex_unlock(&event_mutex);
8688 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
8689 * @name: The name of the trace array to be looked up/created.
8691 * Returns pointer to trace array with given name.
8692 * NULL, if it cannot be created.
8694 * NOTE: This function increments the reference counter associated with the
8695 * trace array returned. This makes sure it cannot be freed while in use.
8696 * Use trace_array_put() once the trace array is no longer needed.
8697 * If the trace_array is to be freed, trace_array_destroy() needs to
8698 * be called after the trace_array_put(), or simply let user space delete
8699 * it from the tracefs instances directory. But until the
8700 * trace_array_put() is called, user space can not delete it.
8703 struct trace_array *trace_array_get_by_name(const char *name)
8705 struct trace_array *tr;
8707 mutex_lock(&event_mutex);
8708 mutex_lock(&trace_types_lock);
8710 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8711 if (tr->name && strcmp(tr->name, name) == 0)
8715 tr = trace_array_create(name);
8723 mutex_unlock(&trace_types_lock);
8724 mutex_unlock(&event_mutex);
8727 EXPORT_SYMBOL_GPL(trace_array_get_by_name);
8729 static int __remove_instance(struct trace_array *tr)
8733 /* Reference counter for a newly created trace array = 1. */
8734 if (tr->ref > 1 || (tr->current_trace && tr->current_trace->ref))
8737 list_del(&tr->list);
8739 /* Disable all the flags that were enabled coming in */
8740 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8741 if ((1 << i) & ZEROED_TRACE_FLAGS)
8742 set_tracer_flag(tr, 1 << i, 0);
8745 tracing_set_nop(tr);
8746 clear_ftrace_function_probes(tr);
8747 event_trace_del_tracer(tr);
8748 ftrace_clear_pids(tr);
8749 ftrace_destroy_function_files(tr);
8750 tracefs_remove(tr->dir);
8751 free_trace_buffers(tr);
8753 for (i = 0; i < tr->nr_topts; i++) {
8754 kfree(tr->topts[i].topts);
8758 free_cpumask_var(tr->tracing_cpumask);
8766 int trace_array_destroy(struct trace_array *this_tr)
8768 struct trace_array *tr;
8774 mutex_lock(&event_mutex);
8775 mutex_lock(&trace_types_lock);
8779 /* Making sure trace array exists before destroying it. */
8780 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8781 if (tr == this_tr) {
8782 ret = __remove_instance(tr);
8787 mutex_unlock(&trace_types_lock);
8788 mutex_unlock(&event_mutex);
8792 EXPORT_SYMBOL_GPL(trace_array_destroy);
8794 static int instance_rmdir(const char *name)
8796 struct trace_array *tr;
8799 mutex_lock(&event_mutex);
8800 mutex_lock(&trace_types_lock);
8803 tr = trace_array_find(name);
8805 ret = __remove_instance(tr);
8807 mutex_unlock(&trace_types_lock);
8808 mutex_unlock(&event_mutex);
8813 static __init void create_trace_instances(struct dentry *d_tracer)
8815 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8818 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
8823 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8825 struct trace_event_file *file;
8828 trace_create_file("available_tracers", 0444, d_tracer,
8829 tr, &show_traces_fops);
8831 trace_create_file("current_tracer", 0644, d_tracer,
8832 tr, &set_tracer_fops);
8834 trace_create_file("tracing_cpumask", 0644, d_tracer,
8835 tr, &tracing_cpumask_fops);
8837 trace_create_file("trace_options", 0644, d_tracer,
8838 tr, &tracing_iter_fops);
8840 trace_create_file("trace", 0644, d_tracer,
8843 trace_create_file("trace_pipe", 0444, d_tracer,
8844 tr, &tracing_pipe_fops);
8846 trace_create_file("buffer_size_kb", 0644, d_tracer,
8847 tr, &tracing_entries_fops);
8849 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
8850 tr, &tracing_total_entries_fops);
8852 trace_create_file("free_buffer", 0200, d_tracer,
8853 tr, &tracing_free_buffer_fops);
8855 trace_create_file("trace_marker", 0220, d_tracer,
8856 tr, &tracing_mark_fops);
8858 file = __find_event_file(tr, "ftrace", "print");
8859 if (file && file->dir)
8860 trace_create_file("trigger", 0644, file->dir, file,
8861 &event_trigger_fops);
8862 tr->trace_marker_file = file;
8864 trace_create_file("trace_marker_raw", 0220, d_tracer,
8865 tr, &tracing_mark_raw_fops);
8867 trace_create_file("trace_clock", 0644, d_tracer, tr,
8870 trace_create_file("tracing_on", 0644, d_tracer,
8871 tr, &rb_simple_fops);
8873 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8874 &trace_time_stamp_mode_fops);
8876 tr->buffer_percent = 50;
8878 trace_create_file("buffer_percent", 0444, d_tracer,
8879 tr, &buffer_percent_fops);
8881 create_trace_options_dir(tr);
8883 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
8884 trace_create_maxlat_file(tr, d_tracer);
8887 if (ftrace_create_function_files(tr, d_tracer))
8888 MEM_FAIL(1, "Could not allocate function filter files");
8890 #ifdef CONFIG_TRACER_SNAPSHOT
8891 trace_create_file("snapshot", 0644, d_tracer,
8892 tr, &snapshot_fops);
8895 trace_create_file("error_log", 0644, d_tracer,
8896 tr, &tracing_err_log_fops);
8898 for_each_tracing_cpu(cpu)
8899 tracing_init_tracefs_percpu(tr, cpu);
8901 ftrace_init_tracefs(tr, d_tracer);
8904 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
8906 struct vfsmount *mnt;
8907 struct file_system_type *type;
8910 * To maintain backward compatibility for tools that mount
8911 * debugfs to get to the tracing facility, tracefs is automatically
8912 * mounted to the debugfs/tracing directory.
8914 type = get_fs_type("tracefs");
8917 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
8918 put_filesystem(type);
8927 * tracing_init_dentry - initialize top level trace array
8929 * This is called when creating files or directories in the tracing
8930 * directory. It is called via fs_initcall() by any of the boot up code
8931 * and expects to return the dentry of the top level tracing directory.
8933 struct dentry *tracing_init_dentry(void)
8935 struct trace_array *tr = &global_trace;
8937 if (security_locked_down(LOCKDOWN_TRACEFS)) {
8938 pr_warn("Tracing disabled due to lockdown\n");
8939 return ERR_PTR(-EPERM);
8942 /* The top level trace array uses NULL as parent */
8946 if (WARN_ON(!tracefs_initialized()) ||
8947 (IS_ENABLED(CONFIG_DEBUG_FS) &&
8948 WARN_ON(!debugfs_initialized())))
8949 return ERR_PTR(-ENODEV);
8952 * As there may still be users that expect the tracing
8953 * files to exist in debugfs/tracing, we must automount
8954 * the tracefs file system there, so older tools still
8955 * work with the newer kerenl.
8957 tr->dir = debugfs_create_automount("tracing", NULL,
8958 trace_automount, NULL);
8963 extern struct trace_eval_map *__start_ftrace_eval_maps[];
8964 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8966 static void __init trace_eval_init(void)
8970 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8971 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8974 #ifdef CONFIG_MODULES
8975 static void trace_module_add_evals(struct module *mod)
8977 if (!mod->num_trace_evals)
8981 * Modules with bad taint do not have events created, do
8982 * not bother with enums either.
8984 if (trace_module_has_bad_taint(mod))
8987 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
8990 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
8991 static void trace_module_remove_evals(struct module *mod)
8993 union trace_eval_map_item *map;
8994 union trace_eval_map_item **last = &trace_eval_maps;
8996 if (!mod->num_trace_evals)
8999 mutex_lock(&trace_eval_mutex);
9001 map = trace_eval_maps;
9004 if (map->head.mod == mod)
9006 map = trace_eval_jmp_to_tail(map);
9007 last = &map->tail.next;
9008 map = map->tail.next;
9013 *last = trace_eval_jmp_to_tail(map)->tail.next;
9016 mutex_unlock(&trace_eval_mutex);
9019 static inline void trace_module_remove_evals(struct module *mod) { }
9020 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9022 static int trace_module_notify(struct notifier_block *self,
9023 unsigned long val, void *data)
9025 struct module *mod = data;
9028 case MODULE_STATE_COMING:
9029 trace_module_add_evals(mod);
9031 case MODULE_STATE_GOING:
9032 trace_module_remove_evals(mod);
9039 static struct notifier_block trace_module_nb = {
9040 .notifier_call = trace_module_notify,
9043 #endif /* CONFIG_MODULES */
9045 static __init int tracer_init_tracefs(void)
9047 struct dentry *d_tracer;
9049 trace_access_lock_init();
9051 d_tracer = tracing_init_dentry();
9052 if (IS_ERR(d_tracer))
9057 init_tracer_tracefs(&global_trace, d_tracer);
9058 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
9060 trace_create_file("tracing_thresh", 0644, d_tracer,
9061 &global_trace, &tracing_thresh_fops);
9063 trace_create_file("README", 0444, d_tracer,
9064 NULL, &tracing_readme_fops);
9066 trace_create_file("saved_cmdlines", 0444, d_tracer,
9067 NULL, &tracing_saved_cmdlines_fops);
9069 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
9070 NULL, &tracing_saved_cmdlines_size_fops);
9072 trace_create_file("saved_tgids", 0444, d_tracer,
9073 NULL, &tracing_saved_tgids_fops);
9077 trace_create_eval_file(d_tracer);
9079 #ifdef CONFIG_MODULES
9080 register_module_notifier(&trace_module_nb);
9083 #ifdef CONFIG_DYNAMIC_FTRACE
9084 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
9085 NULL, &tracing_dyn_info_fops);
9088 create_trace_instances(d_tracer);
9090 update_tracer_options(&global_trace);
9095 static int trace_panic_handler(struct notifier_block *this,
9096 unsigned long event, void *unused)
9098 if (ftrace_dump_on_oops)
9099 ftrace_dump(ftrace_dump_on_oops);
9103 static struct notifier_block trace_panic_notifier = {
9104 .notifier_call = trace_panic_handler,
9106 .priority = 150 /* priority: INT_MAX >= x >= 0 */
9109 static int trace_die_handler(struct notifier_block *self,
9115 if (ftrace_dump_on_oops)
9116 ftrace_dump(ftrace_dump_on_oops);
9124 static struct notifier_block trace_die_notifier = {
9125 .notifier_call = trace_die_handler,
9130 * printk is set to max of 1024, we really don't need it that big.
9131 * Nothing should be printing 1000 characters anyway.
9133 #define TRACE_MAX_PRINT 1000
9136 * Define here KERN_TRACE so that we have one place to modify
9137 * it if we decide to change what log level the ftrace dump
9140 #define KERN_TRACE KERN_EMERG
9143 trace_printk_seq(struct trace_seq *s)
9145 /* Probably should print a warning here. */
9146 if (s->seq.len >= TRACE_MAX_PRINT)
9147 s->seq.len = TRACE_MAX_PRINT;
9150 * More paranoid code. Although the buffer size is set to
9151 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9152 * an extra layer of protection.
9154 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9155 s->seq.len = s->seq.size - 1;
9157 /* should be zero ended, but we are paranoid. */
9158 s->buffer[s->seq.len] = 0;
9160 printk(KERN_TRACE "%s", s->buffer);
9165 void trace_init_global_iter(struct trace_iterator *iter)
9167 iter->tr = &global_trace;
9168 iter->trace = iter->tr->current_trace;
9169 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9170 iter->array_buffer = &global_trace.array_buffer;
9172 if (iter->trace && iter->trace->open)
9173 iter->trace->open(iter);
9175 /* Annotate start of buffers if we had overruns */
9176 if (ring_buffer_overruns(iter->array_buffer->buffer))
9177 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9179 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9180 if (trace_clocks[iter->tr->clock_id].in_ns)
9181 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9184 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9186 /* use static because iter can be a bit big for the stack */
9187 static struct trace_iterator iter;
9188 static atomic_t dump_running;
9189 struct trace_array *tr = &global_trace;
9190 unsigned int old_userobj;
9191 unsigned long flags;
9194 /* Only allow one dump user at a time. */
9195 if (atomic_inc_return(&dump_running) != 1) {
9196 atomic_dec(&dump_running);
9201 * Always turn off tracing when we dump.
9202 * We don't need to show trace output of what happens
9203 * between multiple crashes.
9205 * If the user does a sysrq-z, then they can re-enable
9206 * tracing with echo 1 > tracing_on.
9210 local_irq_save(flags);
9211 printk_nmi_direct_enter();
9213 /* Simulate the iterator */
9214 trace_init_global_iter(&iter);
9215 /* Can not use kmalloc for iter.temp */
9216 iter.temp = static_temp_buf;
9217 iter.temp_size = STATIC_TEMP_BUF_SIZE;
9219 for_each_tracing_cpu(cpu) {
9220 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9223 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9225 /* don't look at user memory in panic mode */
9226 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9228 switch (oops_dump_mode) {
9230 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9233 iter.cpu_file = raw_smp_processor_id();
9238 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9239 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9242 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9244 /* Did function tracer already get disabled? */
9245 if (ftrace_is_dead()) {
9246 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9247 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9251 * We need to stop all tracing on all CPUS to read the
9252 * the next buffer. This is a bit expensive, but is
9253 * not done often. We fill all what we can read,
9254 * and then release the locks again.
9257 while (!trace_empty(&iter)) {
9260 printk(KERN_TRACE "---------------------------------\n");
9264 trace_iterator_reset(&iter);
9265 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9267 if (trace_find_next_entry_inc(&iter) != NULL) {
9270 ret = print_trace_line(&iter);
9271 if (ret != TRACE_TYPE_NO_CONSUME)
9272 trace_consume(&iter);
9274 touch_nmi_watchdog();
9276 trace_printk_seq(&iter.seq);
9280 printk(KERN_TRACE " (ftrace buffer empty)\n");
9282 printk(KERN_TRACE "---------------------------------\n");
9285 tr->trace_flags |= old_userobj;
9287 for_each_tracing_cpu(cpu) {
9288 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9290 atomic_dec(&dump_running);
9291 printk_nmi_direct_exit();
9292 local_irq_restore(flags);
9294 EXPORT_SYMBOL_GPL(ftrace_dump);
9296 int trace_run_command(const char *buf, int (*createfn)(int, char **))
9303 argv = argv_split(GFP_KERNEL, buf, &argc);
9308 ret = createfn(argc, argv);
9315 #define WRITE_BUFSIZE 4096
9317 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9318 size_t count, loff_t *ppos,
9319 int (*createfn)(int, char **))
9321 char *kbuf, *buf, *tmp;
9326 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9330 while (done < count) {
9331 size = count - done;
9333 if (size >= WRITE_BUFSIZE)
9334 size = WRITE_BUFSIZE - 1;
9336 if (copy_from_user(kbuf, buffer + done, size)) {
9343 tmp = strchr(buf, '\n');
9346 size = tmp - buf + 1;
9349 if (done + size < count) {
9352 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9353 pr_warn("Line length is too long: Should be less than %d\n",
9361 /* Remove comments */
9362 tmp = strchr(buf, '#');
9367 ret = trace_run_command(buf, createfn);
9372 } while (done < count);
9382 __init static int tracer_alloc_buffers(void)
9388 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9389 pr_warn("Tracing disabled due to lockdown\n");
9394 * Make sure we don't accidently add more trace options
9395 * than we have bits for.
9397 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9399 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9402 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9403 goto out_free_buffer_mask;
9405 /* Only allocate trace_printk buffers if a trace_printk exists */
9406 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
9407 /* Must be called before global_trace.buffer is allocated */
9408 trace_printk_init_buffers();
9410 /* To save memory, keep the ring buffer size to its minimum */
9411 if (ring_buffer_expanded)
9412 ring_buf_size = trace_buf_size;
9416 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9417 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9419 raw_spin_lock_init(&global_trace.start_lock);
9422 * The prepare callbacks allocates some memory for the ring buffer. We
9423 * don't free the buffer if the if the CPU goes down. If we were to free
9424 * the buffer, then the user would lose any trace that was in the
9425 * buffer. The memory will be removed once the "instance" is removed.
9427 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9428 "trace/RB:preapre", trace_rb_cpu_prepare,
9431 goto out_free_cpumask;
9432 /* Used for event triggers */
9434 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9436 goto out_rm_hp_state;
9438 if (trace_create_savedcmd() < 0)
9439 goto out_free_temp_buffer;
9441 /* TODO: make the number of buffers hot pluggable with CPUS */
9442 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9443 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
9444 goto out_free_savedcmd;
9447 if (global_trace.buffer_disabled)
9450 if (trace_boot_clock) {
9451 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9453 pr_warn("Trace clock %s not defined, going back to default\n",
9458 * register_tracer() might reference current_trace, so it
9459 * needs to be set before we register anything. This is
9460 * just a bootstrap of current_trace anyway.
9462 global_trace.current_trace = &nop_trace;
9464 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9466 ftrace_init_global_array_ops(&global_trace);
9468 init_trace_flags_index(&global_trace);
9470 register_tracer(&nop_trace);
9472 /* Function tracing may start here (via kernel command line) */
9473 init_function_trace();
9475 /* All seems OK, enable tracing */
9476 tracing_disabled = 0;
9478 atomic_notifier_chain_register(&panic_notifier_list,
9479 &trace_panic_notifier);
9481 register_die_notifier(&trace_die_notifier);
9483 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9485 INIT_LIST_HEAD(&global_trace.systems);
9486 INIT_LIST_HEAD(&global_trace.events);
9487 INIT_LIST_HEAD(&global_trace.hist_vars);
9488 INIT_LIST_HEAD(&global_trace.err_log);
9489 list_add(&global_trace.list, &ftrace_trace_arrays);
9491 apply_trace_boot_options();
9493 register_snapshot_cmd();
9498 free_saved_cmdlines_buffer(savedcmd);
9499 out_free_temp_buffer:
9500 ring_buffer_free(temp_buffer);
9502 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9504 free_cpumask_var(global_trace.tracing_cpumask);
9505 out_free_buffer_mask:
9506 free_cpumask_var(tracing_buffer_mask);
9511 void __init early_trace_init(void)
9513 if (tracepoint_printk) {
9514 tracepoint_print_iter =
9515 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
9516 if (MEM_FAIL(!tracepoint_print_iter,
9517 "Failed to allocate trace iterator\n"))
9518 tracepoint_printk = 0;
9520 static_key_enable(&tracepoint_printk_key.key);
9522 tracer_alloc_buffers();
9525 void __init trace_init(void)
9530 __init static int clear_boot_tracer(void)
9533 * The default tracer at boot buffer is an init section.
9534 * This function is called in lateinit. If we did not
9535 * find the boot tracer, then clear it out, to prevent
9536 * later registration from accessing the buffer that is
9537 * about to be freed.
9539 if (!default_bootup_tracer)
9542 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
9543 default_bootup_tracer);
9544 default_bootup_tracer = NULL;
9549 fs_initcall(tracer_init_tracefs);
9550 late_initcall_sync(clear_boot_tracer);
9552 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9553 __init static int tracing_set_default_clock(void)
9555 /* sched_clock_stable() is determined in late_initcall */
9556 if (!trace_boot_clock && !sched_clock_stable()) {
9557 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9558 pr_warn("Can not set tracing clock due to lockdown\n");
9563 "Unstable clock detected, switching default tracing clock to \"global\"\n"
9564 "If you want to keep using the local clock, then add:\n"
9565 " \"trace_clock=local\"\n"
9566 "on the kernel command line\n");
9567 tracing_set_clock(&global_trace, "global");
9572 late_initcall_sync(tracing_set_default_clock);