1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer based function tracer
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/security.h>
21 #include <linux/seq_file.h>
22 #include <linux/notifier.h>
23 #include <linux/irqflags.h>
24 #include <linux/debugfs.h>
25 #include <linux/tracefs.h>
26 #include <linux/pagemap.h>
27 #include <linux/hardirq.h>
28 #include <linux/linkage.h>
29 #include <linux/uaccess.h>
30 #include <linux/vmalloc.h>
31 #include <linux/ftrace.h>
32 #include <linux/module.h>
33 #include <linux/percpu.h>
34 #include <linux/splice.h>
35 #include <linux/kdebug.h>
36 #include <linux/string.h>
37 #include <linux/mount.h>
38 #include <linux/rwsem.h>
39 #include <linux/slab.h>
40 #include <linux/ctype.h>
41 #include <linux/init.h>
42 #include <linux/poll.h>
43 #include <linux/nmi.h>
45 #include <linux/trace.h>
46 #include <linux/sched/clock.h>
47 #include <linux/sched/rt.h>
48 #include <linux/fsnotify.h>
49 #include <linux/irq_work.h>
50 #include <linux/workqueue.h>
53 #include "trace_output.h"
56 * On boot up, the ring buffer is set to the minimum size, so that
57 * we do not waste memory on systems that are not using tracing.
59 bool ring_buffer_expanded;
62 * We need to change this state when a selftest is running.
63 * A selftest will lurk into the ring-buffer to count the
64 * entries inserted during the selftest although some concurrent
65 * insertions into the ring-buffer such as trace_printk could occurred
66 * at the same time, giving false positive or negative results.
68 static bool __read_mostly tracing_selftest_running;
71 * If a tracer is running, we do not want to run SELFTEST.
73 bool __read_mostly tracing_selftest_disabled;
75 /* Pipe tracepoints to printk */
76 struct trace_iterator *tracepoint_print_iter;
77 int tracepoint_printk;
78 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
80 /* For tracers that don't implement custom flags */
81 static struct tracer_opt dummy_tracer_opt[] = {
86 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
92 * To prevent the comm cache from being overwritten when no
93 * tracing is active, only save the comm when a trace event
96 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
99 * Kill all tracing for good (never come back).
100 * It is initialized to 1 but will turn to zero if the initialization
101 * of the tracer is successful. But that is the only place that sets
104 static int tracing_disabled = 1;
106 cpumask_var_t __read_mostly tracing_buffer_mask;
109 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
112 * is set, then ftrace_dump is called. This will output the contents
113 * of the ftrace buffers to the console. This is very useful for
114 * capturing traces that lead to crashes and outputing it to a
117 * It is default off, but you can enable it with either specifying
118 * "ftrace_dump_on_oops" in the kernel command line, or setting
119 * /proc/sys/kernel/ftrace_dump_on_oops
120 * Set 1 if you want to dump buffers of all CPUs
121 * Set 2 if you want to dump the buffer of the CPU that triggered oops
124 enum ftrace_dump_mode ftrace_dump_on_oops;
126 /* When set, tracing will stop when a WARN*() is hit */
127 int __disable_trace_on_warning;
129 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
130 /* Map of enums to their values, for "eval_map" file */
131 struct trace_eval_map_head {
133 unsigned long length;
136 union trace_eval_map_item;
138 struct trace_eval_map_tail {
140 * "end" is first and points to NULL as it must be different
141 * than "mod" or "eval_string"
143 union trace_eval_map_item *next;
144 const char *end; /* points to NULL */
147 static DEFINE_MUTEX(trace_eval_mutex);
150 * The trace_eval_maps are saved in an array with two extra elements,
151 * one at the beginning, and one at the end. The beginning item contains
152 * the count of the saved maps (head.length), and the module they
153 * belong to if not built in (head.mod). The ending item contains a
154 * pointer to the next array of saved eval_map items.
156 union trace_eval_map_item {
157 struct trace_eval_map map;
158 struct trace_eval_map_head head;
159 struct trace_eval_map_tail tail;
162 static union trace_eval_map_item *trace_eval_maps;
163 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
165 int tracing_set_tracer(struct trace_array *tr, const char *buf);
166 static void ftrace_trace_userstack(struct trace_buffer *buffer,
167 unsigned long flags, int pc);
169 #define MAX_TRACER_SIZE 100
170 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
171 static char *default_bootup_tracer;
173 static bool allocate_snapshot;
175 static int __init set_cmdline_ftrace(char *str)
177 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
178 default_bootup_tracer = bootup_tracer_buf;
179 /* We are using ftrace early, expand it */
180 ring_buffer_expanded = true;
183 __setup("ftrace=", set_cmdline_ftrace);
185 static int __init set_ftrace_dump_on_oops(char *str)
187 if (*str++ != '=' || !*str) {
188 ftrace_dump_on_oops = DUMP_ALL;
192 if (!strcmp("orig_cpu", str)) {
193 ftrace_dump_on_oops = DUMP_ORIG;
199 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
201 static int __init stop_trace_on_warning(char *str)
203 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
204 __disable_trace_on_warning = 1;
207 __setup("traceoff_on_warning", stop_trace_on_warning);
209 static int __init boot_alloc_snapshot(char *str)
211 allocate_snapshot = true;
212 /* We also need the main ring buffer expanded */
213 ring_buffer_expanded = true;
216 __setup("alloc_snapshot", boot_alloc_snapshot);
219 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
221 static int __init set_trace_boot_options(char *str)
223 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
226 __setup("trace_options=", set_trace_boot_options);
228 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
229 static char *trace_boot_clock __initdata;
231 static int __init set_trace_boot_clock(char *str)
233 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
234 trace_boot_clock = trace_boot_clock_buf;
237 __setup("trace_clock=", set_trace_boot_clock);
239 static int __init set_tracepoint_printk(char *str)
241 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
242 tracepoint_printk = 1;
245 __setup("tp_printk", set_tracepoint_printk);
247 unsigned long long ns2usecs(u64 nsec)
254 /* trace_flags holds trace_options default values */
255 #define TRACE_DEFAULT_FLAGS \
256 (FUNCTION_DEFAULT_FLAGS | \
257 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
258 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
259 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
260 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
262 /* trace_options that are only supported by global_trace */
263 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
264 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
266 /* trace_flags that are default zero for instances */
267 #define ZEROED_TRACE_FLAGS \
268 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
271 * The global_trace is the descriptor that holds the top-level tracing
272 * buffers for the live tracing.
274 static struct trace_array global_trace = {
275 .trace_flags = TRACE_DEFAULT_FLAGS,
278 LIST_HEAD(ftrace_trace_arrays);
280 int trace_array_get(struct trace_array *this_tr)
282 struct trace_array *tr;
285 mutex_lock(&trace_types_lock);
286 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
293 mutex_unlock(&trace_types_lock);
298 static void __trace_array_put(struct trace_array *this_tr)
300 WARN_ON(!this_tr->ref);
305 * trace_array_put - Decrement the reference counter for this trace array.
307 * NOTE: Use this when we no longer need the trace array returned by
308 * trace_array_get_by_name(). This ensures the trace array can be later
312 void trace_array_put(struct trace_array *this_tr)
317 mutex_lock(&trace_types_lock);
318 __trace_array_put(this_tr);
319 mutex_unlock(&trace_types_lock);
321 EXPORT_SYMBOL_GPL(trace_array_put);
323 int tracing_check_open_get_tr(struct trace_array *tr)
327 ret = security_locked_down(LOCKDOWN_TRACEFS);
331 if (tracing_disabled)
334 if (tr && trace_array_get(tr) < 0)
340 int call_filter_check_discard(struct trace_event_call *call, void *rec,
341 struct trace_buffer *buffer,
342 struct ring_buffer_event *event)
344 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
345 !filter_match_preds(call->filter, rec)) {
346 __trace_event_discard_commit(buffer, event);
353 void trace_free_pid_list(struct trace_pid_list *pid_list)
355 vfree(pid_list->pids);
360 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
361 * @filtered_pids: The list of pids to check
362 * @search_pid: The PID to find in @filtered_pids
364 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
367 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
370 * If pid_max changed after filtered_pids was created, we
371 * by default ignore all pids greater than the previous pid_max.
373 if (search_pid >= filtered_pids->pid_max)
376 return test_bit(search_pid, filtered_pids->pids);
380 * trace_ignore_this_task - should a task be ignored for tracing
381 * @filtered_pids: The list of pids to check
382 * @task: The task that should be ignored if not filtered
384 * Checks if @task should be traced or not from @filtered_pids.
385 * Returns true if @task should *NOT* be traced.
386 * Returns false if @task should be traced.
389 trace_ignore_this_task(struct trace_pid_list *filtered_pids,
390 struct trace_pid_list *filtered_no_pids,
391 struct task_struct *task)
394 * If filterd_no_pids is not empty, and the task's pid is listed
395 * in filtered_no_pids, then return true.
396 * Otherwise, if filtered_pids is empty, that means we can
397 * trace all tasks. If it has content, then only trace pids
398 * within filtered_pids.
401 return (filtered_pids &&
402 !trace_find_filtered_pid(filtered_pids, task->pid)) ||
404 trace_find_filtered_pid(filtered_no_pids, task->pid));
408 * trace_filter_add_remove_task - Add or remove a task from a pid_list
409 * @pid_list: The list to modify
410 * @self: The current task for fork or NULL for exit
411 * @task: The task to add or remove
413 * If adding a task, if @self is defined, the task is only added if @self
414 * is also included in @pid_list. This happens on fork and tasks should
415 * only be added when the parent is listed. If @self is NULL, then the
416 * @task pid will be removed from the list, which would happen on exit
419 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
420 struct task_struct *self,
421 struct task_struct *task)
426 /* For forks, we only add if the forking task is listed */
428 if (!trace_find_filtered_pid(pid_list, self->pid))
432 /* Sorry, but we don't support pid_max changing after setting */
433 if (task->pid >= pid_list->pid_max)
436 /* "self" is set for forks, and NULL for exits */
438 set_bit(task->pid, pid_list->pids);
440 clear_bit(task->pid, pid_list->pids);
444 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
445 * @pid_list: The pid list to show
446 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
447 * @pos: The position of the file
449 * This is used by the seq_file "next" operation to iterate the pids
450 * listed in a trace_pid_list structure.
452 * Returns the pid+1 as we want to display pid of zero, but NULL would
453 * stop the iteration.
455 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
457 unsigned long pid = (unsigned long)v;
461 /* pid already is +1 of the actual prevous bit */
462 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
464 /* Return pid + 1 to allow zero to be represented */
465 if (pid < pid_list->pid_max)
466 return (void *)(pid + 1);
472 * trace_pid_start - Used for seq_file to start reading pid lists
473 * @pid_list: The pid list to show
474 * @pos: The position of the file
476 * This is used by seq_file "start" operation to start the iteration
479 * Returns the pid+1 as we want to display pid of zero, but NULL would
480 * stop the iteration.
482 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
487 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
488 if (pid >= pid_list->pid_max)
491 /* Return pid + 1 so that zero can be the exit value */
492 for (pid++; pid && l < *pos;
493 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
499 * trace_pid_show - show the current pid in seq_file processing
500 * @m: The seq_file structure to write into
501 * @v: A void pointer of the pid (+1) value to display
503 * Can be directly used by seq_file operations to display the current
506 int trace_pid_show(struct seq_file *m, void *v)
508 unsigned long pid = (unsigned long)v - 1;
510 seq_printf(m, "%lu\n", pid);
514 /* 128 should be much more than enough */
515 #define PID_BUF_SIZE 127
517 int trace_pid_write(struct trace_pid_list *filtered_pids,
518 struct trace_pid_list **new_pid_list,
519 const char __user *ubuf, size_t cnt)
521 struct trace_pid_list *pid_list;
522 struct trace_parser parser;
530 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
534 * Always recreate a new array. The write is an all or nothing
535 * operation. Always create a new array when adding new pids by
536 * the user. If the operation fails, then the current list is
539 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
541 trace_parser_put(&parser);
545 pid_list->pid_max = READ_ONCE(pid_max);
547 /* Only truncating will shrink pid_max */
548 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
549 pid_list->pid_max = filtered_pids->pid_max;
551 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
552 if (!pid_list->pids) {
553 trace_parser_put(&parser);
559 /* copy the current bits to the new max */
560 for_each_set_bit(pid, filtered_pids->pids,
561 filtered_pids->pid_max) {
562 set_bit(pid, pid_list->pids);
571 ret = trace_get_user(&parser, ubuf, cnt, &pos);
572 if (ret < 0 || !trace_parser_loaded(&parser))
580 if (kstrtoul(parser.buffer, 0, &val))
582 if (val >= pid_list->pid_max)
587 set_bit(pid, pid_list->pids);
590 trace_parser_clear(&parser);
593 trace_parser_put(&parser);
596 trace_free_pid_list(pid_list);
601 /* Cleared the list of pids */
602 trace_free_pid_list(pid_list);
607 *new_pid_list = pid_list;
612 static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
616 /* Early boot up does not have a buffer yet */
618 return trace_clock_local();
620 ts = ring_buffer_time_stamp(buf->buffer, cpu);
621 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
626 u64 ftrace_now(int cpu)
628 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
632 * tracing_is_enabled - Show if global_trace has been disabled
634 * Shows if the global trace has been enabled or not. It uses the
635 * mirror flag "buffer_disabled" to be used in fast paths such as for
636 * the irqsoff tracer. But it may be inaccurate due to races. If you
637 * need to know the accurate state, use tracing_is_on() which is a little
638 * slower, but accurate.
640 int tracing_is_enabled(void)
643 * For quick access (irqsoff uses this in fast path), just
644 * return the mirror variable of the state of the ring buffer.
645 * It's a little racy, but we don't really care.
648 return !global_trace.buffer_disabled;
652 * trace_buf_size is the size in bytes that is allocated
653 * for a buffer. Note, the number of bytes is always rounded
656 * This number is purposely set to a low number of 16384.
657 * If the dump on oops happens, it will be much appreciated
658 * to not have to wait for all that output. Anyway this can be
659 * boot time and run time configurable.
661 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
663 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
665 /* trace_types holds a link list of available tracers. */
666 static struct tracer *trace_types __read_mostly;
669 * trace_types_lock is used to protect the trace_types list.
671 DEFINE_MUTEX(trace_types_lock);
674 * serialize the access of the ring buffer
676 * ring buffer serializes readers, but it is low level protection.
677 * The validity of the events (which returns by ring_buffer_peek() ..etc)
678 * are not protected by ring buffer.
680 * The content of events may become garbage if we allow other process consumes
681 * these events concurrently:
682 * A) the page of the consumed events may become a normal page
683 * (not reader page) in ring buffer, and this page will be rewrited
684 * by events producer.
685 * B) The page of the consumed events may become a page for splice_read,
686 * and this page will be returned to system.
688 * These primitives allow multi process access to different cpu ring buffer
691 * These primitives don't distinguish read-only and read-consume access.
692 * Multi read-only access are also serialized.
696 static DECLARE_RWSEM(all_cpu_access_lock);
697 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
699 static inline void trace_access_lock(int cpu)
701 if (cpu == RING_BUFFER_ALL_CPUS) {
702 /* gain it for accessing the whole ring buffer. */
703 down_write(&all_cpu_access_lock);
705 /* gain it for accessing a cpu ring buffer. */
707 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
708 down_read(&all_cpu_access_lock);
710 /* Secondly block other access to this @cpu ring buffer. */
711 mutex_lock(&per_cpu(cpu_access_lock, cpu));
715 static inline void trace_access_unlock(int cpu)
717 if (cpu == RING_BUFFER_ALL_CPUS) {
718 up_write(&all_cpu_access_lock);
720 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
721 up_read(&all_cpu_access_lock);
725 static inline void trace_access_lock_init(void)
729 for_each_possible_cpu(cpu)
730 mutex_init(&per_cpu(cpu_access_lock, cpu));
735 static DEFINE_MUTEX(access_lock);
737 static inline void trace_access_lock(int cpu)
740 mutex_lock(&access_lock);
743 static inline void trace_access_unlock(int cpu)
746 mutex_unlock(&access_lock);
749 static inline void trace_access_lock_init(void)
755 #ifdef CONFIG_STACKTRACE
756 static void __ftrace_trace_stack(struct trace_buffer *buffer,
758 int skip, int pc, struct pt_regs *regs);
759 static inline void ftrace_trace_stack(struct trace_array *tr,
760 struct trace_buffer *buffer,
762 int skip, int pc, struct pt_regs *regs);
765 static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
767 int skip, int pc, struct pt_regs *regs)
770 static inline void ftrace_trace_stack(struct trace_array *tr,
771 struct trace_buffer *buffer,
773 int skip, int pc, struct pt_regs *regs)
779 static __always_inline void
780 trace_event_setup(struct ring_buffer_event *event,
781 int type, unsigned long flags, int pc)
783 struct trace_entry *ent = ring_buffer_event_data(event);
785 tracing_generic_entry_update(ent, type, flags, pc);
788 static __always_inline struct ring_buffer_event *
789 __trace_buffer_lock_reserve(struct trace_buffer *buffer,
792 unsigned long flags, int pc)
794 struct ring_buffer_event *event;
796 event = ring_buffer_lock_reserve(buffer, len);
798 trace_event_setup(event, type, flags, pc);
803 void tracer_tracing_on(struct trace_array *tr)
805 if (tr->array_buffer.buffer)
806 ring_buffer_record_on(tr->array_buffer.buffer);
808 * This flag is looked at when buffers haven't been allocated
809 * yet, or by some tracers (like irqsoff), that just want to
810 * know if the ring buffer has been disabled, but it can handle
811 * races of where it gets disabled but we still do a record.
812 * As the check is in the fast path of the tracers, it is more
813 * important to be fast than accurate.
815 tr->buffer_disabled = 0;
816 /* Make the flag seen by readers */
821 * tracing_on - enable tracing buffers
823 * This function enables tracing buffers that may have been
824 * disabled with tracing_off.
826 void tracing_on(void)
828 tracer_tracing_on(&global_trace);
830 EXPORT_SYMBOL_GPL(tracing_on);
833 static __always_inline void
834 __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
836 __this_cpu_write(trace_taskinfo_save, true);
838 /* If this is the temp buffer, we need to commit fully */
839 if (this_cpu_read(trace_buffered_event) == event) {
840 /* Length is in event->array[0] */
841 ring_buffer_write(buffer, event->array[0], &event->array[1]);
842 /* Release the temp buffer */
843 this_cpu_dec(trace_buffered_event_cnt);
845 ring_buffer_unlock_commit(buffer, event);
849 * __trace_puts - write a constant string into the trace buffer.
850 * @ip: The address of the caller
851 * @str: The constant string to write
852 * @size: The size of the string.
854 int __trace_puts(unsigned long ip, const char *str, int size)
856 struct ring_buffer_event *event;
857 struct trace_buffer *buffer;
858 struct print_entry *entry;
859 unsigned long irq_flags;
863 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
866 pc = preempt_count();
868 if (unlikely(tracing_selftest_running || tracing_disabled))
871 alloc = sizeof(*entry) + size + 2; /* possible \n added */
873 local_save_flags(irq_flags);
874 buffer = global_trace.array_buffer.buffer;
875 ring_buffer_nest_start(buffer);
876 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
883 entry = ring_buffer_event_data(event);
886 memcpy(&entry->buf, str, size);
888 /* Add a newline if necessary */
889 if (entry->buf[size - 1] != '\n') {
890 entry->buf[size] = '\n';
891 entry->buf[size + 1] = '\0';
893 entry->buf[size] = '\0';
895 __buffer_unlock_commit(buffer, event);
896 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
898 ring_buffer_nest_end(buffer);
901 EXPORT_SYMBOL_GPL(__trace_puts);
904 * __trace_bputs - write the pointer to a constant string into trace buffer
905 * @ip: The address of the caller
906 * @str: The constant string to write to the buffer to
908 int __trace_bputs(unsigned long ip, const char *str)
910 struct ring_buffer_event *event;
911 struct trace_buffer *buffer;
912 struct bputs_entry *entry;
913 unsigned long irq_flags;
914 int size = sizeof(struct bputs_entry);
918 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
921 pc = preempt_count();
923 if (unlikely(tracing_selftest_running || tracing_disabled))
926 local_save_flags(irq_flags);
927 buffer = global_trace.array_buffer.buffer;
929 ring_buffer_nest_start(buffer);
930 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
935 entry = ring_buffer_event_data(event);
939 __buffer_unlock_commit(buffer, event);
940 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
944 ring_buffer_nest_end(buffer);
947 EXPORT_SYMBOL_GPL(__trace_bputs);
949 #ifdef CONFIG_TRACER_SNAPSHOT
950 static void tracing_snapshot_instance_cond(struct trace_array *tr,
953 struct tracer *tracer = tr->current_trace;
957 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
958 internal_trace_puts("*** snapshot is being ignored ***\n");
962 if (!tr->allocated_snapshot) {
963 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
964 internal_trace_puts("*** stopping trace here! ***\n");
969 /* Note, snapshot can not be used when the tracer uses it */
970 if (tracer->use_max_tr) {
971 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
972 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
976 local_irq_save(flags);
977 update_max_tr(tr, current, smp_processor_id(), cond_data);
978 local_irq_restore(flags);
981 void tracing_snapshot_instance(struct trace_array *tr)
983 tracing_snapshot_instance_cond(tr, NULL);
987 * tracing_snapshot - take a snapshot of the current buffer.
989 * This causes a swap between the snapshot buffer and the current live
990 * tracing buffer. You can use this to take snapshots of the live
991 * trace when some condition is triggered, but continue to trace.
993 * Note, make sure to allocate the snapshot with either
994 * a tracing_snapshot_alloc(), or by doing it manually
995 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
997 * If the snapshot buffer is not allocated, it will stop tracing.
998 * Basically making a permanent snapshot.
1000 void tracing_snapshot(void)
1002 struct trace_array *tr = &global_trace;
1004 tracing_snapshot_instance(tr);
1006 EXPORT_SYMBOL_GPL(tracing_snapshot);
1009 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1010 * @tr: The tracing instance to snapshot
1011 * @cond_data: The data to be tested conditionally, and possibly saved
1013 * This is the same as tracing_snapshot() except that the snapshot is
1014 * conditional - the snapshot will only happen if the
1015 * cond_snapshot.update() implementation receiving the cond_data
1016 * returns true, which means that the trace array's cond_snapshot
1017 * update() operation used the cond_data to determine whether the
1018 * snapshot should be taken, and if it was, presumably saved it along
1019 * with the snapshot.
1021 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1023 tracing_snapshot_instance_cond(tr, cond_data);
1025 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1028 * tracing_snapshot_cond_data - get the user data associated with a snapshot
1029 * @tr: The tracing instance
1031 * When the user enables a conditional snapshot using
1032 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1033 * with the snapshot. This accessor is used to retrieve it.
1035 * Should not be called from cond_snapshot.update(), since it takes
1036 * the tr->max_lock lock, which the code calling
1037 * cond_snapshot.update() has already done.
1039 * Returns the cond_data associated with the trace array's snapshot.
1041 void *tracing_cond_snapshot_data(struct trace_array *tr)
1043 void *cond_data = NULL;
1045 arch_spin_lock(&tr->max_lock);
1047 if (tr->cond_snapshot)
1048 cond_data = tr->cond_snapshot->cond_data;
1050 arch_spin_unlock(&tr->max_lock);
1054 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1056 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1057 struct array_buffer *size_buf, int cpu_id);
1058 static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1060 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1064 if (!tr->allocated_snapshot) {
1066 /* allocate spare buffer */
1067 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1068 &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1072 tr->allocated_snapshot = true;
1078 static void free_snapshot(struct trace_array *tr)
1081 * We don't free the ring buffer. instead, resize it because
1082 * The max_tr ring buffer has some state (e.g. ring->clock) and
1083 * we want preserve it.
1085 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1086 set_buffer_entries(&tr->max_buffer, 1);
1087 tracing_reset_online_cpus(&tr->max_buffer);
1088 tr->allocated_snapshot = false;
1092 * tracing_alloc_snapshot - allocate snapshot buffer.
1094 * This only allocates the snapshot buffer if it isn't already
1095 * allocated - it doesn't also take a snapshot.
1097 * This is meant to be used in cases where the snapshot buffer needs
1098 * to be set up for events that can't sleep but need to be able to
1099 * trigger a snapshot.
1101 int tracing_alloc_snapshot(void)
1103 struct trace_array *tr = &global_trace;
1106 ret = tracing_alloc_snapshot_instance(tr);
1111 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1114 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1116 * This is similar to tracing_snapshot(), but it will allocate the
1117 * snapshot buffer if it isn't already allocated. Use this only
1118 * where it is safe to sleep, as the allocation may sleep.
1120 * This causes a swap between the snapshot buffer and the current live
1121 * tracing buffer. You can use this to take snapshots of the live
1122 * trace when some condition is triggered, but continue to trace.
1124 void tracing_snapshot_alloc(void)
1128 ret = tracing_alloc_snapshot();
1134 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1137 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1138 * @tr: The tracing instance
1139 * @cond_data: User data to associate with the snapshot
1140 * @update: Implementation of the cond_snapshot update function
1142 * Check whether the conditional snapshot for the given instance has
1143 * already been enabled, or if the current tracer is already using a
1144 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1145 * save the cond_data and update function inside.
1147 * Returns 0 if successful, error otherwise.
1149 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1150 cond_update_fn_t update)
1152 struct cond_snapshot *cond_snapshot;
1155 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1159 cond_snapshot->cond_data = cond_data;
1160 cond_snapshot->update = update;
1162 mutex_lock(&trace_types_lock);
1164 ret = tracing_alloc_snapshot_instance(tr);
1168 if (tr->current_trace->use_max_tr) {
1174 * The cond_snapshot can only change to NULL without the
1175 * trace_types_lock. We don't care if we race with it going
1176 * to NULL, but we want to make sure that it's not set to
1177 * something other than NULL when we get here, which we can
1178 * do safely with only holding the trace_types_lock and not
1179 * having to take the max_lock.
1181 if (tr->cond_snapshot) {
1186 arch_spin_lock(&tr->max_lock);
1187 tr->cond_snapshot = cond_snapshot;
1188 arch_spin_unlock(&tr->max_lock);
1190 mutex_unlock(&trace_types_lock);
1195 mutex_unlock(&trace_types_lock);
1196 kfree(cond_snapshot);
1199 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1202 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1203 * @tr: The tracing instance
1205 * Check whether the conditional snapshot for the given instance is
1206 * enabled; if so, free the cond_snapshot associated with it,
1207 * otherwise return -EINVAL.
1209 * Returns 0 if successful, error otherwise.
1211 int tracing_snapshot_cond_disable(struct trace_array *tr)
1215 arch_spin_lock(&tr->max_lock);
1217 if (!tr->cond_snapshot)
1220 kfree(tr->cond_snapshot);
1221 tr->cond_snapshot = NULL;
1224 arch_spin_unlock(&tr->max_lock);
1228 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1230 void tracing_snapshot(void)
1232 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1234 EXPORT_SYMBOL_GPL(tracing_snapshot);
1235 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1237 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1239 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1240 int tracing_alloc_snapshot(void)
1242 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1245 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1246 void tracing_snapshot_alloc(void)
1251 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1252 void *tracing_cond_snapshot_data(struct trace_array *tr)
1256 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1257 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1261 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1262 int tracing_snapshot_cond_disable(struct trace_array *tr)
1266 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1267 #endif /* CONFIG_TRACER_SNAPSHOT */
1269 void tracer_tracing_off(struct trace_array *tr)
1271 if (tr->array_buffer.buffer)
1272 ring_buffer_record_off(tr->array_buffer.buffer);
1274 * This flag is looked at when buffers haven't been allocated
1275 * yet, or by some tracers (like irqsoff), that just want to
1276 * know if the ring buffer has been disabled, but it can handle
1277 * races of where it gets disabled but we still do a record.
1278 * As the check is in the fast path of the tracers, it is more
1279 * important to be fast than accurate.
1281 tr->buffer_disabled = 1;
1282 /* Make the flag seen by readers */
1287 * tracing_off - turn off tracing buffers
1289 * This function stops the tracing buffers from recording data.
1290 * It does not disable any overhead the tracers themselves may
1291 * be causing. This function simply causes all recording to
1292 * the ring buffers to fail.
1294 void tracing_off(void)
1296 tracer_tracing_off(&global_trace);
1298 EXPORT_SYMBOL_GPL(tracing_off);
1300 void disable_trace_on_warning(void)
1302 if (__disable_trace_on_warning)
1307 * tracer_tracing_is_on - show real state of ring buffer enabled
1308 * @tr : the trace array to know if ring buffer is enabled
1310 * Shows real state of the ring buffer if it is enabled or not.
1312 bool tracer_tracing_is_on(struct trace_array *tr)
1314 if (tr->array_buffer.buffer)
1315 return ring_buffer_record_is_on(tr->array_buffer.buffer);
1316 return !tr->buffer_disabled;
1320 * tracing_is_on - show state of ring buffers enabled
1322 int tracing_is_on(void)
1324 return tracer_tracing_is_on(&global_trace);
1326 EXPORT_SYMBOL_GPL(tracing_is_on);
1328 static int __init set_buf_size(char *str)
1330 unsigned long buf_size;
1334 buf_size = memparse(str, &str);
1335 /* nr_entries can not be zero */
1338 trace_buf_size = buf_size;
1341 __setup("trace_buf_size=", set_buf_size);
1343 static int __init set_tracing_thresh(char *str)
1345 unsigned long threshold;
1350 ret = kstrtoul(str, 0, &threshold);
1353 tracing_thresh = threshold * 1000;
1356 __setup("tracing_thresh=", set_tracing_thresh);
1358 unsigned long nsecs_to_usecs(unsigned long nsecs)
1360 return nsecs / 1000;
1364 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1365 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1366 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1367 * of strings in the order that the evals (enum) were defined.
1372 /* These must match the bit postions in trace_iterator_flags */
1373 static const char *trace_options[] = {
1381 int in_ns; /* is this clock in nanoseconds? */
1382 } trace_clocks[] = {
1383 { trace_clock_local, "local", 1 },
1384 { trace_clock_global, "global", 1 },
1385 { trace_clock_counter, "counter", 0 },
1386 { trace_clock_jiffies, "uptime", 0 },
1387 { trace_clock, "perf", 1 },
1388 { ktime_get_mono_fast_ns, "mono", 1 },
1389 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1390 { ktime_get_boot_fast_ns, "boot", 1 },
1394 bool trace_clock_in_ns(struct trace_array *tr)
1396 if (trace_clocks[tr->clock_id].in_ns)
1403 * trace_parser_get_init - gets the buffer for trace parser
1405 int trace_parser_get_init(struct trace_parser *parser, int size)
1407 memset(parser, 0, sizeof(*parser));
1409 parser->buffer = kmalloc(size, GFP_KERNEL);
1410 if (!parser->buffer)
1413 parser->size = size;
1418 * trace_parser_put - frees the buffer for trace parser
1420 void trace_parser_put(struct trace_parser *parser)
1422 kfree(parser->buffer);
1423 parser->buffer = NULL;
1427 * trace_get_user - reads the user input string separated by space
1428 * (matched by isspace(ch))
1430 * For each string found the 'struct trace_parser' is updated,
1431 * and the function returns.
1433 * Returns number of bytes read.
1435 * See kernel/trace/trace.h for 'struct trace_parser' details.
1437 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1438 size_t cnt, loff_t *ppos)
1445 trace_parser_clear(parser);
1447 ret = get_user(ch, ubuf++);
1455 * The parser is not finished with the last write,
1456 * continue reading the user input without skipping spaces.
1458 if (!parser->cont) {
1459 /* skip white space */
1460 while (cnt && isspace(ch)) {
1461 ret = get_user(ch, ubuf++);
1470 /* only spaces were written */
1471 if (isspace(ch) || !ch) {
1478 /* read the non-space input */
1479 while (cnt && !isspace(ch) && ch) {
1480 if (parser->idx < parser->size - 1)
1481 parser->buffer[parser->idx++] = ch;
1486 ret = get_user(ch, ubuf++);
1493 /* We either got finished input or we have to wait for another call. */
1494 if (isspace(ch) || !ch) {
1495 parser->buffer[parser->idx] = 0;
1496 parser->cont = false;
1497 } else if (parser->idx < parser->size - 1) {
1498 parser->cont = true;
1499 parser->buffer[parser->idx++] = ch;
1500 /* Make sure the parsed string always terminates with '\0'. */
1501 parser->buffer[parser->idx] = 0;
1514 /* TODO add a seq_buf_to_buffer() */
1515 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1519 if (trace_seq_used(s) <= s->seq.readpos)
1522 len = trace_seq_used(s) - s->seq.readpos;
1525 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1527 s->seq.readpos += cnt;
1531 unsigned long __read_mostly tracing_thresh;
1532 static const struct file_operations tracing_max_lat_fops;
1534 #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1535 defined(CONFIG_FSNOTIFY)
1537 static struct workqueue_struct *fsnotify_wq;
1539 static void latency_fsnotify_workfn(struct work_struct *work)
1541 struct trace_array *tr = container_of(work, struct trace_array,
1543 fsnotify(tr->d_max_latency->d_inode, FS_MODIFY,
1544 tr->d_max_latency->d_inode, FSNOTIFY_EVENT_INODE, NULL, 0);
1547 static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1549 struct trace_array *tr = container_of(iwork, struct trace_array,
1551 queue_work(fsnotify_wq, &tr->fsnotify_work);
1554 static void trace_create_maxlat_file(struct trace_array *tr,
1555 struct dentry *d_tracer)
1557 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1558 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1559 tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
1560 d_tracer, &tr->max_latency,
1561 &tracing_max_lat_fops);
1564 __init static int latency_fsnotify_init(void)
1566 fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1567 WQ_UNBOUND | WQ_HIGHPRI, 0);
1569 pr_err("Unable to allocate tr_max_lat_wq\n");
1575 late_initcall_sync(latency_fsnotify_init);
1577 void latency_fsnotify(struct trace_array *tr)
1582 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1583 * possible that we are called from __schedule() or do_idle(), which
1584 * could cause a deadlock.
1586 irq_work_queue(&tr->fsnotify_irqwork);
1590 * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1591 * defined(CONFIG_FSNOTIFY)
1595 #define trace_create_maxlat_file(tr, d_tracer) \
1596 trace_create_file("tracing_max_latency", 0644, d_tracer, \
1597 &tr->max_latency, &tracing_max_lat_fops)
1601 #ifdef CONFIG_TRACER_MAX_TRACE
1603 * Copy the new maximum trace into the separate maximum-trace
1604 * structure. (this way the maximum trace is permanently saved,
1605 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1608 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1610 struct array_buffer *trace_buf = &tr->array_buffer;
1611 struct array_buffer *max_buf = &tr->max_buffer;
1612 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1613 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1616 max_buf->time_start = data->preempt_timestamp;
1618 max_data->saved_latency = tr->max_latency;
1619 max_data->critical_start = data->critical_start;
1620 max_data->critical_end = data->critical_end;
1622 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1623 max_data->pid = tsk->pid;
1625 * If tsk == current, then use current_uid(), as that does not use
1626 * RCU. The irq tracer can be called out of RCU scope.
1629 max_data->uid = current_uid();
1631 max_data->uid = task_uid(tsk);
1633 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1634 max_data->policy = tsk->policy;
1635 max_data->rt_priority = tsk->rt_priority;
1637 /* record this tasks comm */
1638 tracing_record_cmdline(tsk);
1639 latency_fsnotify(tr);
1643 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1645 * @tsk: the task with the latency
1646 * @cpu: The cpu that initiated the trace.
1647 * @cond_data: User data associated with a conditional snapshot
1649 * Flip the buffers between the @tr and the max_tr and record information
1650 * about which task was the cause of this latency.
1653 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1659 WARN_ON_ONCE(!irqs_disabled());
1661 if (!tr->allocated_snapshot) {
1662 /* Only the nop tracer should hit this when disabling */
1663 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1667 arch_spin_lock(&tr->max_lock);
1669 /* Inherit the recordable setting from array_buffer */
1670 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1671 ring_buffer_record_on(tr->max_buffer.buffer);
1673 ring_buffer_record_off(tr->max_buffer.buffer);
1675 #ifdef CONFIG_TRACER_SNAPSHOT
1676 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1679 swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1681 __update_max_tr(tr, tsk, cpu);
1684 arch_spin_unlock(&tr->max_lock);
1688 * update_max_tr_single - only copy one trace over, and reset the rest
1690 * @tsk: task with the latency
1691 * @cpu: the cpu of the buffer to copy.
1693 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1696 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1703 WARN_ON_ONCE(!irqs_disabled());
1704 if (!tr->allocated_snapshot) {
1705 /* Only the nop tracer should hit this when disabling */
1706 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1710 arch_spin_lock(&tr->max_lock);
1712 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1714 if (ret == -EBUSY) {
1716 * We failed to swap the buffer due to a commit taking
1717 * place on this CPU. We fail to record, but we reset
1718 * the max trace buffer (no one writes directly to it)
1719 * and flag that it failed.
1721 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1722 "Failed to swap buffers due to commit in progress\n");
1725 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1727 __update_max_tr(tr, tsk, cpu);
1728 arch_spin_unlock(&tr->max_lock);
1730 #endif /* CONFIG_TRACER_MAX_TRACE */
1732 static int wait_on_pipe(struct trace_iterator *iter, int full)
1734 /* Iterators are static, they should be filled or empty */
1735 if (trace_buffer_iter(iter, iter->cpu_file))
1738 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1742 #ifdef CONFIG_FTRACE_STARTUP_TEST
1743 static bool selftests_can_run;
1745 struct trace_selftests {
1746 struct list_head list;
1747 struct tracer *type;
1750 static LIST_HEAD(postponed_selftests);
1752 static int save_selftest(struct tracer *type)
1754 struct trace_selftests *selftest;
1756 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1760 selftest->type = type;
1761 list_add(&selftest->list, &postponed_selftests);
1765 static int run_tracer_selftest(struct tracer *type)
1767 struct trace_array *tr = &global_trace;
1768 struct tracer *saved_tracer = tr->current_trace;
1771 if (!type->selftest || tracing_selftest_disabled)
1775 * If a tracer registers early in boot up (before scheduling is
1776 * initialized and such), then do not run its selftests yet.
1777 * Instead, run it a little later in the boot process.
1779 if (!selftests_can_run)
1780 return save_selftest(type);
1783 * Run a selftest on this tracer.
1784 * Here we reset the trace buffer, and set the current
1785 * tracer to be this tracer. The tracer can then run some
1786 * internal tracing to verify that everything is in order.
1787 * If we fail, we do not register this tracer.
1789 tracing_reset_online_cpus(&tr->array_buffer);
1791 tr->current_trace = type;
1793 #ifdef CONFIG_TRACER_MAX_TRACE
1794 if (type->use_max_tr) {
1795 /* If we expanded the buffers, make sure the max is expanded too */
1796 if (ring_buffer_expanded)
1797 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1798 RING_BUFFER_ALL_CPUS);
1799 tr->allocated_snapshot = true;
1803 /* the test is responsible for initializing and enabling */
1804 pr_info("Testing tracer %s: ", type->name);
1805 ret = type->selftest(type, tr);
1806 /* the test is responsible for resetting too */
1807 tr->current_trace = saved_tracer;
1809 printk(KERN_CONT "FAILED!\n");
1810 /* Add the warning after printing 'FAILED' */
1814 /* Only reset on passing, to avoid touching corrupted buffers */
1815 tracing_reset_online_cpus(&tr->array_buffer);
1817 #ifdef CONFIG_TRACER_MAX_TRACE
1818 if (type->use_max_tr) {
1819 tr->allocated_snapshot = false;
1821 /* Shrink the max buffer again */
1822 if (ring_buffer_expanded)
1823 ring_buffer_resize(tr->max_buffer.buffer, 1,
1824 RING_BUFFER_ALL_CPUS);
1828 printk(KERN_CONT "PASSED\n");
1832 static __init int init_trace_selftests(void)
1834 struct trace_selftests *p, *n;
1835 struct tracer *t, **last;
1838 selftests_can_run = true;
1840 mutex_lock(&trace_types_lock);
1842 if (list_empty(&postponed_selftests))
1845 pr_info("Running postponed tracer tests:\n");
1847 tracing_selftest_running = true;
1848 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1849 /* This loop can take minutes when sanitizers are enabled, so
1850 * lets make sure we allow RCU processing.
1853 ret = run_tracer_selftest(p->type);
1854 /* If the test fails, then warn and remove from available_tracers */
1856 WARN(1, "tracer: %s failed selftest, disabling\n",
1858 last = &trace_types;
1859 for (t = trace_types; t; t = t->next) {
1870 tracing_selftest_running = false;
1873 mutex_unlock(&trace_types_lock);
1877 core_initcall(init_trace_selftests);
1879 static inline int run_tracer_selftest(struct tracer *type)
1883 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1885 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1887 static void __init apply_trace_boot_options(void);
1890 * register_tracer - register a tracer with the ftrace system.
1891 * @type: the plugin for the tracer
1893 * Register a new plugin tracer.
1895 int __init register_tracer(struct tracer *type)
1901 pr_info("Tracer must have a name\n");
1905 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1906 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1910 if (security_locked_down(LOCKDOWN_TRACEFS)) {
1911 pr_warn("Can not register tracer %s due to lockdown\n",
1916 mutex_lock(&trace_types_lock);
1918 tracing_selftest_running = true;
1920 for (t = trace_types; t; t = t->next) {
1921 if (strcmp(type->name, t->name) == 0) {
1923 pr_info("Tracer %s already registered\n",
1930 if (!type->set_flag)
1931 type->set_flag = &dummy_set_flag;
1933 /*allocate a dummy tracer_flags*/
1934 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1939 type->flags->val = 0;
1940 type->flags->opts = dummy_tracer_opt;
1942 if (!type->flags->opts)
1943 type->flags->opts = dummy_tracer_opt;
1945 /* store the tracer for __set_tracer_option */
1946 type->flags->trace = type;
1948 ret = run_tracer_selftest(type);
1952 type->next = trace_types;
1954 add_tracer_options(&global_trace, type);
1957 tracing_selftest_running = false;
1958 mutex_unlock(&trace_types_lock);
1960 if (ret || !default_bootup_tracer)
1963 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1966 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1967 /* Do we want this tracer to start on bootup? */
1968 tracing_set_tracer(&global_trace, type->name);
1969 default_bootup_tracer = NULL;
1971 apply_trace_boot_options();
1973 /* disable other selftests, since this will break it. */
1974 tracing_selftest_disabled = true;
1975 #ifdef CONFIG_FTRACE_STARTUP_TEST
1976 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1984 static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
1986 struct trace_buffer *buffer = buf->buffer;
1991 ring_buffer_record_disable(buffer);
1993 /* Make sure all commits have finished */
1995 ring_buffer_reset_cpu(buffer, cpu);
1997 ring_buffer_record_enable(buffer);
2000 void tracing_reset_online_cpus(struct array_buffer *buf)
2002 struct trace_buffer *buffer = buf->buffer;
2008 ring_buffer_record_disable(buffer);
2010 /* Make sure all commits have finished */
2013 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2015 for_each_online_cpu(cpu)
2016 ring_buffer_reset_cpu(buffer, cpu);
2018 ring_buffer_record_enable(buffer);
2021 /* Must have trace_types_lock held */
2022 void tracing_reset_all_online_cpus(void)
2024 struct trace_array *tr;
2026 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2027 if (!tr->clear_trace)
2029 tr->clear_trace = false;
2030 tracing_reset_online_cpus(&tr->array_buffer);
2031 #ifdef CONFIG_TRACER_MAX_TRACE
2032 tracing_reset_online_cpus(&tr->max_buffer);
2037 static int *tgid_map;
2039 #define SAVED_CMDLINES_DEFAULT 128
2040 #define NO_CMDLINE_MAP UINT_MAX
2041 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2042 struct saved_cmdlines_buffer {
2043 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2044 unsigned *map_cmdline_to_pid;
2045 unsigned cmdline_num;
2047 char *saved_cmdlines;
2049 static struct saved_cmdlines_buffer *savedcmd;
2051 /* temporary disable recording */
2052 static atomic_t trace_record_taskinfo_disabled __read_mostly;
2054 static inline char *get_saved_cmdlines(int idx)
2056 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2059 static inline void set_cmdline(int idx, const char *cmdline)
2061 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2064 static int allocate_cmdlines_buffer(unsigned int val,
2065 struct saved_cmdlines_buffer *s)
2067 s->map_cmdline_to_pid = kmalloc_array(val,
2068 sizeof(*s->map_cmdline_to_pid),
2070 if (!s->map_cmdline_to_pid)
2073 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2074 if (!s->saved_cmdlines) {
2075 kfree(s->map_cmdline_to_pid);
2080 s->cmdline_num = val;
2081 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2082 sizeof(s->map_pid_to_cmdline));
2083 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2084 val * sizeof(*s->map_cmdline_to_pid));
2089 static int trace_create_savedcmd(void)
2093 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2097 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2107 int is_tracing_stopped(void)
2109 return global_trace.stop_count;
2113 * tracing_start - quick start of the tracer
2115 * If tracing is enabled but was stopped by tracing_stop,
2116 * this will start the tracer back up.
2118 void tracing_start(void)
2120 struct trace_buffer *buffer;
2121 unsigned long flags;
2123 if (tracing_disabled)
2126 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2127 if (--global_trace.stop_count) {
2128 if (global_trace.stop_count < 0) {
2129 /* Someone screwed up their debugging */
2131 global_trace.stop_count = 0;
2136 /* Prevent the buffers from switching */
2137 arch_spin_lock(&global_trace.max_lock);
2139 buffer = global_trace.array_buffer.buffer;
2141 ring_buffer_record_enable(buffer);
2143 #ifdef CONFIG_TRACER_MAX_TRACE
2144 buffer = global_trace.max_buffer.buffer;
2146 ring_buffer_record_enable(buffer);
2149 arch_spin_unlock(&global_trace.max_lock);
2152 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2155 static void tracing_start_tr(struct trace_array *tr)
2157 struct trace_buffer *buffer;
2158 unsigned long flags;
2160 if (tracing_disabled)
2163 /* If global, we need to also start the max tracer */
2164 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2165 return tracing_start();
2167 raw_spin_lock_irqsave(&tr->start_lock, flags);
2169 if (--tr->stop_count) {
2170 if (tr->stop_count < 0) {
2171 /* Someone screwed up their debugging */
2178 buffer = tr->array_buffer.buffer;
2180 ring_buffer_record_enable(buffer);
2183 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2187 * tracing_stop - quick stop of the tracer
2189 * Light weight way to stop tracing. Use in conjunction with
2192 void tracing_stop(void)
2194 struct trace_buffer *buffer;
2195 unsigned long flags;
2197 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2198 if (global_trace.stop_count++)
2201 /* Prevent the buffers from switching */
2202 arch_spin_lock(&global_trace.max_lock);
2204 buffer = global_trace.array_buffer.buffer;
2206 ring_buffer_record_disable(buffer);
2208 #ifdef CONFIG_TRACER_MAX_TRACE
2209 buffer = global_trace.max_buffer.buffer;
2211 ring_buffer_record_disable(buffer);
2214 arch_spin_unlock(&global_trace.max_lock);
2217 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2220 static void tracing_stop_tr(struct trace_array *tr)
2222 struct trace_buffer *buffer;
2223 unsigned long flags;
2225 /* If global, we need to also stop the max tracer */
2226 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2227 return tracing_stop();
2229 raw_spin_lock_irqsave(&tr->start_lock, flags);
2230 if (tr->stop_count++)
2233 buffer = tr->array_buffer.buffer;
2235 ring_buffer_record_disable(buffer);
2238 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2241 static int trace_save_cmdline(struct task_struct *tsk)
2245 /* treat recording of idle task as a success */
2249 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
2253 * It's not the end of the world if we don't get
2254 * the lock, but we also don't want to spin
2255 * nor do we want to disable interrupts,
2256 * so if we miss here, then better luck next time.
2258 if (!arch_spin_trylock(&trace_cmdline_lock))
2261 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2262 if (idx == NO_CMDLINE_MAP) {
2263 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2266 * Check whether the cmdline buffer at idx has a pid
2267 * mapped. We are going to overwrite that entry so we
2268 * need to clear the map_pid_to_cmdline. Otherwise we
2269 * would read the new comm for the old pid.
2271 pid = savedcmd->map_cmdline_to_pid[idx];
2272 if (pid != NO_CMDLINE_MAP)
2273 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
2275 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2276 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
2278 savedcmd->cmdline_idx = idx;
2281 set_cmdline(idx, tsk->comm);
2283 arch_spin_unlock(&trace_cmdline_lock);
2288 static void __trace_find_cmdline(int pid, char comm[])
2293 strcpy(comm, "<idle>");
2297 if (WARN_ON_ONCE(pid < 0)) {
2298 strcpy(comm, "<XXX>");
2302 if (pid > PID_MAX_DEFAULT) {
2303 strcpy(comm, "<...>");
2307 map = savedcmd->map_pid_to_cmdline[pid];
2308 if (map != NO_CMDLINE_MAP)
2309 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2311 strcpy(comm, "<...>");
2314 void trace_find_cmdline(int pid, char comm[])
2317 arch_spin_lock(&trace_cmdline_lock);
2319 __trace_find_cmdline(pid, comm);
2321 arch_spin_unlock(&trace_cmdline_lock);
2325 int trace_find_tgid(int pid)
2327 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2330 return tgid_map[pid];
2333 static int trace_save_tgid(struct task_struct *tsk)
2335 /* treat recording of idle task as a success */
2339 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2342 tgid_map[tsk->pid] = tsk->tgid;
2346 static bool tracing_record_taskinfo_skip(int flags)
2348 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2350 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2352 if (!__this_cpu_read(trace_taskinfo_save))
2358 * tracing_record_taskinfo - record the task info of a task
2360 * @task: task to record
2361 * @flags: TRACE_RECORD_CMDLINE for recording comm
2362 * TRACE_RECORD_TGID for recording tgid
2364 void tracing_record_taskinfo(struct task_struct *task, int flags)
2368 if (tracing_record_taskinfo_skip(flags))
2372 * Record as much task information as possible. If some fail, continue
2373 * to try to record the others.
2375 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2376 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2378 /* If recording any information failed, retry again soon. */
2382 __this_cpu_write(trace_taskinfo_save, false);
2386 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2388 * @prev: previous task during sched_switch
2389 * @next: next task during sched_switch
2390 * @flags: TRACE_RECORD_CMDLINE for recording comm
2391 * TRACE_RECORD_TGID for recording tgid
2393 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2394 struct task_struct *next, int flags)
2398 if (tracing_record_taskinfo_skip(flags))
2402 * Record as much task information as possible. If some fail, continue
2403 * to try to record the others.
2405 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2406 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2407 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2408 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2410 /* If recording any information failed, retry again soon. */
2414 __this_cpu_write(trace_taskinfo_save, false);
2417 /* Helpers to record a specific task information */
2418 void tracing_record_cmdline(struct task_struct *task)
2420 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2423 void tracing_record_tgid(struct task_struct *task)
2425 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2429 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2430 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2431 * simplifies those functions and keeps them in sync.
2433 enum print_line_t trace_handle_return(struct trace_seq *s)
2435 return trace_seq_has_overflowed(s) ?
2436 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2438 EXPORT_SYMBOL_GPL(trace_handle_return);
2441 tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
2442 unsigned long flags, int pc)
2444 struct task_struct *tsk = current;
2446 entry->preempt_count = pc & 0xff;
2447 entry->pid = (tsk) ? tsk->pid : 0;
2450 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2451 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2453 TRACE_FLAG_IRQS_NOSUPPORT |
2455 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
2456 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2457 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2458 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2459 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2461 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2463 struct ring_buffer_event *
2464 trace_buffer_lock_reserve(struct trace_buffer *buffer,
2467 unsigned long flags, int pc)
2469 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2472 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2473 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2474 static int trace_buffered_event_ref;
2477 * trace_buffered_event_enable - enable buffering events
2479 * When events are being filtered, it is quicker to use a temporary
2480 * buffer to write the event data into if there's a likely chance
2481 * that it will not be committed. The discard of the ring buffer
2482 * is not as fast as committing, and is much slower than copying
2485 * When an event is to be filtered, allocate per cpu buffers to
2486 * write the event data into, and if the event is filtered and discarded
2487 * it is simply dropped, otherwise, the entire data is to be committed
2490 void trace_buffered_event_enable(void)
2492 struct ring_buffer_event *event;
2496 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2498 if (trace_buffered_event_ref++)
2501 for_each_tracing_cpu(cpu) {
2502 page = alloc_pages_node(cpu_to_node(cpu),
2503 GFP_KERNEL | __GFP_NORETRY, 0);
2507 event = page_address(page);
2508 memset(event, 0, sizeof(*event));
2510 per_cpu(trace_buffered_event, cpu) = event;
2513 if (cpu == smp_processor_id() &&
2514 this_cpu_read(trace_buffered_event) !=
2515 per_cpu(trace_buffered_event, cpu))
2522 trace_buffered_event_disable();
2525 static void enable_trace_buffered_event(void *data)
2527 /* Probably not needed, but do it anyway */
2529 this_cpu_dec(trace_buffered_event_cnt);
2532 static void disable_trace_buffered_event(void *data)
2534 this_cpu_inc(trace_buffered_event_cnt);
2538 * trace_buffered_event_disable - disable buffering events
2540 * When a filter is removed, it is faster to not use the buffered
2541 * events, and to commit directly into the ring buffer. Free up
2542 * the temp buffers when there are no more users. This requires
2543 * special synchronization with current events.
2545 void trace_buffered_event_disable(void)
2549 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2551 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2554 if (--trace_buffered_event_ref)
2558 /* For each CPU, set the buffer as used. */
2559 smp_call_function_many(tracing_buffer_mask,
2560 disable_trace_buffered_event, NULL, 1);
2563 /* Wait for all current users to finish */
2566 for_each_tracing_cpu(cpu) {
2567 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2568 per_cpu(trace_buffered_event, cpu) = NULL;
2571 * Make sure trace_buffered_event is NULL before clearing
2572 * trace_buffered_event_cnt.
2577 /* Do the work on each cpu */
2578 smp_call_function_many(tracing_buffer_mask,
2579 enable_trace_buffered_event, NULL, 1);
2583 static struct trace_buffer *temp_buffer;
2585 struct ring_buffer_event *
2586 trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2587 struct trace_event_file *trace_file,
2588 int type, unsigned long len,
2589 unsigned long flags, int pc)
2591 struct ring_buffer_event *entry;
2594 *current_rb = trace_file->tr->array_buffer.buffer;
2596 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2597 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2598 (entry = this_cpu_read(trace_buffered_event))) {
2599 /* Try to use the per cpu buffer first */
2600 val = this_cpu_inc_return(trace_buffered_event_cnt);
2602 trace_event_setup(entry, type, flags, pc);
2603 entry->array[0] = len;
2606 this_cpu_dec(trace_buffered_event_cnt);
2609 entry = __trace_buffer_lock_reserve(*current_rb,
2610 type, len, flags, pc);
2612 * If tracing is off, but we have triggers enabled
2613 * we still need to look at the event data. Use the temp_buffer
2614 * to store the trace event for the tigger to use. It's recusive
2615 * safe and will not be recorded anywhere.
2617 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2618 *current_rb = temp_buffer;
2619 entry = __trace_buffer_lock_reserve(*current_rb,
2620 type, len, flags, pc);
2624 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2626 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2627 static DEFINE_MUTEX(tracepoint_printk_mutex);
2629 static void output_printk(struct trace_event_buffer *fbuffer)
2631 struct trace_event_call *event_call;
2632 struct trace_event_file *file;
2633 struct trace_event *event;
2634 unsigned long flags;
2635 struct trace_iterator *iter = tracepoint_print_iter;
2637 /* We should never get here if iter is NULL */
2638 if (WARN_ON_ONCE(!iter))
2641 event_call = fbuffer->trace_file->event_call;
2642 if (!event_call || !event_call->event.funcs ||
2643 !event_call->event.funcs->trace)
2646 file = fbuffer->trace_file;
2647 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2648 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2649 !filter_match_preds(file->filter, fbuffer->entry)))
2652 event = &fbuffer->trace_file->event_call->event;
2654 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2655 trace_seq_init(&iter->seq);
2656 iter->ent = fbuffer->entry;
2657 event_call->event.funcs->trace(iter, 0, event);
2658 trace_seq_putc(&iter->seq, 0);
2659 printk("%s", iter->seq.buffer);
2661 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2664 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2665 void __user *buffer, size_t *lenp,
2668 int save_tracepoint_printk;
2671 mutex_lock(&tracepoint_printk_mutex);
2672 save_tracepoint_printk = tracepoint_printk;
2674 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2677 * This will force exiting early, as tracepoint_printk
2678 * is always zero when tracepoint_printk_iter is not allocated
2680 if (!tracepoint_print_iter)
2681 tracepoint_printk = 0;
2683 if (save_tracepoint_printk == tracepoint_printk)
2686 if (tracepoint_printk)
2687 static_key_enable(&tracepoint_printk_key.key);
2689 static_key_disable(&tracepoint_printk_key.key);
2692 mutex_unlock(&tracepoint_printk_mutex);
2697 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2699 if (static_key_false(&tracepoint_printk_key.key))
2700 output_printk(fbuffer);
2702 event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer,
2703 fbuffer->event, fbuffer->entry,
2704 fbuffer->flags, fbuffer->pc, fbuffer->regs);
2706 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2711 * trace_buffer_unlock_commit_regs()
2712 * trace_event_buffer_commit()
2713 * trace_event_raw_event_xxx()
2715 # define STACK_SKIP 3
2717 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2718 struct trace_buffer *buffer,
2719 struct ring_buffer_event *event,
2720 unsigned long flags, int pc,
2721 struct pt_regs *regs)
2723 __buffer_unlock_commit(buffer, event);
2726 * If regs is not set, then skip the necessary functions.
2727 * Note, we can still get here via blktrace, wakeup tracer
2728 * and mmiotrace, but that's ok if they lose a function or
2729 * two. They are not that meaningful.
2731 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2732 ftrace_trace_userstack(buffer, flags, pc);
2736 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2739 trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2740 struct ring_buffer_event *event)
2742 __buffer_unlock_commit(buffer, event);
2746 trace_process_export(struct trace_export *export,
2747 struct ring_buffer_event *event)
2749 struct trace_entry *entry;
2750 unsigned int size = 0;
2752 entry = ring_buffer_event_data(event);
2753 size = ring_buffer_event_length(event);
2754 export->write(export, entry, size);
2757 static DEFINE_MUTEX(ftrace_export_lock);
2759 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2761 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2763 static inline void ftrace_exports_enable(void)
2765 static_branch_enable(&ftrace_exports_enabled);
2768 static inline void ftrace_exports_disable(void)
2770 static_branch_disable(&ftrace_exports_enabled);
2773 static void ftrace_exports(struct ring_buffer_event *event)
2775 struct trace_export *export;
2777 preempt_disable_notrace();
2779 export = rcu_dereference_raw_check(ftrace_exports_list);
2781 trace_process_export(export, event);
2782 export = rcu_dereference_raw_check(export->next);
2785 preempt_enable_notrace();
2789 add_trace_export(struct trace_export **list, struct trace_export *export)
2791 rcu_assign_pointer(export->next, *list);
2793 * We are entering export into the list but another
2794 * CPU might be walking that list. We need to make sure
2795 * the export->next pointer is valid before another CPU sees
2796 * the export pointer included into the list.
2798 rcu_assign_pointer(*list, export);
2802 rm_trace_export(struct trace_export **list, struct trace_export *export)
2804 struct trace_export **p;
2806 for (p = list; *p != NULL; p = &(*p)->next)
2813 rcu_assign_pointer(*p, (*p)->next);
2819 add_ftrace_export(struct trace_export **list, struct trace_export *export)
2822 ftrace_exports_enable();
2824 add_trace_export(list, export);
2828 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2832 ret = rm_trace_export(list, export);
2834 ftrace_exports_disable();
2839 int register_ftrace_export(struct trace_export *export)
2841 if (WARN_ON_ONCE(!export->write))
2844 mutex_lock(&ftrace_export_lock);
2846 add_ftrace_export(&ftrace_exports_list, export);
2848 mutex_unlock(&ftrace_export_lock);
2852 EXPORT_SYMBOL_GPL(register_ftrace_export);
2854 int unregister_ftrace_export(struct trace_export *export)
2858 mutex_lock(&ftrace_export_lock);
2860 ret = rm_ftrace_export(&ftrace_exports_list, export);
2862 mutex_unlock(&ftrace_export_lock);
2866 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2869 trace_function(struct trace_array *tr,
2870 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2873 struct trace_event_call *call = &event_function;
2874 struct trace_buffer *buffer = tr->array_buffer.buffer;
2875 struct ring_buffer_event *event;
2876 struct ftrace_entry *entry;
2878 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2882 entry = ring_buffer_event_data(event);
2884 entry->parent_ip = parent_ip;
2886 if (!call_filter_check_discard(call, entry, buffer, event)) {
2887 if (static_branch_unlikely(&ftrace_exports_enabled))
2888 ftrace_exports(event);
2889 __buffer_unlock_commit(buffer, event);
2893 #ifdef CONFIG_STACKTRACE
2895 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2896 #define FTRACE_KSTACK_NESTING 4
2898 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2900 struct ftrace_stack {
2901 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2905 struct ftrace_stacks {
2906 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2909 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2910 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2912 static void __ftrace_trace_stack(struct trace_buffer *buffer,
2913 unsigned long flags,
2914 int skip, int pc, struct pt_regs *regs)
2916 struct trace_event_call *call = &event_kernel_stack;
2917 struct ring_buffer_event *event;
2918 unsigned int size, nr_entries;
2919 struct ftrace_stack *fstack;
2920 struct stack_entry *entry;
2924 * Add one, for this function and the call to save_stack_trace()
2925 * If regs is set, then these functions will not be in the way.
2927 #ifndef CONFIG_UNWINDER_ORC
2933 * Since events can happen in NMIs there's no safe way to
2934 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2935 * or NMI comes in, it will just have to use the default
2936 * FTRACE_STACK_SIZE.
2938 preempt_disable_notrace();
2940 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2942 /* This should never happen. If it does, yell once and skip */
2943 if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING))
2947 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2948 * interrupt will either see the value pre increment or post
2949 * increment. If the interrupt happens pre increment it will have
2950 * restored the counter when it returns. We just need a barrier to
2951 * keep gcc from moving things around.
2955 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2956 size = ARRAY_SIZE(fstack->calls);
2959 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2962 nr_entries = stack_trace_save(fstack->calls, size, skip);
2965 size = nr_entries * sizeof(unsigned long);
2966 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2967 sizeof(*entry) + size, flags, pc);
2970 entry = ring_buffer_event_data(event);
2972 memcpy(&entry->caller, fstack->calls, size);
2973 entry->size = nr_entries;
2975 if (!call_filter_check_discard(call, entry, buffer, event))
2976 __buffer_unlock_commit(buffer, event);
2979 /* Again, don't let gcc optimize things here */
2981 __this_cpu_dec(ftrace_stack_reserve);
2982 preempt_enable_notrace();
2986 static inline void ftrace_trace_stack(struct trace_array *tr,
2987 struct trace_buffer *buffer,
2988 unsigned long flags,
2989 int skip, int pc, struct pt_regs *regs)
2991 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2994 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2997 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
3000 struct trace_buffer *buffer = tr->array_buffer.buffer;
3002 if (rcu_is_watching()) {
3003 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
3008 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
3009 * but if the above rcu_is_watching() failed, then the NMI
3010 * triggered someplace critical, and rcu_irq_enter() should
3011 * not be called from NMI.
3013 if (unlikely(in_nmi()))
3016 rcu_irq_enter_irqson();
3017 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
3018 rcu_irq_exit_irqson();
3022 * trace_dump_stack - record a stack back trace in the trace buffer
3023 * @skip: Number of functions to skip (helper handlers)
3025 void trace_dump_stack(int skip)
3027 unsigned long flags;
3029 if (tracing_disabled || tracing_selftest_running)
3032 local_save_flags(flags);
3034 #ifndef CONFIG_UNWINDER_ORC
3035 /* Skip 1 to skip this function. */
3038 __ftrace_trace_stack(global_trace.array_buffer.buffer,
3039 flags, skip, preempt_count(), NULL);
3041 EXPORT_SYMBOL_GPL(trace_dump_stack);
3043 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
3044 static DEFINE_PER_CPU(int, user_stack_count);
3047 ftrace_trace_userstack(struct trace_buffer *buffer, unsigned long flags, int pc)
3049 struct trace_event_call *call = &event_user_stack;
3050 struct ring_buffer_event *event;
3051 struct userstack_entry *entry;
3053 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
3057 * NMIs can not handle page faults, even with fix ups.
3058 * The save user stack can (and often does) fault.
3060 if (unlikely(in_nmi()))
3064 * prevent recursion, since the user stack tracing may
3065 * trigger other kernel events.
3068 if (__this_cpu_read(user_stack_count))
3071 __this_cpu_inc(user_stack_count);
3073 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3074 sizeof(*entry), flags, pc);
3076 goto out_drop_count;
3077 entry = ring_buffer_event_data(event);
3079 entry->tgid = current->tgid;
3080 memset(&entry->caller, 0, sizeof(entry->caller));
3082 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3083 if (!call_filter_check_discard(call, entry, buffer, event))
3084 __buffer_unlock_commit(buffer, event);
3087 __this_cpu_dec(user_stack_count);
3091 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
3092 static void ftrace_trace_userstack(struct trace_buffer *buffer,
3093 unsigned long flags, int pc)
3096 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3098 #endif /* CONFIG_STACKTRACE */
3100 /* created for use with alloc_percpu */
3101 struct trace_buffer_struct {
3103 char buffer[4][TRACE_BUF_SIZE];
3106 static struct trace_buffer_struct *trace_percpu_buffer;
3109 * Thise allows for lockless recording. If we're nested too deeply, then
3110 * this returns NULL.
3112 static char *get_trace_buf(void)
3114 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3116 if (!buffer || buffer->nesting >= 4)
3121 /* Interrupts must see nesting incremented before we use the buffer */
3123 return &buffer->buffer[buffer->nesting][0];
3126 static void put_trace_buf(void)
3128 /* Don't let the decrement of nesting leak before this */
3130 this_cpu_dec(trace_percpu_buffer->nesting);
3133 static int alloc_percpu_trace_buffer(void)
3135 struct trace_buffer_struct *buffers;
3137 buffers = alloc_percpu(struct trace_buffer_struct);
3138 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3141 trace_percpu_buffer = buffers;
3145 static int buffers_allocated;
3147 void trace_printk_init_buffers(void)
3149 if (buffers_allocated)
3152 if (alloc_percpu_trace_buffer())
3155 /* trace_printk() is for debug use only. Don't use it in production. */
3158 pr_warn("**********************************************************\n");
3159 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3161 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3163 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3164 pr_warn("** unsafe for production use. **\n");
3166 pr_warn("** If you see this message and you are not debugging **\n");
3167 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3169 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3170 pr_warn("**********************************************************\n");
3172 /* Expand the buffers to set size */
3173 tracing_update_buffers();
3175 buffers_allocated = 1;
3178 * trace_printk_init_buffers() can be called by modules.
3179 * If that happens, then we need to start cmdline recording
3180 * directly here. If the global_trace.buffer is already
3181 * allocated here, then this was called by module code.
3183 if (global_trace.array_buffer.buffer)
3184 tracing_start_cmdline_record();
3186 EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3188 void trace_printk_start_comm(void)
3190 /* Start tracing comms if trace printk is set */
3191 if (!buffers_allocated)
3193 tracing_start_cmdline_record();
3196 static void trace_printk_start_stop_comm(int enabled)
3198 if (!buffers_allocated)
3202 tracing_start_cmdline_record();
3204 tracing_stop_cmdline_record();
3208 * trace_vbprintk - write binary msg to tracing buffer
3209 * @ip: The address of the caller
3210 * @fmt: The string format to write to the buffer
3211 * @args: Arguments for @fmt
3213 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3215 struct trace_event_call *call = &event_bprint;
3216 struct ring_buffer_event *event;
3217 struct trace_buffer *buffer;
3218 struct trace_array *tr = &global_trace;
3219 struct bprint_entry *entry;
3220 unsigned long flags;
3222 int len = 0, size, pc;
3224 if (unlikely(tracing_selftest_running || tracing_disabled))
3227 /* Don't pollute graph traces with trace_vprintk internals */
3228 pause_graph_tracing();
3230 pc = preempt_count();
3231 preempt_disable_notrace();
3233 tbuffer = get_trace_buf();
3239 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3241 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3244 local_save_flags(flags);
3245 size = sizeof(*entry) + sizeof(u32) * len;
3246 buffer = tr->array_buffer.buffer;
3247 ring_buffer_nest_start(buffer);
3248 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3252 entry = ring_buffer_event_data(event);
3256 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3257 if (!call_filter_check_discard(call, entry, buffer, event)) {
3258 __buffer_unlock_commit(buffer, event);
3259 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
3263 ring_buffer_nest_end(buffer);
3268 preempt_enable_notrace();
3269 unpause_graph_tracing();
3273 EXPORT_SYMBOL_GPL(trace_vbprintk);
3277 __trace_array_vprintk(struct trace_buffer *buffer,
3278 unsigned long ip, const char *fmt, va_list args)
3280 struct trace_event_call *call = &event_print;
3281 struct ring_buffer_event *event;
3282 int len = 0, size, pc;
3283 struct print_entry *entry;
3284 unsigned long flags;
3287 if (tracing_disabled || tracing_selftest_running)
3290 /* Don't pollute graph traces with trace_vprintk internals */
3291 pause_graph_tracing();
3293 pc = preempt_count();
3294 preempt_disable_notrace();
3297 tbuffer = get_trace_buf();
3303 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3305 local_save_flags(flags);
3306 size = sizeof(*entry) + len + 1;
3307 ring_buffer_nest_start(buffer);
3308 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3312 entry = ring_buffer_event_data(event);
3315 memcpy(&entry->buf, tbuffer, len + 1);
3316 if (!call_filter_check_discard(call, entry, buffer, event)) {
3317 __buffer_unlock_commit(buffer, event);
3318 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3322 ring_buffer_nest_end(buffer);
3326 preempt_enable_notrace();
3327 unpause_graph_tracing();
3333 int trace_array_vprintk(struct trace_array *tr,
3334 unsigned long ip, const char *fmt, va_list args)
3336 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3340 int trace_array_printk(struct trace_array *tr,
3341 unsigned long ip, const char *fmt, ...)
3346 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3353 ret = trace_array_vprintk(tr, ip, fmt, ap);
3357 EXPORT_SYMBOL_GPL(trace_array_printk);
3360 int trace_array_printk_buf(struct trace_buffer *buffer,
3361 unsigned long ip, const char *fmt, ...)
3366 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3370 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3376 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3378 return trace_array_vprintk(&global_trace, ip, fmt, args);
3380 EXPORT_SYMBOL_GPL(trace_vprintk);
3382 static void trace_iterator_increment(struct trace_iterator *iter)
3384 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3388 ring_buffer_iter_advance(buf_iter);
3391 static struct trace_entry *
3392 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3393 unsigned long *lost_events)
3395 struct ring_buffer_event *event;
3396 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3399 event = ring_buffer_iter_peek(buf_iter, ts);
3401 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
3402 (unsigned long)-1 : 0;
3404 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3409 iter->ent_size = ring_buffer_event_length(event);
3410 return ring_buffer_event_data(event);
3416 static struct trace_entry *
3417 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3418 unsigned long *missing_events, u64 *ent_ts)
3420 struct trace_buffer *buffer = iter->array_buffer->buffer;
3421 struct trace_entry *ent, *next = NULL;
3422 unsigned long lost_events = 0, next_lost = 0;
3423 int cpu_file = iter->cpu_file;
3424 u64 next_ts = 0, ts;
3430 * If we are in a per_cpu trace file, don't bother by iterating over
3431 * all cpu and peek directly.
3433 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3434 if (ring_buffer_empty_cpu(buffer, cpu_file))
3436 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3438 *ent_cpu = cpu_file;
3443 for_each_tracing_cpu(cpu) {
3445 if (ring_buffer_empty_cpu(buffer, cpu))
3448 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3451 * Pick the entry with the smallest timestamp:
3453 if (ent && (!next || ts < next_ts)) {
3457 next_lost = lost_events;
3458 next_size = iter->ent_size;
3462 iter->ent_size = next_size;
3465 *ent_cpu = next_cpu;
3471 *missing_events = next_lost;
3476 #define STATIC_TEMP_BUF_SIZE 128
3477 static char static_temp_buf[STATIC_TEMP_BUF_SIZE];
3479 /* Find the next real entry, without updating the iterator itself */
3480 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3481 int *ent_cpu, u64 *ent_ts)
3483 /* __find_next_entry will reset ent_size */
3484 int ent_size = iter->ent_size;
3485 struct trace_entry *entry;
3488 * If called from ftrace_dump(), then the iter->temp buffer
3489 * will be the static_temp_buf and not created from kmalloc.
3490 * If the entry size is greater than the buffer, we can
3491 * not save it. Just return NULL in that case. This is only
3492 * used to add markers when two consecutive events' time
3493 * stamps have a large delta. See trace_print_lat_context()
3495 if (iter->temp == static_temp_buf &&
3496 STATIC_TEMP_BUF_SIZE < ent_size)
3500 * The __find_next_entry() may call peek_next_entry(), which may
3501 * call ring_buffer_peek() that may make the contents of iter->ent
3502 * undefined. Need to copy iter->ent now.
3504 if (iter->ent && iter->ent != iter->temp) {
3505 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
3506 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3508 iter->temp = kmalloc(iter->ent_size, GFP_KERNEL);
3512 memcpy(iter->temp, iter->ent, iter->ent_size);
3513 iter->temp_size = iter->ent_size;
3514 iter->ent = iter->temp;
3516 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3517 /* Put back the original ent_size */
3518 iter->ent_size = ent_size;
3523 /* Find the next real entry, and increment the iterator to the next entry */
3524 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3526 iter->ent = __find_next_entry(iter, &iter->cpu,
3527 &iter->lost_events, &iter->ts);
3530 trace_iterator_increment(iter);
3532 return iter->ent ? iter : NULL;
3535 static void trace_consume(struct trace_iterator *iter)
3537 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
3538 &iter->lost_events);
3541 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3543 struct trace_iterator *iter = m->private;
3547 WARN_ON_ONCE(iter->leftover);
3551 /* can't go backwards */
3556 ent = trace_find_next_entry_inc(iter);
3560 while (ent && iter->idx < i)
3561 ent = trace_find_next_entry_inc(iter);
3568 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3570 struct ring_buffer_event *event;
3571 struct ring_buffer_iter *buf_iter;
3572 unsigned long entries = 0;
3575 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
3577 buf_iter = trace_buffer_iter(iter, cpu);
3581 ring_buffer_iter_reset(buf_iter);
3584 * We could have the case with the max latency tracers
3585 * that a reset never took place on a cpu. This is evident
3586 * by the timestamp being before the start of the buffer.
3588 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3589 if (ts >= iter->array_buffer->time_start)
3592 ring_buffer_iter_advance(buf_iter);
3595 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
3599 * The current tracer is copied to avoid a global locking
3602 static void *s_start(struct seq_file *m, loff_t *pos)
3604 struct trace_iterator *iter = m->private;
3605 struct trace_array *tr = iter->tr;
3606 int cpu_file = iter->cpu_file;
3612 * copy the tracer to avoid using a global lock all around.
3613 * iter->trace is a copy of current_trace, the pointer to the
3614 * name may be used instead of a strcmp(), as iter->trace->name
3615 * will point to the same string as current_trace->name.
3617 mutex_lock(&trace_types_lock);
3618 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3619 *iter->trace = *tr->current_trace;
3620 mutex_unlock(&trace_types_lock);
3622 #ifdef CONFIG_TRACER_MAX_TRACE
3623 if (iter->snapshot && iter->trace->use_max_tr)
3624 return ERR_PTR(-EBUSY);
3627 if (!iter->snapshot)
3628 atomic_inc(&trace_record_taskinfo_disabled);
3630 if (*pos != iter->pos) {
3635 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3636 for_each_tracing_cpu(cpu)
3637 tracing_iter_reset(iter, cpu);
3639 tracing_iter_reset(iter, cpu_file);
3642 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3647 * If we overflowed the seq_file before, then we want
3648 * to just reuse the trace_seq buffer again.
3654 p = s_next(m, p, &l);
3658 trace_event_read_lock();
3659 trace_access_lock(cpu_file);
3663 static void s_stop(struct seq_file *m, void *p)
3665 struct trace_iterator *iter = m->private;
3667 #ifdef CONFIG_TRACER_MAX_TRACE
3668 if (iter->snapshot && iter->trace->use_max_tr)
3672 if (!iter->snapshot)
3673 atomic_dec(&trace_record_taskinfo_disabled);
3675 trace_access_unlock(iter->cpu_file);
3676 trace_event_read_unlock();
3680 get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
3681 unsigned long *entries, int cpu)
3683 unsigned long count;
3685 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3687 * If this buffer has skipped entries, then we hold all
3688 * entries for the trace and we need to ignore the
3689 * ones before the time stamp.
3691 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3692 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3693 /* total is the same as the entries */
3697 ring_buffer_overrun_cpu(buf->buffer, cpu);
3702 get_total_entries(struct array_buffer *buf,
3703 unsigned long *total, unsigned long *entries)
3711 for_each_tracing_cpu(cpu) {
3712 get_total_entries_cpu(buf, &t, &e, cpu);
3718 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
3720 unsigned long total, entries;
3725 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
3730 unsigned long trace_total_entries(struct trace_array *tr)
3732 unsigned long total, entries;
3737 get_total_entries(&tr->array_buffer, &total, &entries);
3742 static void print_lat_help_header(struct seq_file *m)
3744 seq_puts(m, "# _------=> CPU# \n"
3745 "# / _-----=> irqs-off \n"
3746 "# | / _----=> need-resched \n"
3747 "# || / _---=> hardirq/softirq \n"
3748 "# ||| / _--=> preempt-depth \n"
3750 "# cmd pid ||||| time | caller \n"
3751 "# \\ / ||||| \\ | / \n");
3754 static void print_event_info(struct array_buffer *buf, struct seq_file *m)
3756 unsigned long total;
3757 unsigned long entries;
3759 get_total_entries(buf, &total, &entries);
3760 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3761 entries, total, num_online_cpus());
3765 static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
3768 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3770 print_event_info(buf, m);
3772 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3773 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
3776 static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
3779 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3780 const char *space = " ";
3781 int prec = tgid ? 10 : 2;
3783 print_event_info(buf, m);
3785 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space);
3786 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
3787 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
3788 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
3789 seq_printf(m, "# %.*s||| / delay\n", prec, space);
3790 seq_printf(m, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID ");
3791 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | ");
3795 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3797 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3798 struct array_buffer *buf = iter->array_buffer;
3799 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3800 struct tracer *type = iter->trace;
3801 unsigned long entries;
3802 unsigned long total;
3803 const char *name = "preemption";
3807 get_total_entries(buf, &total, &entries);
3809 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3811 seq_puts(m, "# -----------------------------------"
3812 "---------------------------------\n");
3813 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3814 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3815 nsecs_to_usecs(data->saved_latency),
3819 #if defined(CONFIG_PREEMPT_NONE)
3821 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3823 #elif defined(CONFIG_PREEMPT)
3825 #elif defined(CONFIG_PREEMPT_RT)
3830 /* These are reserved for later use */
3833 seq_printf(m, " #P:%d)\n", num_online_cpus());
3837 seq_puts(m, "# -----------------\n");
3838 seq_printf(m, "# | task: %.16s-%d "
3839 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3840 data->comm, data->pid,
3841 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3842 data->policy, data->rt_priority);
3843 seq_puts(m, "# -----------------\n");
3845 if (data->critical_start) {
3846 seq_puts(m, "# => started at: ");
3847 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3848 trace_print_seq(m, &iter->seq);
3849 seq_puts(m, "\n# => ended at: ");
3850 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3851 trace_print_seq(m, &iter->seq);
3852 seq_puts(m, "\n#\n");
3858 static void test_cpu_buff_start(struct trace_iterator *iter)
3860 struct trace_seq *s = &iter->seq;
3861 struct trace_array *tr = iter->tr;
3863 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3866 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3869 if (cpumask_available(iter->started) &&
3870 cpumask_test_cpu(iter->cpu, iter->started))
3873 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
3876 if (cpumask_available(iter->started))
3877 cpumask_set_cpu(iter->cpu, iter->started);
3879 /* Don't print started cpu buffer for the first entry of the trace */
3881 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3885 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3887 struct trace_array *tr = iter->tr;
3888 struct trace_seq *s = &iter->seq;
3889 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3890 struct trace_entry *entry;
3891 struct trace_event *event;
3895 test_cpu_buff_start(iter);
3897 event = ftrace_find_event(entry->type);
3899 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3900 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3901 trace_print_lat_context(iter);
3903 trace_print_context(iter);
3906 if (trace_seq_has_overflowed(s))
3907 return TRACE_TYPE_PARTIAL_LINE;
3910 return event->funcs->trace(iter, sym_flags, event);
3912 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3914 return trace_handle_return(s);
3917 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3919 struct trace_array *tr = iter->tr;
3920 struct trace_seq *s = &iter->seq;
3921 struct trace_entry *entry;
3922 struct trace_event *event;
3926 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3927 trace_seq_printf(s, "%d %d %llu ",
3928 entry->pid, iter->cpu, iter->ts);
3930 if (trace_seq_has_overflowed(s))
3931 return TRACE_TYPE_PARTIAL_LINE;
3933 event = ftrace_find_event(entry->type);
3935 return event->funcs->raw(iter, 0, event);
3937 trace_seq_printf(s, "%d ?\n", entry->type);
3939 return trace_handle_return(s);
3942 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3944 struct trace_array *tr = iter->tr;
3945 struct trace_seq *s = &iter->seq;
3946 unsigned char newline = '\n';
3947 struct trace_entry *entry;
3948 struct trace_event *event;
3952 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3953 SEQ_PUT_HEX_FIELD(s, entry->pid);
3954 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3955 SEQ_PUT_HEX_FIELD(s, iter->ts);
3956 if (trace_seq_has_overflowed(s))
3957 return TRACE_TYPE_PARTIAL_LINE;
3960 event = ftrace_find_event(entry->type);
3962 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3963 if (ret != TRACE_TYPE_HANDLED)
3967 SEQ_PUT_FIELD(s, newline);
3969 return trace_handle_return(s);
3972 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3974 struct trace_array *tr = iter->tr;
3975 struct trace_seq *s = &iter->seq;
3976 struct trace_entry *entry;
3977 struct trace_event *event;
3981 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3982 SEQ_PUT_FIELD(s, entry->pid);
3983 SEQ_PUT_FIELD(s, iter->cpu);
3984 SEQ_PUT_FIELD(s, iter->ts);
3985 if (trace_seq_has_overflowed(s))
3986 return TRACE_TYPE_PARTIAL_LINE;
3989 event = ftrace_find_event(entry->type);
3990 return event ? event->funcs->binary(iter, 0, event) :
3994 int trace_empty(struct trace_iterator *iter)
3996 struct ring_buffer_iter *buf_iter;
3999 /* If we are looking at one CPU buffer, only check that one */
4000 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4001 cpu = iter->cpu_file;
4002 buf_iter = trace_buffer_iter(iter, cpu);
4004 if (!ring_buffer_iter_empty(buf_iter))
4007 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4013 for_each_tracing_cpu(cpu) {
4014 buf_iter = trace_buffer_iter(iter, cpu);
4016 if (!ring_buffer_iter_empty(buf_iter))
4019 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4027 /* Called with trace_event_read_lock() held. */
4028 enum print_line_t print_trace_line(struct trace_iterator *iter)
4030 struct trace_array *tr = iter->tr;
4031 unsigned long trace_flags = tr->trace_flags;
4032 enum print_line_t ret;
4034 if (iter->lost_events) {
4035 if (iter->lost_events == (unsigned long)-1)
4036 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4039 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4040 iter->cpu, iter->lost_events);
4041 if (trace_seq_has_overflowed(&iter->seq))
4042 return TRACE_TYPE_PARTIAL_LINE;
4045 if (iter->trace && iter->trace->print_line) {
4046 ret = iter->trace->print_line(iter);
4047 if (ret != TRACE_TYPE_UNHANDLED)
4051 if (iter->ent->type == TRACE_BPUTS &&
4052 trace_flags & TRACE_ITER_PRINTK &&
4053 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4054 return trace_print_bputs_msg_only(iter);
4056 if (iter->ent->type == TRACE_BPRINT &&
4057 trace_flags & TRACE_ITER_PRINTK &&
4058 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4059 return trace_print_bprintk_msg_only(iter);
4061 if (iter->ent->type == TRACE_PRINT &&
4062 trace_flags & TRACE_ITER_PRINTK &&
4063 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4064 return trace_print_printk_msg_only(iter);
4066 if (trace_flags & TRACE_ITER_BIN)
4067 return print_bin_fmt(iter);
4069 if (trace_flags & TRACE_ITER_HEX)
4070 return print_hex_fmt(iter);
4072 if (trace_flags & TRACE_ITER_RAW)
4073 return print_raw_fmt(iter);
4075 return print_trace_fmt(iter);
4078 void trace_latency_header(struct seq_file *m)
4080 struct trace_iterator *iter = m->private;
4081 struct trace_array *tr = iter->tr;
4083 /* print nothing if the buffers are empty */
4084 if (trace_empty(iter))
4087 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4088 print_trace_header(m, iter);
4090 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4091 print_lat_help_header(m);
4094 void trace_default_header(struct seq_file *m)
4096 struct trace_iterator *iter = m->private;
4097 struct trace_array *tr = iter->tr;
4098 unsigned long trace_flags = tr->trace_flags;
4100 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4103 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4104 /* print nothing if the buffers are empty */
4105 if (trace_empty(iter))
4107 print_trace_header(m, iter);
4108 if (!(trace_flags & TRACE_ITER_VERBOSE))
4109 print_lat_help_header(m);
4111 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4112 if (trace_flags & TRACE_ITER_IRQ_INFO)
4113 print_func_help_header_irq(iter->array_buffer,
4116 print_func_help_header(iter->array_buffer, m,
4122 static void test_ftrace_alive(struct seq_file *m)
4124 if (!ftrace_is_dead())
4126 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4127 "# MAY BE MISSING FUNCTION EVENTS\n");
4130 #ifdef CONFIG_TRACER_MAX_TRACE
4131 static void show_snapshot_main_help(struct seq_file *m)
4133 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4134 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4135 "# Takes a snapshot of the main buffer.\n"
4136 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4137 "# (Doesn't have to be '2' works with any number that\n"
4138 "# is not a '0' or '1')\n");
4141 static void show_snapshot_percpu_help(struct seq_file *m)
4143 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4144 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4145 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4146 "# Takes a snapshot of the main buffer for this cpu.\n");
4148 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4149 "# Must use main snapshot file to allocate.\n");
4151 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4152 "# (Doesn't have to be '2' works with any number that\n"
4153 "# is not a '0' or '1')\n");
4156 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4158 if (iter->tr->allocated_snapshot)
4159 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4161 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4163 seq_puts(m, "# Snapshot commands:\n");
4164 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4165 show_snapshot_main_help(m);
4167 show_snapshot_percpu_help(m);
4170 /* Should never be called */
4171 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4174 static int s_show(struct seq_file *m, void *v)
4176 struct trace_iterator *iter = v;
4179 if (iter->ent == NULL) {
4181 seq_printf(m, "# tracer: %s\n", iter->trace->name);
4183 test_ftrace_alive(m);
4185 if (iter->snapshot && trace_empty(iter))
4186 print_snapshot_help(m, iter);
4187 else if (iter->trace && iter->trace->print_header)
4188 iter->trace->print_header(m);
4190 trace_default_header(m);
4192 } else if (iter->leftover) {
4194 * If we filled the seq_file buffer earlier, we
4195 * want to just show it now.
4197 ret = trace_print_seq(m, &iter->seq);
4199 /* ret should this time be zero, but you never know */
4200 iter->leftover = ret;
4203 print_trace_line(iter);
4204 ret = trace_print_seq(m, &iter->seq);
4206 * If we overflow the seq_file buffer, then it will
4207 * ask us for this data again at start up.
4209 * ret is 0 if seq_file write succeeded.
4212 iter->leftover = ret;
4219 * Should be used after trace_array_get(), trace_types_lock
4220 * ensures that i_cdev was already initialized.
4222 static inline int tracing_get_cpu(struct inode *inode)
4224 if (inode->i_cdev) /* See trace_create_cpu_file() */
4225 return (long)inode->i_cdev - 1;
4226 return RING_BUFFER_ALL_CPUS;
4229 static const struct seq_operations tracer_seq_ops = {
4236 static struct trace_iterator *
4237 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4239 struct trace_array *tr = inode->i_private;
4240 struct trace_iterator *iter;
4243 if (tracing_disabled)
4244 return ERR_PTR(-ENODEV);
4246 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4248 return ERR_PTR(-ENOMEM);
4250 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4252 if (!iter->buffer_iter)
4256 * trace_find_next_entry() may need to save off iter->ent.
4257 * It will place it into the iter->temp buffer. As most
4258 * events are less than 128, allocate a buffer of that size.
4259 * If one is greater, then trace_find_next_entry() will
4260 * allocate a new buffer to adjust for the bigger iter->ent.
4261 * It's not critical if it fails to get allocated here.
4263 iter->temp = kmalloc(128, GFP_KERNEL);
4265 iter->temp_size = 128;
4268 * We make a copy of the current tracer to avoid concurrent
4269 * changes on it while we are reading.
4271 mutex_lock(&trace_types_lock);
4272 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4276 *iter->trace = *tr->current_trace;
4278 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4283 #ifdef CONFIG_TRACER_MAX_TRACE
4284 /* Currently only the top directory has a snapshot */
4285 if (tr->current_trace->print_max || snapshot)
4286 iter->array_buffer = &tr->max_buffer;
4289 iter->array_buffer = &tr->array_buffer;
4290 iter->snapshot = snapshot;
4292 iter->cpu_file = tracing_get_cpu(inode);
4293 mutex_init(&iter->mutex);
4295 /* Notify the tracer early; before we stop tracing. */
4296 if (iter->trace->open)
4297 iter->trace->open(iter);
4299 /* Annotate start of buffers if we had overruns */
4300 if (ring_buffer_overruns(iter->array_buffer->buffer))
4301 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4303 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4304 if (trace_clocks[tr->clock_id].in_ns)
4305 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4308 * If pause-on-trace is enabled, then stop the trace while
4309 * dumping, unless this is the "snapshot" file
4311 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4312 tracing_stop_tr(tr);
4314 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4315 for_each_tracing_cpu(cpu) {
4316 iter->buffer_iter[cpu] =
4317 ring_buffer_read_prepare(iter->array_buffer->buffer,
4320 ring_buffer_read_prepare_sync();
4321 for_each_tracing_cpu(cpu) {
4322 ring_buffer_read_start(iter->buffer_iter[cpu]);
4323 tracing_iter_reset(iter, cpu);
4326 cpu = iter->cpu_file;
4327 iter->buffer_iter[cpu] =
4328 ring_buffer_read_prepare(iter->array_buffer->buffer,
4330 ring_buffer_read_prepare_sync();
4331 ring_buffer_read_start(iter->buffer_iter[cpu]);
4332 tracing_iter_reset(iter, cpu);
4335 mutex_unlock(&trace_types_lock);
4340 mutex_unlock(&trace_types_lock);
4343 kfree(iter->buffer_iter);
4345 seq_release_private(inode, file);
4346 return ERR_PTR(-ENOMEM);
4349 int tracing_open_generic(struct inode *inode, struct file *filp)
4353 ret = tracing_check_open_get_tr(NULL);
4357 filp->private_data = inode->i_private;
4361 bool tracing_is_disabled(void)
4363 return (tracing_disabled) ? true: false;
4367 * Open and update trace_array ref count.
4368 * Must have the current trace_array passed to it.
4370 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4372 struct trace_array *tr = inode->i_private;
4375 ret = tracing_check_open_get_tr(tr);
4379 filp->private_data = inode->i_private;
4384 static int tracing_release(struct inode *inode, struct file *file)
4386 struct trace_array *tr = inode->i_private;
4387 struct seq_file *m = file->private_data;
4388 struct trace_iterator *iter;
4391 if (!(file->f_mode & FMODE_READ)) {
4392 trace_array_put(tr);
4396 /* Writes do not use seq_file */
4398 mutex_lock(&trace_types_lock);
4400 for_each_tracing_cpu(cpu) {
4401 if (iter->buffer_iter[cpu])
4402 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4405 if (iter->trace && iter->trace->close)
4406 iter->trace->close(iter);
4408 if (!iter->snapshot && tr->stop_count)
4409 /* reenable tracing if it was previously enabled */
4410 tracing_start_tr(tr);
4412 __trace_array_put(tr);
4414 mutex_unlock(&trace_types_lock);
4416 mutex_destroy(&iter->mutex);
4417 free_cpumask_var(iter->started);
4420 kfree(iter->buffer_iter);
4421 seq_release_private(inode, file);
4426 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4428 struct trace_array *tr = inode->i_private;
4430 trace_array_put(tr);
4434 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4436 struct trace_array *tr = inode->i_private;
4438 trace_array_put(tr);
4440 return single_release(inode, file);
4443 static int tracing_open(struct inode *inode, struct file *file)
4445 struct trace_array *tr = inode->i_private;
4446 struct trace_iterator *iter;
4449 ret = tracing_check_open_get_tr(tr);
4453 /* If this file was open for write, then erase contents */
4454 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4455 int cpu = tracing_get_cpu(inode);
4456 struct array_buffer *trace_buf = &tr->array_buffer;
4458 #ifdef CONFIG_TRACER_MAX_TRACE
4459 if (tr->current_trace->print_max)
4460 trace_buf = &tr->max_buffer;
4463 if (cpu == RING_BUFFER_ALL_CPUS)
4464 tracing_reset_online_cpus(trace_buf);
4466 tracing_reset_cpu(trace_buf, cpu);
4469 if (file->f_mode & FMODE_READ) {
4470 iter = __tracing_open(inode, file, false);
4472 ret = PTR_ERR(iter);
4473 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4474 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4478 trace_array_put(tr);
4484 * Some tracers are not suitable for instance buffers.
4485 * A tracer is always available for the global array (toplevel)
4486 * or if it explicitly states that it is.
4489 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4491 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4494 /* Find the next tracer that this trace array may use */
4495 static struct tracer *
4496 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4498 while (t && !trace_ok_for_array(t, tr))
4505 t_next(struct seq_file *m, void *v, loff_t *pos)
4507 struct trace_array *tr = m->private;
4508 struct tracer *t = v;
4513 t = get_tracer_for_array(tr, t->next);
4518 static void *t_start(struct seq_file *m, loff_t *pos)
4520 struct trace_array *tr = m->private;
4524 mutex_lock(&trace_types_lock);
4526 t = get_tracer_for_array(tr, trace_types);
4527 for (; t && l < *pos; t = t_next(m, t, &l))
4533 static void t_stop(struct seq_file *m, void *p)
4535 mutex_unlock(&trace_types_lock);
4538 static int t_show(struct seq_file *m, void *v)
4540 struct tracer *t = v;
4545 seq_puts(m, t->name);
4554 static const struct seq_operations show_traces_seq_ops = {
4561 static int show_traces_open(struct inode *inode, struct file *file)
4563 struct trace_array *tr = inode->i_private;
4567 ret = tracing_check_open_get_tr(tr);
4571 ret = seq_open(file, &show_traces_seq_ops);
4573 trace_array_put(tr);
4577 m = file->private_data;
4583 static int show_traces_release(struct inode *inode, struct file *file)
4585 struct trace_array *tr = inode->i_private;
4587 trace_array_put(tr);
4588 return seq_release(inode, file);
4592 tracing_write_stub(struct file *filp, const char __user *ubuf,
4593 size_t count, loff_t *ppos)
4598 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4602 if (file->f_mode & FMODE_READ)
4603 ret = seq_lseek(file, offset, whence);
4605 file->f_pos = ret = 0;
4610 static const struct file_operations tracing_fops = {
4611 .open = tracing_open,
4613 .write = tracing_write_stub,
4614 .llseek = tracing_lseek,
4615 .release = tracing_release,
4618 static const struct file_operations show_traces_fops = {
4619 .open = show_traces_open,
4621 .llseek = seq_lseek,
4622 .release = show_traces_release,
4626 tracing_cpumask_read(struct file *filp, char __user *ubuf,
4627 size_t count, loff_t *ppos)
4629 struct trace_array *tr = file_inode(filp)->i_private;
4633 len = snprintf(NULL, 0, "%*pb\n",
4634 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4635 mask_str = kmalloc(len, GFP_KERNEL);
4639 len = snprintf(mask_str, len, "%*pb\n",
4640 cpumask_pr_args(tr->tracing_cpumask));
4645 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4653 int tracing_set_cpumask(struct trace_array *tr,
4654 cpumask_var_t tracing_cpumask_new)
4661 local_irq_disable();
4662 arch_spin_lock(&tr->max_lock);
4663 for_each_tracing_cpu(cpu) {
4665 * Increase/decrease the disabled counter if we are
4666 * about to flip a bit in the cpumask:
4668 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4669 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4670 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4671 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
4673 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4674 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4675 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4676 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
4679 arch_spin_unlock(&tr->max_lock);
4682 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4688 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4689 size_t count, loff_t *ppos)
4691 struct trace_array *tr = file_inode(filp)->i_private;
4692 cpumask_var_t tracing_cpumask_new;
4695 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4698 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4702 err = tracing_set_cpumask(tr, tracing_cpumask_new);
4706 free_cpumask_var(tracing_cpumask_new);
4711 free_cpumask_var(tracing_cpumask_new);
4716 static const struct file_operations tracing_cpumask_fops = {
4717 .open = tracing_open_generic_tr,
4718 .read = tracing_cpumask_read,
4719 .write = tracing_cpumask_write,
4720 .release = tracing_release_generic_tr,
4721 .llseek = generic_file_llseek,
4724 static int tracing_trace_options_show(struct seq_file *m, void *v)
4726 struct tracer_opt *trace_opts;
4727 struct trace_array *tr = m->private;
4731 mutex_lock(&trace_types_lock);
4732 tracer_flags = tr->current_trace->flags->val;
4733 trace_opts = tr->current_trace->flags->opts;
4735 for (i = 0; trace_options[i]; i++) {
4736 if (tr->trace_flags & (1 << i))
4737 seq_printf(m, "%s\n", trace_options[i]);
4739 seq_printf(m, "no%s\n", trace_options[i]);
4742 for (i = 0; trace_opts[i].name; i++) {
4743 if (tracer_flags & trace_opts[i].bit)
4744 seq_printf(m, "%s\n", trace_opts[i].name);
4746 seq_printf(m, "no%s\n", trace_opts[i].name);
4748 mutex_unlock(&trace_types_lock);
4753 static int __set_tracer_option(struct trace_array *tr,
4754 struct tracer_flags *tracer_flags,
4755 struct tracer_opt *opts, int neg)
4757 struct tracer *trace = tracer_flags->trace;
4760 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4765 tracer_flags->val &= ~opts->bit;
4767 tracer_flags->val |= opts->bit;
4771 /* Try to assign a tracer specific option */
4772 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4774 struct tracer *trace = tr->current_trace;
4775 struct tracer_flags *tracer_flags = trace->flags;
4776 struct tracer_opt *opts = NULL;
4779 for (i = 0; tracer_flags->opts[i].name; i++) {
4780 opts = &tracer_flags->opts[i];
4782 if (strcmp(cmp, opts->name) == 0)
4783 return __set_tracer_option(tr, trace->flags, opts, neg);
4789 /* Some tracers require overwrite to stay enabled */
4790 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4792 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4798 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4800 if ((mask == TRACE_ITER_RECORD_TGID) ||
4801 (mask == TRACE_ITER_RECORD_CMD))
4802 lockdep_assert_held(&event_mutex);
4804 /* do nothing if flag is already set */
4805 if (!!(tr->trace_flags & mask) == !!enabled)
4808 /* Give the tracer a chance to approve the change */
4809 if (tr->current_trace->flag_changed)
4810 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4814 tr->trace_flags |= mask;
4816 tr->trace_flags &= ~mask;
4818 if (mask == TRACE_ITER_RECORD_CMD)
4819 trace_event_enable_cmd_record(enabled);
4821 if (mask == TRACE_ITER_RECORD_TGID) {
4823 tgid_map = kvcalloc(PID_MAX_DEFAULT + 1,
4827 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4831 trace_event_enable_tgid_record(enabled);
4834 if (mask == TRACE_ITER_EVENT_FORK)
4835 trace_event_follow_fork(tr, enabled);
4837 if (mask == TRACE_ITER_FUNC_FORK)
4838 ftrace_pid_follow_fork(tr, enabled);
4840 if (mask == TRACE_ITER_OVERWRITE) {
4841 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
4842 #ifdef CONFIG_TRACER_MAX_TRACE
4843 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4847 if (mask == TRACE_ITER_PRINTK) {
4848 trace_printk_start_stop_comm(enabled);
4849 trace_printk_control(enabled);
4855 int trace_set_options(struct trace_array *tr, char *option)
4860 size_t orig_len = strlen(option);
4863 cmp = strstrip(option);
4865 len = str_has_prefix(cmp, "no");
4871 mutex_lock(&event_mutex);
4872 mutex_lock(&trace_types_lock);
4874 ret = match_string(trace_options, -1, cmp);
4875 /* If no option could be set, test the specific tracer options */
4877 ret = set_tracer_option(tr, cmp, neg);
4879 ret = set_tracer_flag(tr, 1 << ret, !neg);
4881 mutex_unlock(&trace_types_lock);
4882 mutex_unlock(&event_mutex);
4885 * If the first trailing whitespace is replaced with '\0' by strstrip,
4886 * turn it back into a space.
4888 if (orig_len > strlen(option))
4889 option[strlen(option)] = ' ';
4894 static void __init apply_trace_boot_options(void)
4896 char *buf = trace_boot_options_buf;
4900 option = strsep(&buf, ",");
4906 trace_set_options(&global_trace, option);
4908 /* Put back the comma to allow this to be called again */
4915 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4916 size_t cnt, loff_t *ppos)
4918 struct seq_file *m = filp->private_data;
4919 struct trace_array *tr = m->private;
4923 if (cnt >= sizeof(buf))
4926 if (copy_from_user(buf, ubuf, cnt))
4931 ret = trace_set_options(tr, buf);
4940 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4942 struct trace_array *tr = inode->i_private;
4945 ret = tracing_check_open_get_tr(tr);
4949 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4951 trace_array_put(tr);
4956 static const struct file_operations tracing_iter_fops = {
4957 .open = tracing_trace_options_open,
4959 .llseek = seq_lseek,
4960 .release = tracing_single_release_tr,
4961 .write = tracing_trace_options_write,
4964 static const char readme_msg[] =
4965 "tracing mini-HOWTO:\n\n"
4966 "# echo 0 > tracing_on : quick way to disable tracing\n"
4967 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4968 " Important files:\n"
4969 " trace\t\t\t- The static contents of the buffer\n"
4970 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4971 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4972 " current_tracer\t- function and latency tracers\n"
4973 " available_tracers\t- list of configured tracers for current_tracer\n"
4974 " error_log\t- error log for failed commands (that support it)\n"
4975 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4976 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4977 " trace_clock\t\t-change the clock used to order events\n"
4978 " local: Per cpu clock but may not be synced across CPUs\n"
4979 " global: Synced across CPUs but slows tracing down.\n"
4980 " counter: Not a clock, but just an increment\n"
4981 " uptime: Jiffy counter from time of boot\n"
4982 " perf: Same clock that perf events use\n"
4983 #ifdef CONFIG_X86_64
4984 " x86-tsc: TSC cycle counter\n"
4986 "\n timestamp_mode\t-view the mode used to timestamp events\n"
4987 " delta: Delta difference against a buffer-wide timestamp\n"
4988 " absolute: Absolute (standalone) timestamp\n"
4989 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4990 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4991 " tracing_cpumask\t- Limit which CPUs to trace\n"
4992 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4993 "\t\t\t Remove sub-buffer with rmdir\n"
4994 " trace_options\t\t- Set format or modify how tracing happens\n"
4995 "\t\t\t Disable an option by prefixing 'no' to the\n"
4996 "\t\t\t option name\n"
4997 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4998 #ifdef CONFIG_DYNAMIC_FTRACE
4999 "\n available_filter_functions - list of functions that can be filtered on\n"
5000 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5001 "\t\t\t functions\n"
5002 "\t accepts: func_full_name or glob-matching-pattern\n"
5003 "\t modules: Can select a group via module\n"
5004 "\t Format: :mod:<module-name>\n"
5005 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
5006 "\t triggers: a command to perform when function is hit\n"
5007 "\t Format: <function>:<trigger>[:count]\n"
5008 "\t trigger: traceon, traceoff\n"
5009 "\t\t enable_event:<system>:<event>\n"
5010 "\t\t disable_event:<system>:<event>\n"
5011 #ifdef CONFIG_STACKTRACE
5014 #ifdef CONFIG_TRACER_SNAPSHOT
5019 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5020 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5021 "\t The first one will disable tracing every time do_fault is hit\n"
5022 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
5023 "\t The first time do trap is hit and it disables tracing, the\n"
5024 "\t counter will decrement to 2. If tracing is already disabled,\n"
5025 "\t the counter will not decrement. It only decrements when the\n"
5026 "\t trigger did work\n"
5027 "\t To remove trigger without count:\n"
5028 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5029 "\t To remove trigger with a count:\n"
5030 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5031 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5032 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5033 "\t modules: Can select a group via module command :mod:\n"
5034 "\t Does not accept triggers\n"
5035 #endif /* CONFIG_DYNAMIC_FTRACE */
5036 #ifdef CONFIG_FUNCTION_TRACER
5037 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5039 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5042 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5043 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5044 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5045 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5047 #ifdef CONFIG_TRACER_SNAPSHOT
5048 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5049 "\t\t\t snapshot buffer. Read the contents for more\n"
5050 "\t\t\t information\n"
5052 #ifdef CONFIG_STACK_TRACER
5053 " stack_trace\t\t- Shows the max stack trace when active\n"
5054 " stack_max_size\t- Shows current max stack size that was traced\n"
5055 "\t\t\t Write into this file to reset the max size (trigger a\n"
5056 "\t\t\t new trace)\n"
5057 #ifdef CONFIG_DYNAMIC_FTRACE
5058 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5061 #endif /* CONFIG_STACK_TRACER */
5062 #ifdef CONFIG_DYNAMIC_EVENTS
5063 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5064 "\t\t\t Write into this file to define/undefine new trace events.\n"
5066 #ifdef CONFIG_KPROBE_EVENTS
5067 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5068 "\t\t\t Write into this file to define/undefine new trace events.\n"
5070 #ifdef CONFIG_UPROBE_EVENTS
5071 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5072 "\t\t\t Write into this file to define/undefine new trace events.\n"
5074 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5075 "\t accepts: event-definitions (one definition per line)\n"
5076 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
5077 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
5078 #ifdef CONFIG_HIST_TRIGGERS
5079 "\t s:[synthetic/]<event> <field> [<field>]\n"
5081 "\t -:[<group>/]<event>\n"
5082 #ifdef CONFIG_KPROBE_EVENTS
5083 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5084 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5086 #ifdef CONFIG_UPROBE_EVENTS
5087 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
5089 "\t args: <name>=fetcharg[:type]\n"
5090 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
5091 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5092 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5094 "\t $stack<index>, $stack, $retval, $comm,\n"
5096 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5097 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
5098 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5099 "\t <type>\\[<array-size>\\]\n"
5100 #ifdef CONFIG_HIST_TRIGGERS
5101 "\t field: <stype> <name>;\n"
5102 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5103 "\t [unsigned] char/int/long\n"
5106 " events/\t\t- Directory containing all trace event subsystems:\n"
5107 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5108 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5109 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5111 " filter\t\t- If set, only events passing filter are traced\n"
5112 " events/<system>/<event>/\t- Directory containing control files for\n"
5114 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5115 " filter\t\t- If set, only events passing filter are traced\n"
5116 " trigger\t\t- If set, a command to perform when event is hit\n"
5117 "\t Format: <trigger>[:count][if <filter>]\n"
5118 "\t trigger: traceon, traceoff\n"
5119 "\t enable_event:<system>:<event>\n"
5120 "\t disable_event:<system>:<event>\n"
5121 #ifdef CONFIG_HIST_TRIGGERS
5122 "\t enable_hist:<system>:<event>\n"
5123 "\t disable_hist:<system>:<event>\n"
5125 #ifdef CONFIG_STACKTRACE
5128 #ifdef CONFIG_TRACER_SNAPSHOT
5131 #ifdef CONFIG_HIST_TRIGGERS
5132 "\t\t hist (see below)\n"
5134 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
5135 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5136 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5137 "\t events/block/block_unplug/trigger\n"
5138 "\t The first disables tracing every time block_unplug is hit.\n"
5139 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
5140 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
5141 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5142 "\t Like function triggers, the counter is only decremented if it\n"
5143 "\t enabled or disabled tracing.\n"
5144 "\t To remove a trigger without a count:\n"
5145 "\t echo '!<trigger> > <system>/<event>/trigger\n"
5146 "\t To remove a trigger with a count:\n"
5147 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5148 "\t Filters can be ignored when removing a trigger.\n"
5149 #ifdef CONFIG_HIST_TRIGGERS
5150 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5151 "\t Format: hist:keys=<field1[,field2,...]>\n"
5152 "\t [:values=<field1[,field2,...]>]\n"
5153 "\t [:sort=<field1[,field2,...]>]\n"
5154 "\t [:size=#entries]\n"
5155 "\t [:pause][:continue][:clear]\n"
5156 "\t [:name=histname1]\n"
5157 "\t [:<handler>.<action>]\n"
5158 "\t [if <filter>]\n\n"
5159 "\t When a matching event is hit, an entry is added to a hash\n"
5160 "\t table using the key(s) and value(s) named, and the value of a\n"
5161 "\t sum called 'hitcount' is incremented. Keys and values\n"
5162 "\t correspond to fields in the event's format description. Keys\n"
5163 "\t can be any field, or the special string 'stacktrace'.\n"
5164 "\t Compound keys consisting of up to two fields can be specified\n"
5165 "\t by the 'keys' keyword. Values must correspond to numeric\n"
5166 "\t fields. Sort keys consisting of up to two fields can be\n"
5167 "\t specified using the 'sort' keyword. The sort direction can\n"
5168 "\t be modified by appending '.descending' or '.ascending' to a\n"
5169 "\t sort field. The 'size' parameter can be used to specify more\n"
5170 "\t or fewer than the default 2048 entries for the hashtable size.\n"
5171 "\t If a hist trigger is given a name using the 'name' parameter,\n"
5172 "\t its histogram data will be shared with other triggers of the\n"
5173 "\t same name, and trigger hits will update this common data.\n\n"
5174 "\t Reading the 'hist' file for the event will dump the hash\n"
5175 "\t table in its entirety to stdout. If there are multiple hist\n"
5176 "\t triggers attached to an event, there will be a table for each\n"
5177 "\t trigger in the output. The table displayed for a named\n"
5178 "\t trigger will be the same as any other instance having the\n"
5179 "\t same name. The default format used to display a given field\n"
5180 "\t can be modified by appending any of the following modifiers\n"
5181 "\t to the field name, as applicable:\n\n"
5182 "\t .hex display a number as a hex value\n"
5183 "\t .sym display an address as a symbol\n"
5184 "\t .sym-offset display an address as a symbol and offset\n"
5185 "\t .execname display a common_pid as a program name\n"
5186 "\t .syscall display a syscall id as a syscall name\n"
5187 "\t .log2 display log2 value rather than raw number\n"
5188 "\t .usecs display a common_timestamp in microseconds\n\n"
5189 "\t The 'pause' parameter can be used to pause an existing hist\n"
5190 "\t trigger or to start a hist trigger but not log any events\n"
5191 "\t until told to do so. 'continue' can be used to start or\n"
5192 "\t restart a paused hist trigger.\n\n"
5193 "\t The 'clear' parameter will clear the contents of a running\n"
5194 "\t hist trigger and leave its current paused/active state\n"
5196 "\t The enable_hist and disable_hist triggers can be used to\n"
5197 "\t have one event conditionally start and stop another event's\n"
5198 "\t already-attached hist trigger. The syntax is analogous to\n"
5199 "\t the enable_event and disable_event triggers.\n\n"
5200 "\t Hist trigger handlers and actions are executed whenever a\n"
5201 "\t a histogram entry is added or updated. They take the form:\n\n"
5202 "\t <handler>.<action>\n\n"
5203 "\t The available handlers are:\n\n"
5204 "\t onmatch(matching.event) - invoke on addition or update\n"
5205 "\t onmax(var) - invoke if var exceeds current max\n"
5206 "\t onchange(var) - invoke action if var changes\n\n"
5207 "\t The available actions are:\n\n"
5208 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5209 "\t save(field,...) - save current event fields\n"
5210 #ifdef CONFIG_TRACER_SNAPSHOT
5211 "\t snapshot() - snapshot the trace buffer\n"
5217 tracing_readme_read(struct file *filp, char __user *ubuf,
5218 size_t cnt, loff_t *ppos)
5220 return simple_read_from_buffer(ubuf, cnt, ppos,
5221 readme_msg, strlen(readme_msg));
5224 static const struct file_operations tracing_readme_fops = {
5225 .open = tracing_open_generic,
5226 .read = tracing_readme_read,
5227 .llseek = generic_file_llseek,
5230 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5234 if (*pos || m->count)
5239 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
5240 if (trace_find_tgid(*ptr))
5247 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5257 v = saved_tgids_next(m, v, &l);
5265 static void saved_tgids_stop(struct seq_file *m, void *v)
5269 static int saved_tgids_show(struct seq_file *m, void *v)
5271 int pid = (int *)v - tgid_map;
5273 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5277 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5278 .start = saved_tgids_start,
5279 .stop = saved_tgids_stop,
5280 .next = saved_tgids_next,
5281 .show = saved_tgids_show,
5284 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5288 ret = tracing_check_open_get_tr(NULL);
5292 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5296 static const struct file_operations tracing_saved_tgids_fops = {
5297 .open = tracing_saved_tgids_open,
5299 .llseek = seq_lseek,
5300 .release = seq_release,
5303 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5305 unsigned int *ptr = v;
5307 if (*pos || m->count)
5312 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5314 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5323 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5329 arch_spin_lock(&trace_cmdline_lock);
5331 v = &savedcmd->map_cmdline_to_pid[0];
5333 v = saved_cmdlines_next(m, v, &l);
5341 static void saved_cmdlines_stop(struct seq_file *m, void *v)
5343 arch_spin_unlock(&trace_cmdline_lock);
5347 static int saved_cmdlines_show(struct seq_file *m, void *v)
5349 char buf[TASK_COMM_LEN];
5350 unsigned int *pid = v;
5352 __trace_find_cmdline(*pid, buf);
5353 seq_printf(m, "%d %s\n", *pid, buf);
5357 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5358 .start = saved_cmdlines_start,
5359 .next = saved_cmdlines_next,
5360 .stop = saved_cmdlines_stop,
5361 .show = saved_cmdlines_show,
5364 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5368 ret = tracing_check_open_get_tr(NULL);
5372 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5375 static const struct file_operations tracing_saved_cmdlines_fops = {
5376 .open = tracing_saved_cmdlines_open,
5378 .llseek = seq_lseek,
5379 .release = seq_release,
5383 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5384 size_t cnt, loff_t *ppos)
5389 arch_spin_lock(&trace_cmdline_lock);
5390 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5391 arch_spin_unlock(&trace_cmdline_lock);
5393 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5396 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5398 kfree(s->saved_cmdlines);
5399 kfree(s->map_cmdline_to_pid);
5403 static int tracing_resize_saved_cmdlines(unsigned int val)
5405 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5407 s = kmalloc(sizeof(*s), GFP_KERNEL);
5411 if (allocate_cmdlines_buffer(val, s) < 0) {
5416 arch_spin_lock(&trace_cmdline_lock);
5417 savedcmd_temp = savedcmd;
5419 arch_spin_unlock(&trace_cmdline_lock);
5420 free_saved_cmdlines_buffer(savedcmd_temp);
5426 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5427 size_t cnt, loff_t *ppos)
5432 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5436 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5437 if (!val || val > PID_MAX_DEFAULT)
5440 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5449 static const struct file_operations tracing_saved_cmdlines_size_fops = {
5450 .open = tracing_open_generic,
5451 .read = tracing_saved_cmdlines_size_read,
5452 .write = tracing_saved_cmdlines_size_write,
5455 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5456 static union trace_eval_map_item *
5457 update_eval_map(union trace_eval_map_item *ptr)
5459 if (!ptr->map.eval_string) {
5460 if (ptr->tail.next) {
5461 ptr = ptr->tail.next;
5462 /* Set ptr to the next real item (skip head) */
5470 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5472 union trace_eval_map_item *ptr = v;
5475 * Paranoid! If ptr points to end, we don't want to increment past it.
5476 * This really should never happen.
5479 ptr = update_eval_map(ptr);
5480 if (WARN_ON_ONCE(!ptr))
5484 ptr = update_eval_map(ptr);
5489 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5491 union trace_eval_map_item *v;
5494 mutex_lock(&trace_eval_mutex);
5496 v = trace_eval_maps;
5500 while (v && l < *pos) {
5501 v = eval_map_next(m, v, &l);
5507 static void eval_map_stop(struct seq_file *m, void *v)
5509 mutex_unlock(&trace_eval_mutex);
5512 static int eval_map_show(struct seq_file *m, void *v)
5514 union trace_eval_map_item *ptr = v;
5516 seq_printf(m, "%s %ld (%s)\n",
5517 ptr->map.eval_string, ptr->map.eval_value,
5523 static const struct seq_operations tracing_eval_map_seq_ops = {
5524 .start = eval_map_start,
5525 .next = eval_map_next,
5526 .stop = eval_map_stop,
5527 .show = eval_map_show,
5530 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5534 ret = tracing_check_open_get_tr(NULL);
5538 return seq_open(filp, &tracing_eval_map_seq_ops);
5541 static const struct file_operations tracing_eval_map_fops = {
5542 .open = tracing_eval_map_open,
5544 .llseek = seq_lseek,
5545 .release = seq_release,
5548 static inline union trace_eval_map_item *
5549 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5551 /* Return tail of array given the head */
5552 return ptr + ptr->head.length + 1;
5556 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5559 struct trace_eval_map **stop;
5560 struct trace_eval_map **map;
5561 union trace_eval_map_item *map_array;
5562 union trace_eval_map_item *ptr;
5567 * The trace_eval_maps contains the map plus a head and tail item,
5568 * where the head holds the module and length of array, and the
5569 * tail holds a pointer to the next list.
5571 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5573 pr_warn("Unable to allocate trace eval mapping\n");
5577 mutex_lock(&trace_eval_mutex);
5579 if (!trace_eval_maps)
5580 trace_eval_maps = map_array;
5582 ptr = trace_eval_maps;
5584 ptr = trace_eval_jmp_to_tail(ptr);
5585 if (!ptr->tail.next)
5587 ptr = ptr->tail.next;
5590 ptr->tail.next = map_array;
5592 map_array->head.mod = mod;
5593 map_array->head.length = len;
5596 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5597 map_array->map = **map;
5600 memset(map_array, 0, sizeof(*map_array));
5602 mutex_unlock(&trace_eval_mutex);
5605 static void trace_create_eval_file(struct dentry *d_tracer)
5607 trace_create_file("eval_map", 0444, d_tracer,
5608 NULL, &tracing_eval_map_fops);
5611 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5612 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5613 static inline void trace_insert_eval_map_file(struct module *mod,
5614 struct trace_eval_map **start, int len) { }
5615 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5617 static void trace_insert_eval_map(struct module *mod,
5618 struct trace_eval_map **start, int len)
5620 struct trace_eval_map **map;
5627 trace_event_eval_update(map, len);
5629 trace_insert_eval_map_file(mod, start, len);
5633 tracing_set_trace_read(struct file *filp, char __user *ubuf,
5634 size_t cnt, loff_t *ppos)
5636 struct trace_array *tr = filp->private_data;
5637 char buf[MAX_TRACER_SIZE+2];
5640 mutex_lock(&trace_types_lock);
5641 r = sprintf(buf, "%s\n", tr->current_trace->name);
5642 mutex_unlock(&trace_types_lock);
5644 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5647 int tracer_init(struct tracer *t, struct trace_array *tr)
5649 tracing_reset_online_cpus(&tr->array_buffer);
5653 static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
5657 for_each_tracing_cpu(cpu)
5658 per_cpu_ptr(buf->data, cpu)->entries = val;
5661 #ifdef CONFIG_TRACER_MAX_TRACE
5662 /* resize @tr's buffer to the size of @size_tr's entries */
5663 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
5664 struct array_buffer *size_buf, int cpu_id)
5668 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5669 for_each_tracing_cpu(cpu) {
5670 ret = ring_buffer_resize(trace_buf->buffer,
5671 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5674 per_cpu_ptr(trace_buf->data, cpu)->entries =
5675 per_cpu_ptr(size_buf->data, cpu)->entries;
5678 ret = ring_buffer_resize(trace_buf->buffer,
5679 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5681 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5682 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5687 #endif /* CONFIG_TRACER_MAX_TRACE */
5689 static int __tracing_resize_ring_buffer(struct trace_array *tr,
5690 unsigned long size, int cpu)
5695 * If kernel or user changes the size of the ring buffer
5696 * we use the size that was given, and we can forget about
5697 * expanding it later.
5699 ring_buffer_expanded = true;
5701 /* May be called before buffers are initialized */
5702 if (!tr->array_buffer.buffer)
5705 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
5709 #ifdef CONFIG_TRACER_MAX_TRACE
5710 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5711 !tr->current_trace->use_max_tr)
5714 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5716 int r = resize_buffer_duplicate_size(&tr->array_buffer,
5717 &tr->array_buffer, cpu);
5720 * AARGH! We are left with different
5721 * size max buffer!!!!
5722 * The max buffer is our "snapshot" buffer.
5723 * When a tracer needs a snapshot (one of the
5724 * latency tracers), it swaps the max buffer
5725 * with the saved snap shot. We succeeded to
5726 * update the size of the main buffer, but failed to
5727 * update the size of the max buffer. But when we tried
5728 * to reset the main buffer to the original size, we
5729 * failed there too. This is very unlikely to
5730 * happen, but if it does, warn and kill all
5734 tracing_disabled = 1;
5739 if (cpu == RING_BUFFER_ALL_CPUS)
5740 set_buffer_entries(&tr->max_buffer, size);
5742 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5745 #endif /* CONFIG_TRACER_MAX_TRACE */
5747 if (cpu == RING_BUFFER_ALL_CPUS)
5748 set_buffer_entries(&tr->array_buffer, size);
5750 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
5755 ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5756 unsigned long size, int cpu_id)
5760 mutex_lock(&trace_types_lock);
5762 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5763 /* make sure, this cpu is enabled in the mask */
5764 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5770 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5775 mutex_unlock(&trace_types_lock);
5782 * tracing_update_buffers - used by tracing facility to expand ring buffers
5784 * To save on memory when the tracing is never used on a system with it
5785 * configured in. The ring buffers are set to a minimum size. But once
5786 * a user starts to use the tracing facility, then they need to grow
5787 * to their default size.
5789 * This function is to be called when a tracer is about to be used.
5791 int tracing_update_buffers(void)
5795 mutex_lock(&trace_types_lock);
5796 if (!ring_buffer_expanded)
5797 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5798 RING_BUFFER_ALL_CPUS);
5799 mutex_unlock(&trace_types_lock);
5804 struct trace_option_dentry;
5807 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5810 * Used to clear out the tracer before deletion of an instance.
5811 * Must have trace_types_lock held.
5813 static void tracing_set_nop(struct trace_array *tr)
5815 if (tr->current_trace == &nop_trace)
5818 tr->current_trace->enabled--;
5820 if (tr->current_trace->reset)
5821 tr->current_trace->reset(tr);
5823 tr->current_trace = &nop_trace;
5826 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5828 /* Only enable if the directory has been created already. */
5832 create_trace_option_files(tr, t);
5835 int tracing_set_tracer(struct trace_array *tr, const char *buf)
5838 #ifdef CONFIG_TRACER_MAX_TRACE
5843 mutex_lock(&trace_types_lock);
5845 if (!ring_buffer_expanded) {
5846 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5847 RING_BUFFER_ALL_CPUS);
5853 for (t = trace_types; t; t = t->next) {
5854 if (strcmp(t->name, buf) == 0)
5861 if (t == tr->current_trace)
5864 #ifdef CONFIG_TRACER_SNAPSHOT
5865 if (t->use_max_tr) {
5866 arch_spin_lock(&tr->max_lock);
5867 if (tr->cond_snapshot)
5869 arch_spin_unlock(&tr->max_lock);
5874 /* Some tracers won't work on kernel command line */
5875 if (system_state < SYSTEM_RUNNING && t->noboot) {
5876 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5881 /* Some tracers are only allowed for the top level buffer */
5882 if (!trace_ok_for_array(t, tr)) {
5887 /* If trace pipe files are being read, we can't change the tracer */
5888 if (tr->current_trace->ref) {
5893 trace_branch_disable();
5895 tr->current_trace->enabled--;
5897 if (tr->current_trace->reset)
5898 tr->current_trace->reset(tr);
5900 /* Current trace needs to be nop_trace before synchronize_rcu */
5901 tr->current_trace = &nop_trace;
5903 #ifdef CONFIG_TRACER_MAX_TRACE
5904 had_max_tr = tr->allocated_snapshot;
5906 if (had_max_tr && !t->use_max_tr) {
5908 * We need to make sure that the update_max_tr sees that
5909 * current_trace changed to nop_trace to keep it from
5910 * swapping the buffers after we resize it.
5911 * The update_max_tr is called from interrupts disabled
5912 * so a synchronized_sched() is sufficient.
5919 #ifdef CONFIG_TRACER_MAX_TRACE
5920 if (t->use_max_tr && !had_max_tr) {
5921 ret = tracing_alloc_snapshot_instance(tr);
5928 ret = tracer_init(t, tr);
5933 tr->current_trace = t;
5934 tr->current_trace->enabled++;
5935 trace_branch_enable(tr);
5937 mutex_unlock(&trace_types_lock);
5943 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5944 size_t cnt, loff_t *ppos)
5946 struct trace_array *tr = filp->private_data;
5947 char buf[MAX_TRACER_SIZE+1];
5954 if (cnt > MAX_TRACER_SIZE)
5955 cnt = MAX_TRACER_SIZE;
5957 if (copy_from_user(buf, ubuf, cnt))
5962 /* strip ending whitespace. */
5963 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5966 err = tracing_set_tracer(tr, buf);
5976 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5977 size_t cnt, loff_t *ppos)
5982 r = snprintf(buf, sizeof(buf), "%ld\n",
5983 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5984 if (r > sizeof(buf))
5986 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5990 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5991 size_t cnt, loff_t *ppos)
5996 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6006 tracing_thresh_read(struct file *filp, char __user *ubuf,
6007 size_t cnt, loff_t *ppos)
6009 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6013 tracing_thresh_write(struct file *filp, const char __user *ubuf,
6014 size_t cnt, loff_t *ppos)
6016 struct trace_array *tr = filp->private_data;
6019 mutex_lock(&trace_types_lock);
6020 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6024 if (tr->current_trace->update_thresh) {
6025 ret = tr->current_trace->update_thresh(tr);
6032 mutex_unlock(&trace_types_lock);
6037 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6040 tracing_max_lat_read(struct file *filp, char __user *ubuf,
6041 size_t cnt, loff_t *ppos)
6043 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
6047 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6048 size_t cnt, loff_t *ppos)
6050 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
6055 static int tracing_open_pipe(struct inode *inode, struct file *filp)
6057 struct trace_array *tr = inode->i_private;
6058 struct trace_iterator *iter;
6061 ret = tracing_check_open_get_tr(tr);
6065 mutex_lock(&trace_types_lock);
6067 /* create a buffer to store the information to pass to userspace */
6068 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6071 __trace_array_put(tr);
6075 trace_seq_init(&iter->seq);
6076 iter->trace = tr->current_trace;
6078 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6083 /* trace pipe does not show start of buffer */
6084 cpumask_setall(iter->started);
6086 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6087 iter->iter_flags |= TRACE_FILE_LAT_FMT;
6089 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6090 if (trace_clocks[tr->clock_id].in_ns)
6091 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6094 iter->array_buffer = &tr->array_buffer;
6095 iter->cpu_file = tracing_get_cpu(inode);
6096 mutex_init(&iter->mutex);
6097 filp->private_data = iter;
6099 if (iter->trace->pipe_open)
6100 iter->trace->pipe_open(iter);
6102 nonseekable_open(inode, filp);
6104 tr->current_trace->ref++;
6106 mutex_unlock(&trace_types_lock);
6111 __trace_array_put(tr);
6112 mutex_unlock(&trace_types_lock);
6116 static int tracing_release_pipe(struct inode *inode, struct file *file)
6118 struct trace_iterator *iter = file->private_data;
6119 struct trace_array *tr = inode->i_private;
6121 mutex_lock(&trace_types_lock);
6123 tr->current_trace->ref--;
6125 if (iter->trace->pipe_close)
6126 iter->trace->pipe_close(iter);
6128 mutex_unlock(&trace_types_lock);
6130 free_cpumask_var(iter->started);
6131 mutex_destroy(&iter->mutex);
6134 trace_array_put(tr);
6140 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6142 struct trace_array *tr = iter->tr;
6144 /* Iterators are static, they should be filled or empty */
6145 if (trace_buffer_iter(iter, iter->cpu_file))
6146 return EPOLLIN | EPOLLRDNORM;
6148 if (tr->trace_flags & TRACE_ITER_BLOCK)
6150 * Always select as readable when in blocking mode
6152 return EPOLLIN | EPOLLRDNORM;
6154 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6159 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6161 struct trace_iterator *iter = filp->private_data;
6163 return trace_poll(iter, filp, poll_table);
6166 /* Must be called with iter->mutex held. */
6167 static int tracing_wait_pipe(struct file *filp)
6169 struct trace_iterator *iter = filp->private_data;
6172 while (trace_empty(iter)) {
6174 if ((filp->f_flags & O_NONBLOCK)) {
6179 * We block until we read something and tracing is disabled.
6180 * We still block if tracing is disabled, but we have never
6181 * read anything. This allows a user to cat this file, and
6182 * then enable tracing. But after we have read something,
6183 * we give an EOF when tracing is again disabled.
6185 * iter->pos will be 0 if we haven't read anything.
6187 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6190 mutex_unlock(&iter->mutex);
6192 ret = wait_on_pipe(iter, 0);
6194 mutex_lock(&iter->mutex);
6207 tracing_read_pipe(struct file *filp, char __user *ubuf,
6208 size_t cnt, loff_t *ppos)
6210 struct trace_iterator *iter = filp->private_data;
6214 * Avoid more than one consumer on a single file descriptor
6215 * This is just a matter of traces coherency, the ring buffer itself
6218 mutex_lock(&iter->mutex);
6220 /* return any leftover data */
6221 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6225 trace_seq_init(&iter->seq);
6227 if (iter->trace->read) {
6228 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6234 sret = tracing_wait_pipe(filp);
6238 /* stop when tracing is finished */
6239 if (trace_empty(iter)) {
6244 if (cnt >= PAGE_SIZE)
6245 cnt = PAGE_SIZE - 1;
6247 /* reset all but tr, trace, and overruns */
6248 memset(&iter->seq, 0,
6249 sizeof(struct trace_iterator) -
6250 offsetof(struct trace_iterator, seq));
6251 cpumask_clear(iter->started);
6252 trace_seq_init(&iter->seq);
6255 trace_event_read_lock();
6256 trace_access_lock(iter->cpu_file);
6257 while (trace_find_next_entry_inc(iter) != NULL) {
6258 enum print_line_t ret;
6259 int save_len = iter->seq.seq.len;
6261 ret = print_trace_line(iter);
6262 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6263 /* don't print partial lines */
6264 iter->seq.seq.len = save_len;
6267 if (ret != TRACE_TYPE_NO_CONSUME)
6268 trace_consume(iter);
6270 if (trace_seq_used(&iter->seq) >= cnt)
6274 * Setting the full flag means we reached the trace_seq buffer
6275 * size and we should leave by partial output condition above.
6276 * One of the trace_seq_* functions is not used properly.
6278 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6281 trace_access_unlock(iter->cpu_file);
6282 trace_event_read_unlock();
6284 /* Now copy what we have to the user */
6285 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6286 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6287 trace_seq_init(&iter->seq);
6290 * If there was nothing to send to user, in spite of consuming trace
6291 * entries, go back to wait for more entries.
6297 mutex_unlock(&iter->mutex);
6302 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6305 __free_page(spd->pages[idx]);
6308 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
6309 .confirm = generic_pipe_buf_confirm,
6310 .release = generic_pipe_buf_release,
6311 .steal = generic_pipe_buf_steal,
6312 .get = generic_pipe_buf_get,
6316 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6322 /* Seq buffer is page-sized, exactly what we need. */
6324 save_len = iter->seq.seq.len;
6325 ret = print_trace_line(iter);
6327 if (trace_seq_has_overflowed(&iter->seq)) {
6328 iter->seq.seq.len = save_len;
6333 * This should not be hit, because it should only
6334 * be set if the iter->seq overflowed. But check it
6335 * anyway to be safe.
6337 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6338 iter->seq.seq.len = save_len;
6342 count = trace_seq_used(&iter->seq) - save_len;
6345 iter->seq.seq.len = save_len;
6349 if (ret != TRACE_TYPE_NO_CONSUME)
6350 trace_consume(iter);
6352 if (!trace_find_next_entry_inc(iter)) {
6362 static ssize_t tracing_splice_read_pipe(struct file *filp,
6364 struct pipe_inode_info *pipe,
6368 struct page *pages_def[PIPE_DEF_BUFFERS];
6369 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6370 struct trace_iterator *iter = filp->private_data;
6371 struct splice_pipe_desc spd = {
6373 .partial = partial_def,
6374 .nr_pages = 0, /* This gets updated below. */
6375 .nr_pages_max = PIPE_DEF_BUFFERS,
6376 .ops = &tracing_pipe_buf_ops,
6377 .spd_release = tracing_spd_release_pipe,
6383 if (splice_grow_spd(pipe, &spd))
6386 mutex_lock(&iter->mutex);
6388 if (iter->trace->splice_read) {
6389 ret = iter->trace->splice_read(iter, filp,
6390 ppos, pipe, len, flags);
6395 ret = tracing_wait_pipe(filp);
6399 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6404 trace_event_read_lock();
6405 trace_access_lock(iter->cpu_file);
6407 /* Fill as many pages as possible. */
6408 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6409 spd.pages[i] = alloc_page(GFP_KERNEL);
6413 rem = tracing_fill_pipe_page(rem, iter);
6415 /* Copy the data into the page, so we can start over. */
6416 ret = trace_seq_to_buffer(&iter->seq,
6417 page_address(spd.pages[i]),
6418 trace_seq_used(&iter->seq));
6420 __free_page(spd.pages[i]);
6423 spd.partial[i].offset = 0;
6424 spd.partial[i].len = trace_seq_used(&iter->seq);
6426 trace_seq_init(&iter->seq);
6429 trace_access_unlock(iter->cpu_file);
6430 trace_event_read_unlock();
6431 mutex_unlock(&iter->mutex);
6436 ret = splice_to_pipe(pipe, &spd);
6440 splice_shrink_spd(&spd);
6444 mutex_unlock(&iter->mutex);
6449 tracing_entries_read(struct file *filp, char __user *ubuf,
6450 size_t cnt, loff_t *ppos)
6452 struct inode *inode = file_inode(filp);
6453 struct trace_array *tr = inode->i_private;
6454 int cpu = tracing_get_cpu(inode);
6459 mutex_lock(&trace_types_lock);
6461 if (cpu == RING_BUFFER_ALL_CPUS) {
6462 int cpu, buf_size_same;
6467 /* check if all cpu sizes are same */
6468 for_each_tracing_cpu(cpu) {
6469 /* fill in the size from first enabled cpu */
6471 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6472 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6478 if (buf_size_same) {
6479 if (!ring_buffer_expanded)
6480 r = sprintf(buf, "%lu (expanded: %lu)\n",
6482 trace_buf_size >> 10);
6484 r = sprintf(buf, "%lu\n", size >> 10);
6486 r = sprintf(buf, "X\n");
6488 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6490 mutex_unlock(&trace_types_lock);
6492 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6497 tracing_entries_write(struct file *filp, const char __user *ubuf,
6498 size_t cnt, loff_t *ppos)
6500 struct inode *inode = file_inode(filp);
6501 struct trace_array *tr = inode->i_private;
6505 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6509 /* must have at least 1 entry */
6513 /* value is in KB */
6515 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6525 tracing_total_entries_read(struct file *filp, char __user *ubuf,
6526 size_t cnt, loff_t *ppos)
6528 struct trace_array *tr = filp->private_data;
6531 unsigned long size = 0, expanded_size = 0;
6533 mutex_lock(&trace_types_lock);
6534 for_each_tracing_cpu(cpu) {
6535 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
6536 if (!ring_buffer_expanded)
6537 expanded_size += trace_buf_size >> 10;
6539 if (ring_buffer_expanded)
6540 r = sprintf(buf, "%lu\n", size);
6542 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6543 mutex_unlock(&trace_types_lock);
6545 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6549 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6550 size_t cnt, loff_t *ppos)
6553 * There is no need to read what the user has written, this function
6554 * is just to make sure that there is no error when "echo" is used
6563 tracing_free_buffer_release(struct inode *inode, struct file *filp)
6565 struct trace_array *tr = inode->i_private;
6567 /* disable tracing ? */
6568 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6569 tracer_tracing_off(tr);
6570 /* resize the ring buffer to 0 */
6571 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6573 trace_array_put(tr);
6579 tracing_mark_write(struct file *filp, const char __user *ubuf,
6580 size_t cnt, loff_t *fpos)
6582 struct trace_array *tr = filp->private_data;
6583 struct ring_buffer_event *event;
6584 enum event_trigger_type tt = ETT_NONE;
6585 struct trace_buffer *buffer;
6586 struct print_entry *entry;
6587 unsigned long irq_flags;
6592 /* Used in tracing_mark_raw_write() as well */
6593 #define FAULTED_STR "<faulted>"
6594 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6596 if (tracing_disabled)
6599 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6602 if (cnt > TRACE_BUF_SIZE)
6603 cnt = TRACE_BUF_SIZE;
6605 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6607 local_save_flags(irq_flags);
6608 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6610 /* If less than "<faulted>", then make sure we can still add that */
6611 if (cnt < FAULTED_SIZE)
6612 size += FAULTED_SIZE - cnt;
6614 buffer = tr->array_buffer.buffer;
6615 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6616 irq_flags, preempt_count());
6617 if (unlikely(!event))
6618 /* Ring buffer disabled, return as if not open for write */
6621 entry = ring_buffer_event_data(event);
6622 entry->ip = _THIS_IP_;
6624 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6626 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6633 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6634 /* do not add \n before testing triggers, but add \0 */
6635 entry->buf[cnt] = '\0';
6636 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6639 if (entry->buf[cnt - 1] != '\n') {
6640 entry->buf[cnt] = '\n';
6641 entry->buf[cnt + 1] = '\0';
6643 entry->buf[cnt] = '\0';
6645 __buffer_unlock_commit(buffer, event);
6648 event_triggers_post_call(tr->trace_marker_file, tt);
6656 /* Limit it for now to 3K (including tag) */
6657 #define RAW_DATA_MAX_SIZE (1024*3)
6660 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6661 size_t cnt, loff_t *fpos)
6663 struct trace_array *tr = filp->private_data;
6664 struct ring_buffer_event *event;
6665 struct trace_buffer *buffer;
6666 struct raw_data_entry *entry;
6667 unsigned long irq_flags;
6672 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6674 if (tracing_disabled)
6677 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6680 /* The marker must at least have a tag id */
6681 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6684 if (cnt > TRACE_BUF_SIZE)
6685 cnt = TRACE_BUF_SIZE;
6687 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6689 local_save_flags(irq_flags);
6690 size = sizeof(*entry) + cnt;
6691 if (cnt < FAULT_SIZE_ID)
6692 size += FAULT_SIZE_ID - cnt;
6694 buffer = tr->array_buffer.buffer;
6695 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6696 irq_flags, preempt_count());
6698 /* Ring buffer disabled, return as if not open for write */
6701 entry = ring_buffer_event_data(event);
6703 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6706 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6711 __buffer_unlock_commit(buffer, event);
6719 static int tracing_clock_show(struct seq_file *m, void *v)
6721 struct trace_array *tr = m->private;
6724 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6726 "%s%s%s%s", i ? " " : "",
6727 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6728 i == tr->clock_id ? "]" : "");
6734 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6738 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6739 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6742 if (i == ARRAY_SIZE(trace_clocks))
6745 mutex_lock(&trace_types_lock);
6749 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
6752 * New clock may not be consistent with the previous clock.
6753 * Reset the buffer so that it doesn't have incomparable timestamps.
6755 tracing_reset_online_cpus(&tr->array_buffer);
6757 #ifdef CONFIG_TRACER_MAX_TRACE
6758 if (tr->max_buffer.buffer)
6759 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6760 tracing_reset_online_cpus(&tr->max_buffer);
6763 mutex_unlock(&trace_types_lock);
6768 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6769 size_t cnt, loff_t *fpos)
6771 struct seq_file *m = filp->private_data;
6772 struct trace_array *tr = m->private;
6774 const char *clockstr;
6777 if (cnt >= sizeof(buf))
6780 if (copy_from_user(buf, ubuf, cnt))
6785 clockstr = strstrip(buf);
6787 ret = tracing_set_clock(tr, clockstr);
6796 static int tracing_clock_open(struct inode *inode, struct file *file)
6798 struct trace_array *tr = inode->i_private;
6801 ret = tracing_check_open_get_tr(tr);
6805 ret = single_open(file, tracing_clock_show, inode->i_private);
6807 trace_array_put(tr);
6812 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6814 struct trace_array *tr = m->private;
6816 mutex_lock(&trace_types_lock);
6818 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
6819 seq_puts(m, "delta [absolute]\n");
6821 seq_puts(m, "[delta] absolute\n");
6823 mutex_unlock(&trace_types_lock);
6828 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6830 struct trace_array *tr = inode->i_private;
6833 ret = tracing_check_open_get_tr(tr);
6837 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6839 trace_array_put(tr);
6844 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6848 mutex_lock(&trace_types_lock);
6850 if (abs && tr->time_stamp_abs_ref++)
6854 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6859 if (--tr->time_stamp_abs_ref)
6863 ring_buffer_set_time_stamp_abs(tr->array_buffer.buffer, abs);
6865 #ifdef CONFIG_TRACER_MAX_TRACE
6866 if (tr->max_buffer.buffer)
6867 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6870 mutex_unlock(&trace_types_lock);
6875 struct ftrace_buffer_info {
6876 struct trace_iterator iter;
6878 unsigned int spare_cpu;
6882 #ifdef CONFIG_TRACER_SNAPSHOT
6883 static int tracing_snapshot_open(struct inode *inode, struct file *file)
6885 struct trace_array *tr = inode->i_private;
6886 struct trace_iterator *iter;
6890 ret = tracing_check_open_get_tr(tr);
6894 if (file->f_mode & FMODE_READ) {
6895 iter = __tracing_open(inode, file, true);
6897 ret = PTR_ERR(iter);
6899 /* Writes still need the seq_file to hold the private data */
6901 m = kzalloc(sizeof(*m), GFP_KERNEL);
6904 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6912 iter->array_buffer = &tr->max_buffer;
6913 iter->cpu_file = tracing_get_cpu(inode);
6915 file->private_data = m;
6919 trace_array_put(tr);
6925 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6928 struct seq_file *m = filp->private_data;
6929 struct trace_iterator *iter = m->private;
6930 struct trace_array *tr = iter->tr;
6934 ret = tracing_update_buffers();
6938 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6942 mutex_lock(&trace_types_lock);
6944 if (tr->current_trace->use_max_tr) {
6949 arch_spin_lock(&tr->max_lock);
6950 if (tr->cond_snapshot)
6952 arch_spin_unlock(&tr->max_lock);
6958 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6962 if (tr->allocated_snapshot)
6966 /* Only allow per-cpu swap if the ring buffer supports it */
6967 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6968 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6973 if (tr->allocated_snapshot)
6974 ret = resize_buffer_duplicate_size(&tr->max_buffer,
6975 &tr->array_buffer, iter->cpu_file);
6977 ret = tracing_alloc_snapshot_instance(tr);
6980 local_irq_disable();
6981 /* Now, we're going to swap */
6982 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6983 update_max_tr(tr, current, smp_processor_id(), NULL);
6985 update_max_tr_single(tr, current, iter->cpu_file);
6989 if (tr->allocated_snapshot) {
6990 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6991 tracing_reset_online_cpus(&tr->max_buffer);
6993 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7003 mutex_unlock(&trace_types_lock);
7007 static int tracing_snapshot_release(struct inode *inode, struct file *file)
7009 struct seq_file *m = file->private_data;
7012 ret = tracing_release(inode, file);
7014 if (file->f_mode & FMODE_READ)
7017 /* If write only, the seq_file is just a stub */
7025 static int tracing_buffers_open(struct inode *inode, struct file *filp);
7026 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7027 size_t count, loff_t *ppos);
7028 static int tracing_buffers_release(struct inode *inode, struct file *file);
7029 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7030 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7032 static int snapshot_raw_open(struct inode *inode, struct file *filp)
7034 struct ftrace_buffer_info *info;
7037 /* The following checks for tracefs lockdown */
7038 ret = tracing_buffers_open(inode, filp);
7042 info = filp->private_data;
7044 if (info->iter.trace->use_max_tr) {
7045 tracing_buffers_release(inode, filp);
7049 info->iter.snapshot = true;
7050 info->iter.array_buffer = &info->iter.tr->max_buffer;
7055 #endif /* CONFIG_TRACER_SNAPSHOT */
7058 static const struct file_operations tracing_thresh_fops = {
7059 .open = tracing_open_generic,
7060 .read = tracing_thresh_read,
7061 .write = tracing_thresh_write,
7062 .llseek = generic_file_llseek,
7065 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7066 static const struct file_operations tracing_max_lat_fops = {
7067 .open = tracing_open_generic,
7068 .read = tracing_max_lat_read,
7069 .write = tracing_max_lat_write,
7070 .llseek = generic_file_llseek,
7074 static const struct file_operations set_tracer_fops = {
7075 .open = tracing_open_generic,
7076 .read = tracing_set_trace_read,
7077 .write = tracing_set_trace_write,
7078 .llseek = generic_file_llseek,
7081 static const struct file_operations tracing_pipe_fops = {
7082 .open = tracing_open_pipe,
7083 .poll = tracing_poll_pipe,
7084 .read = tracing_read_pipe,
7085 .splice_read = tracing_splice_read_pipe,
7086 .release = tracing_release_pipe,
7087 .llseek = no_llseek,
7090 static const struct file_operations tracing_entries_fops = {
7091 .open = tracing_open_generic_tr,
7092 .read = tracing_entries_read,
7093 .write = tracing_entries_write,
7094 .llseek = generic_file_llseek,
7095 .release = tracing_release_generic_tr,
7098 static const struct file_operations tracing_total_entries_fops = {
7099 .open = tracing_open_generic_tr,
7100 .read = tracing_total_entries_read,
7101 .llseek = generic_file_llseek,
7102 .release = tracing_release_generic_tr,
7105 static const struct file_operations tracing_free_buffer_fops = {
7106 .open = tracing_open_generic_tr,
7107 .write = tracing_free_buffer_write,
7108 .release = tracing_free_buffer_release,
7111 static const struct file_operations tracing_mark_fops = {
7112 .open = tracing_open_generic_tr,
7113 .write = tracing_mark_write,
7114 .llseek = generic_file_llseek,
7115 .release = tracing_release_generic_tr,
7118 static const struct file_operations tracing_mark_raw_fops = {
7119 .open = tracing_open_generic_tr,
7120 .write = tracing_mark_raw_write,
7121 .llseek = generic_file_llseek,
7122 .release = tracing_release_generic_tr,
7125 static const struct file_operations trace_clock_fops = {
7126 .open = tracing_clock_open,
7128 .llseek = seq_lseek,
7129 .release = tracing_single_release_tr,
7130 .write = tracing_clock_write,
7133 static const struct file_operations trace_time_stamp_mode_fops = {
7134 .open = tracing_time_stamp_mode_open,
7136 .llseek = seq_lseek,
7137 .release = tracing_single_release_tr,
7140 #ifdef CONFIG_TRACER_SNAPSHOT
7141 static const struct file_operations snapshot_fops = {
7142 .open = tracing_snapshot_open,
7144 .write = tracing_snapshot_write,
7145 .llseek = tracing_lseek,
7146 .release = tracing_snapshot_release,
7149 static const struct file_operations snapshot_raw_fops = {
7150 .open = snapshot_raw_open,
7151 .read = tracing_buffers_read,
7152 .release = tracing_buffers_release,
7153 .splice_read = tracing_buffers_splice_read,
7154 .llseek = no_llseek,
7157 #endif /* CONFIG_TRACER_SNAPSHOT */
7159 #define TRACING_LOG_ERRS_MAX 8
7160 #define TRACING_LOG_LOC_MAX 128
7162 #define CMD_PREFIX " Command: "
7165 const char **errs; /* ptr to loc-specific array of err strings */
7166 u8 type; /* index into errs -> specific err string */
7167 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
7171 struct tracing_log_err {
7172 struct list_head list;
7173 struct err_info info;
7174 char loc[TRACING_LOG_LOC_MAX]; /* err location */
7175 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
7178 static DEFINE_MUTEX(tracing_err_log_lock);
7180 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
7182 struct tracing_log_err *err;
7184 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7185 err = kzalloc(sizeof(*err), GFP_KERNEL);
7187 err = ERR_PTR(-ENOMEM);
7188 tr->n_err_log_entries++;
7193 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7194 list_del(&err->list);
7200 * err_pos - find the position of a string within a command for error careting
7201 * @cmd: The tracing command that caused the error
7202 * @str: The string to position the caret at within @cmd
7204 * Finds the position of the first occurence of @str within @cmd. The
7205 * return value can be passed to tracing_log_err() for caret placement
7208 * Returns the index within @cmd of the first occurence of @str or 0
7209 * if @str was not found.
7211 unsigned int err_pos(char *cmd, const char *str)
7215 if (WARN_ON(!strlen(cmd)))
7218 found = strstr(cmd, str);
7226 * tracing_log_err - write an error to the tracing error log
7227 * @tr: The associated trace array for the error (NULL for top level array)
7228 * @loc: A string describing where the error occurred
7229 * @cmd: The tracing command that caused the error
7230 * @errs: The array of loc-specific static error strings
7231 * @type: The index into errs[], which produces the specific static err string
7232 * @pos: The position the caret should be placed in the cmd
7234 * Writes an error into tracing/error_log of the form:
7236 * <loc>: error: <text>
7240 * tracing/error_log is a small log file containing the last
7241 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7242 * unless there has been a tracing error, and the error log can be
7243 * cleared and have its memory freed by writing the empty string in
7244 * truncation mode to it i.e. echo > tracing/error_log.
7246 * NOTE: the @errs array along with the @type param are used to
7247 * produce a static error string - this string is not copied and saved
7248 * when the error is logged - only a pointer to it is saved. See
7249 * existing callers for examples of how static strings are typically
7250 * defined for use with tracing_log_err().
7252 void tracing_log_err(struct trace_array *tr,
7253 const char *loc, const char *cmd,
7254 const char **errs, u8 type, u8 pos)
7256 struct tracing_log_err *err;
7261 mutex_lock(&tracing_err_log_lock);
7262 err = get_tracing_log_err(tr);
7263 if (PTR_ERR(err) == -ENOMEM) {
7264 mutex_unlock(&tracing_err_log_lock);
7268 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7269 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7271 err->info.errs = errs;
7272 err->info.type = type;
7273 err->info.pos = pos;
7274 err->info.ts = local_clock();
7276 list_add_tail(&err->list, &tr->err_log);
7277 mutex_unlock(&tracing_err_log_lock);
7280 static void clear_tracing_err_log(struct trace_array *tr)
7282 struct tracing_log_err *err, *next;
7284 mutex_lock(&tracing_err_log_lock);
7285 list_for_each_entry_safe(err, next, &tr->err_log, list) {
7286 list_del(&err->list);
7290 tr->n_err_log_entries = 0;
7291 mutex_unlock(&tracing_err_log_lock);
7294 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7296 struct trace_array *tr = m->private;
7298 mutex_lock(&tracing_err_log_lock);
7300 return seq_list_start(&tr->err_log, *pos);
7303 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7305 struct trace_array *tr = m->private;
7307 return seq_list_next(v, &tr->err_log, pos);
7310 static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7312 mutex_unlock(&tracing_err_log_lock);
7315 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7319 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7321 for (i = 0; i < pos; i++)
7326 static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7328 struct tracing_log_err *err = v;
7331 const char *err_text = err->info.errs[err->info.type];
7332 u64 sec = err->info.ts;
7335 nsec = do_div(sec, NSEC_PER_SEC);
7336 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7337 err->loc, err_text);
7338 seq_printf(m, "%s", err->cmd);
7339 tracing_err_log_show_pos(m, err->info.pos);
7345 static const struct seq_operations tracing_err_log_seq_ops = {
7346 .start = tracing_err_log_seq_start,
7347 .next = tracing_err_log_seq_next,
7348 .stop = tracing_err_log_seq_stop,
7349 .show = tracing_err_log_seq_show
7352 static int tracing_err_log_open(struct inode *inode, struct file *file)
7354 struct trace_array *tr = inode->i_private;
7357 ret = tracing_check_open_get_tr(tr);
7361 /* If this file was opened for write, then erase contents */
7362 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7363 clear_tracing_err_log(tr);
7365 if (file->f_mode & FMODE_READ) {
7366 ret = seq_open(file, &tracing_err_log_seq_ops);
7368 struct seq_file *m = file->private_data;
7371 trace_array_put(tr);
7377 static ssize_t tracing_err_log_write(struct file *file,
7378 const char __user *buffer,
7379 size_t count, loff_t *ppos)
7384 static int tracing_err_log_release(struct inode *inode, struct file *file)
7386 struct trace_array *tr = inode->i_private;
7388 trace_array_put(tr);
7390 if (file->f_mode & FMODE_READ)
7391 seq_release(inode, file);
7396 static const struct file_operations tracing_err_log_fops = {
7397 .open = tracing_err_log_open,
7398 .write = tracing_err_log_write,
7400 .llseek = seq_lseek,
7401 .release = tracing_err_log_release,
7404 static int tracing_buffers_open(struct inode *inode, struct file *filp)
7406 struct trace_array *tr = inode->i_private;
7407 struct ftrace_buffer_info *info;
7410 ret = tracing_check_open_get_tr(tr);
7414 info = kzalloc(sizeof(*info), GFP_KERNEL);
7416 trace_array_put(tr);
7420 mutex_lock(&trace_types_lock);
7423 info->iter.cpu_file = tracing_get_cpu(inode);
7424 info->iter.trace = tr->current_trace;
7425 info->iter.array_buffer = &tr->array_buffer;
7427 /* Force reading ring buffer for first read */
7428 info->read = (unsigned int)-1;
7430 filp->private_data = info;
7432 tr->current_trace->ref++;
7434 mutex_unlock(&trace_types_lock);
7436 ret = nonseekable_open(inode, filp);
7438 trace_array_put(tr);
7444 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7446 struct ftrace_buffer_info *info = filp->private_data;
7447 struct trace_iterator *iter = &info->iter;
7449 return trace_poll(iter, filp, poll_table);
7453 tracing_buffers_read(struct file *filp, char __user *ubuf,
7454 size_t count, loff_t *ppos)
7456 struct ftrace_buffer_info *info = filp->private_data;
7457 struct trace_iterator *iter = &info->iter;
7464 #ifdef CONFIG_TRACER_MAX_TRACE
7465 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7470 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
7472 if (IS_ERR(info->spare)) {
7473 ret = PTR_ERR(info->spare);
7476 info->spare_cpu = iter->cpu_file;
7482 /* Do we have previous read data to read? */
7483 if (info->read < PAGE_SIZE)
7487 trace_access_lock(iter->cpu_file);
7488 ret = ring_buffer_read_page(iter->array_buffer->buffer,
7492 trace_access_unlock(iter->cpu_file);
7495 if (trace_empty(iter)) {
7496 if ((filp->f_flags & O_NONBLOCK))
7499 ret = wait_on_pipe(iter, 0);
7510 size = PAGE_SIZE - info->read;
7514 ret = copy_to_user(ubuf, info->spare + info->read, size);
7526 static int tracing_buffers_release(struct inode *inode, struct file *file)
7528 struct ftrace_buffer_info *info = file->private_data;
7529 struct trace_iterator *iter = &info->iter;
7531 mutex_lock(&trace_types_lock);
7533 iter->tr->current_trace->ref--;
7535 __trace_array_put(iter->tr);
7538 ring_buffer_free_read_page(iter->array_buffer->buffer,
7539 info->spare_cpu, info->spare);
7542 mutex_unlock(&trace_types_lock);
7548 struct trace_buffer *buffer;
7551 refcount_t refcount;
7554 static void buffer_ref_release(struct buffer_ref *ref)
7556 if (!refcount_dec_and_test(&ref->refcount))
7558 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7562 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7563 struct pipe_buffer *buf)
7565 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7567 buffer_ref_release(ref);
7571 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7572 struct pipe_buffer *buf)
7574 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7576 if (refcount_read(&ref->refcount) > INT_MAX/2)
7579 refcount_inc(&ref->refcount);
7583 /* Pipe buffer operations for a buffer. */
7584 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7585 .confirm = generic_pipe_buf_confirm,
7586 .release = buffer_pipe_buf_release,
7587 .steal = generic_pipe_buf_nosteal,
7588 .get = buffer_pipe_buf_get,
7592 * Callback from splice_to_pipe(), if we need to release some pages
7593 * at the end of the spd in case we error'ed out in filling the pipe.
7595 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7597 struct buffer_ref *ref =
7598 (struct buffer_ref *)spd->partial[i].private;
7600 buffer_ref_release(ref);
7601 spd->partial[i].private = 0;
7605 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7606 struct pipe_inode_info *pipe, size_t len,
7609 struct ftrace_buffer_info *info = file->private_data;
7610 struct trace_iterator *iter = &info->iter;
7611 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7612 struct page *pages_def[PIPE_DEF_BUFFERS];
7613 struct splice_pipe_desc spd = {
7615 .partial = partial_def,
7616 .nr_pages_max = PIPE_DEF_BUFFERS,
7617 .ops = &buffer_pipe_buf_ops,
7618 .spd_release = buffer_spd_release,
7620 struct buffer_ref *ref;
7624 #ifdef CONFIG_TRACER_MAX_TRACE
7625 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7629 if (*ppos & (PAGE_SIZE - 1))
7632 if (len & (PAGE_SIZE - 1)) {
7633 if (len < PAGE_SIZE)
7638 if (splice_grow_spd(pipe, &spd))
7642 trace_access_lock(iter->cpu_file);
7643 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7645 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
7649 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
7655 refcount_set(&ref->refcount, 1);
7656 ref->buffer = iter->array_buffer->buffer;
7657 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
7658 if (IS_ERR(ref->page)) {
7659 ret = PTR_ERR(ref->page);
7664 ref->cpu = iter->cpu_file;
7666 r = ring_buffer_read_page(ref->buffer, &ref->page,
7667 len, iter->cpu_file, 1);
7669 ring_buffer_free_read_page(ref->buffer, ref->cpu,
7675 page = virt_to_page(ref->page);
7677 spd.pages[i] = page;
7678 spd.partial[i].len = PAGE_SIZE;
7679 spd.partial[i].offset = 0;
7680 spd.partial[i].private = (unsigned long)ref;
7684 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7687 trace_access_unlock(iter->cpu_file);
7690 /* did we read anything? */
7691 if (!spd.nr_pages) {
7696 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7699 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
7706 ret = splice_to_pipe(pipe, &spd);
7708 splice_shrink_spd(&spd);
7713 static const struct file_operations tracing_buffers_fops = {
7714 .open = tracing_buffers_open,
7715 .read = tracing_buffers_read,
7716 .poll = tracing_buffers_poll,
7717 .release = tracing_buffers_release,
7718 .splice_read = tracing_buffers_splice_read,
7719 .llseek = no_llseek,
7723 tracing_stats_read(struct file *filp, char __user *ubuf,
7724 size_t count, loff_t *ppos)
7726 struct inode *inode = file_inode(filp);
7727 struct trace_array *tr = inode->i_private;
7728 struct array_buffer *trace_buf = &tr->array_buffer;
7729 int cpu = tracing_get_cpu(inode);
7730 struct trace_seq *s;
7732 unsigned long long t;
7733 unsigned long usec_rem;
7735 s = kmalloc(sizeof(*s), GFP_KERNEL);
7741 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7742 trace_seq_printf(s, "entries: %ld\n", cnt);
7744 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7745 trace_seq_printf(s, "overrun: %ld\n", cnt);
7747 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7748 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7750 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7751 trace_seq_printf(s, "bytes: %ld\n", cnt);
7753 if (trace_clocks[tr->clock_id].in_ns) {
7754 /* local or global for trace_clock */
7755 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7756 usec_rem = do_div(t, USEC_PER_SEC);
7757 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7760 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
7761 usec_rem = do_div(t, USEC_PER_SEC);
7762 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7764 /* counter or tsc mode for trace_clock */
7765 trace_seq_printf(s, "oldest event ts: %llu\n",
7766 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7768 trace_seq_printf(s, "now ts: %llu\n",
7769 ring_buffer_time_stamp(trace_buf->buffer, cpu));
7772 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7773 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7775 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7776 trace_seq_printf(s, "read events: %ld\n", cnt);
7778 count = simple_read_from_buffer(ubuf, count, ppos,
7779 s->buffer, trace_seq_used(s));
7786 static const struct file_operations tracing_stats_fops = {
7787 .open = tracing_open_generic_tr,
7788 .read = tracing_stats_read,
7789 .llseek = generic_file_llseek,
7790 .release = tracing_release_generic_tr,
7793 #ifdef CONFIG_DYNAMIC_FTRACE
7796 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7797 size_t cnt, loff_t *ppos)
7803 /* 256 should be plenty to hold the amount needed */
7804 buf = kmalloc(256, GFP_KERNEL);
7808 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
7809 ftrace_update_tot_cnt,
7810 ftrace_number_of_pages,
7811 ftrace_number_of_groups);
7813 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7818 static const struct file_operations tracing_dyn_info_fops = {
7819 .open = tracing_open_generic,
7820 .read = tracing_read_dyn_info,
7821 .llseek = generic_file_llseek,
7823 #endif /* CONFIG_DYNAMIC_FTRACE */
7825 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7827 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7828 struct trace_array *tr, struct ftrace_probe_ops *ops,
7831 tracing_snapshot_instance(tr);
7835 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7836 struct trace_array *tr, struct ftrace_probe_ops *ops,
7839 struct ftrace_func_mapper *mapper = data;
7843 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7853 tracing_snapshot_instance(tr);
7857 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7858 struct ftrace_probe_ops *ops, void *data)
7860 struct ftrace_func_mapper *mapper = data;
7863 seq_printf(m, "%ps:", (void *)ip);
7865 seq_puts(m, "snapshot");
7868 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7871 seq_printf(m, ":count=%ld\n", *count);
7873 seq_puts(m, ":unlimited\n");
7879 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7880 unsigned long ip, void *init_data, void **data)
7882 struct ftrace_func_mapper *mapper = *data;
7885 mapper = allocate_ftrace_func_mapper();
7891 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7895 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7896 unsigned long ip, void *data)
7898 struct ftrace_func_mapper *mapper = data;
7903 free_ftrace_func_mapper(mapper, NULL);
7907 ftrace_func_mapper_remove_ip(mapper, ip);
7910 static struct ftrace_probe_ops snapshot_probe_ops = {
7911 .func = ftrace_snapshot,
7912 .print = ftrace_snapshot_print,
7915 static struct ftrace_probe_ops snapshot_count_probe_ops = {
7916 .func = ftrace_count_snapshot,
7917 .print = ftrace_snapshot_print,
7918 .init = ftrace_snapshot_init,
7919 .free = ftrace_snapshot_free,
7923 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7924 char *glob, char *cmd, char *param, int enable)
7926 struct ftrace_probe_ops *ops;
7927 void *count = (void *)-1;
7934 /* hash funcs only work with set_ftrace_filter */
7938 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7941 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
7946 number = strsep(¶m, ":");
7948 if (!strlen(number))
7952 * We use the callback data field (which is a pointer)
7955 ret = kstrtoul(number, 0, (unsigned long *)&count);
7960 ret = tracing_alloc_snapshot_instance(tr);
7964 ret = register_ftrace_function_probe(glob, tr, ops, count);
7967 return ret < 0 ? ret : 0;
7970 static struct ftrace_func_command ftrace_snapshot_cmd = {
7972 .func = ftrace_trace_snapshot_callback,
7975 static __init int register_snapshot_cmd(void)
7977 return register_ftrace_command(&ftrace_snapshot_cmd);
7980 static inline __init int register_snapshot_cmd(void) { return 0; }
7981 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7983 static struct dentry *tracing_get_dentry(struct trace_array *tr)
7985 if (WARN_ON(!tr->dir))
7986 return ERR_PTR(-ENODEV);
7988 /* Top directory uses NULL as the parent */
7989 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7992 /* All sub buffers have a descriptor */
7996 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7998 struct dentry *d_tracer;
8001 return tr->percpu_dir;
8003 d_tracer = tracing_get_dentry(tr);
8004 if (IS_ERR(d_tracer))
8007 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8009 MEM_FAIL(!tr->percpu_dir,
8010 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8012 return tr->percpu_dir;
8015 static struct dentry *
8016 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8017 void *data, long cpu, const struct file_operations *fops)
8019 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8021 if (ret) /* See tracing_get_cpu() */
8022 d_inode(ret)->i_cdev = (void *)(cpu + 1);
8027 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8029 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8030 struct dentry *d_cpu;
8031 char cpu_dir[30]; /* 30 characters should be more than enough */
8036 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8037 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8039 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8043 /* per cpu trace_pipe */
8044 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
8045 tr, cpu, &tracing_pipe_fops);
8048 trace_create_cpu_file("trace", 0644, d_cpu,
8049 tr, cpu, &tracing_fops);
8051 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
8052 tr, cpu, &tracing_buffers_fops);
8054 trace_create_cpu_file("stats", 0444, d_cpu,
8055 tr, cpu, &tracing_stats_fops);
8057 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
8058 tr, cpu, &tracing_entries_fops);
8060 #ifdef CONFIG_TRACER_SNAPSHOT
8061 trace_create_cpu_file("snapshot", 0644, d_cpu,
8062 tr, cpu, &snapshot_fops);
8064 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
8065 tr, cpu, &snapshot_raw_fops);
8069 #ifdef CONFIG_FTRACE_SELFTEST
8070 /* Let selftest have access to static functions in this file */
8071 #include "trace_selftest.c"
8075 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8078 struct trace_option_dentry *topt = filp->private_data;
8081 if (topt->flags->val & topt->opt->bit)
8086 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8090 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8093 struct trace_option_dentry *topt = filp->private_data;
8097 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8101 if (val != 0 && val != 1)
8104 if (!!(topt->flags->val & topt->opt->bit) != val) {
8105 mutex_lock(&trace_types_lock);
8106 ret = __set_tracer_option(topt->tr, topt->flags,
8108 mutex_unlock(&trace_types_lock);
8119 static const struct file_operations trace_options_fops = {
8120 .open = tracing_open_generic,
8121 .read = trace_options_read,
8122 .write = trace_options_write,
8123 .llseek = generic_file_llseek,
8127 * In order to pass in both the trace_array descriptor as well as the index
8128 * to the flag that the trace option file represents, the trace_array
8129 * has a character array of trace_flags_index[], which holds the index
8130 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8131 * The address of this character array is passed to the flag option file
8132 * read/write callbacks.
8134 * In order to extract both the index and the trace_array descriptor,
8135 * get_tr_index() uses the following algorithm.
8139 * As the pointer itself contains the address of the index (remember
8142 * Then to get the trace_array descriptor, by subtracting that index
8143 * from the ptr, we get to the start of the index itself.
8145 * ptr - idx == &index[0]
8147 * Then a simple container_of() from that pointer gets us to the
8148 * trace_array descriptor.
8150 static void get_tr_index(void *data, struct trace_array **ptr,
8151 unsigned int *pindex)
8153 *pindex = *(unsigned char *)data;
8155 *ptr = container_of(data - *pindex, struct trace_array,
8160 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8163 void *tr_index = filp->private_data;
8164 struct trace_array *tr;
8168 get_tr_index(tr_index, &tr, &index);
8170 if (tr->trace_flags & (1 << index))
8175 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8179 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8182 void *tr_index = filp->private_data;
8183 struct trace_array *tr;
8188 get_tr_index(tr_index, &tr, &index);
8190 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8194 if (val != 0 && val != 1)
8197 mutex_lock(&event_mutex);
8198 mutex_lock(&trace_types_lock);
8199 ret = set_tracer_flag(tr, 1 << index, val);
8200 mutex_unlock(&trace_types_lock);
8201 mutex_unlock(&event_mutex);
8211 static const struct file_operations trace_options_core_fops = {
8212 .open = tracing_open_generic,
8213 .read = trace_options_core_read,
8214 .write = trace_options_core_write,
8215 .llseek = generic_file_llseek,
8218 struct dentry *trace_create_file(const char *name,
8220 struct dentry *parent,
8222 const struct file_operations *fops)
8226 ret = tracefs_create_file(name, mode, parent, data, fops);
8228 pr_warn("Could not create tracefs '%s' entry\n", name);
8234 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8236 struct dentry *d_tracer;
8241 d_tracer = tracing_get_dentry(tr);
8242 if (IS_ERR(d_tracer))
8245 tr->options = tracefs_create_dir("options", d_tracer);
8247 pr_warn("Could not create tracefs directory 'options'\n");
8255 create_trace_option_file(struct trace_array *tr,
8256 struct trace_option_dentry *topt,
8257 struct tracer_flags *flags,
8258 struct tracer_opt *opt)
8260 struct dentry *t_options;
8262 t_options = trace_options_init_dentry(tr);
8266 topt->flags = flags;
8270 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
8271 &trace_options_fops);
8276 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8278 struct trace_option_dentry *topts;
8279 struct trace_options *tr_topts;
8280 struct tracer_flags *flags;
8281 struct tracer_opt *opts;
8288 flags = tracer->flags;
8290 if (!flags || !flags->opts)
8294 * If this is an instance, only create flags for tracers
8295 * the instance may have.
8297 if (!trace_ok_for_array(tracer, tr))
8300 for (i = 0; i < tr->nr_topts; i++) {
8301 /* Make sure there's no duplicate flags. */
8302 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8308 for (cnt = 0; opts[cnt].name; cnt++)
8311 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8315 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8322 tr->topts = tr_topts;
8323 tr->topts[tr->nr_topts].tracer = tracer;
8324 tr->topts[tr->nr_topts].topts = topts;
8327 for (cnt = 0; opts[cnt].name; cnt++) {
8328 create_trace_option_file(tr, &topts[cnt], flags,
8330 MEM_FAIL(topts[cnt].entry == NULL,
8331 "Failed to create trace option: %s",
8336 static struct dentry *
8337 create_trace_option_core_file(struct trace_array *tr,
8338 const char *option, long index)
8340 struct dentry *t_options;
8342 t_options = trace_options_init_dentry(tr);
8346 return trace_create_file(option, 0644, t_options,
8347 (void *)&tr->trace_flags_index[index],
8348 &trace_options_core_fops);
8351 static void create_trace_options_dir(struct trace_array *tr)
8353 struct dentry *t_options;
8354 bool top_level = tr == &global_trace;
8357 t_options = trace_options_init_dentry(tr);
8361 for (i = 0; trace_options[i]; i++) {
8363 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8364 create_trace_option_core_file(tr, trace_options[i], i);
8369 rb_simple_read(struct file *filp, char __user *ubuf,
8370 size_t cnt, loff_t *ppos)
8372 struct trace_array *tr = filp->private_data;
8376 r = tracer_tracing_is_on(tr);
8377 r = sprintf(buf, "%d\n", r);
8379 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8383 rb_simple_write(struct file *filp, const char __user *ubuf,
8384 size_t cnt, loff_t *ppos)
8386 struct trace_array *tr = filp->private_data;
8387 struct trace_buffer *buffer = tr->array_buffer.buffer;
8391 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8396 mutex_lock(&trace_types_lock);
8397 if (!!val == tracer_tracing_is_on(tr)) {
8398 val = 0; /* do nothing */
8400 tracer_tracing_on(tr);
8401 if (tr->current_trace->start)
8402 tr->current_trace->start(tr);
8404 tracer_tracing_off(tr);
8405 if (tr->current_trace->stop)
8406 tr->current_trace->stop(tr);
8408 mutex_unlock(&trace_types_lock);
8416 static const struct file_operations rb_simple_fops = {
8417 .open = tracing_open_generic_tr,
8418 .read = rb_simple_read,
8419 .write = rb_simple_write,
8420 .release = tracing_release_generic_tr,
8421 .llseek = default_llseek,
8425 buffer_percent_read(struct file *filp, char __user *ubuf,
8426 size_t cnt, loff_t *ppos)
8428 struct trace_array *tr = filp->private_data;
8432 r = tr->buffer_percent;
8433 r = sprintf(buf, "%d\n", r);
8435 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8439 buffer_percent_write(struct file *filp, const char __user *ubuf,
8440 size_t cnt, loff_t *ppos)
8442 struct trace_array *tr = filp->private_data;
8446 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8456 tr->buffer_percent = val;
8463 static const struct file_operations buffer_percent_fops = {
8464 .open = tracing_open_generic_tr,
8465 .read = buffer_percent_read,
8466 .write = buffer_percent_write,
8467 .release = tracing_release_generic_tr,
8468 .llseek = default_llseek,
8471 static struct dentry *trace_instance_dir;
8474 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8477 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
8479 enum ring_buffer_flags rb_flags;
8481 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
8485 buf->buffer = ring_buffer_alloc(size, rb_flags);
8489 buf->data = alloc_percpu(struct trace_array_cpu);
8491 ring_buffer_free(buf->buffer);
8496 /* Allocate the first page for all buffers */
8497 set_buffer_entries(&tr->array_buffer,
8498 ring_buffer_size(tr->array_buffer.buffer, 0));
8503 static int allocate_trace_buffers(struct trace_array *tr, int size)
8507 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
8511 #ifdef CONFIG_TRACER_MAX_TRACE
8512 ret = allocate_trace_buffer(tr, &tr->max_buffer,
8513 allocate_snapshot ? size : 1);
8514 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
8515 ring_buffer_free(tr->array_buffer.buffer);
8516 tr->array_buffer.buffer = NULL;
8517 free_percpu(tr->array_buffer.data);
8518 tr->array_buffer.data = NULL;
8521 tr->allocated_snapshot = allocate_snapshot;
8524 * Only the top level trace array gets its snapshot allocated
8525 * from the kernel command line.
8527 allocate_snapshot = false;
8531 * Because of some magic with the way alloc_percpu() works on
8532 * x86_64, we need to synchronize the pgd of all the tables,
8533 * otherwise the trace events that happen in x86_64 page fault
8534 * handlers can't cope with accessing the chance that a
8535 * alloc_percpu()'d memory might be touched in the page fault trace
8536 * event. Oh, and we need to audit all other alloc_percpu() and vmalloc()
8537 * calls in tracing, because something might get triggered within a
8538 * page fault trace event!
8540 vmalloc_sync_mappings();
8545 static void free_trace_buffer(struct array_buffer *buf)
8548 ring_buffer_free(buf->buffer);
8550 free_percpu(buf->data);
8555 static void free_trace_buffers(struct trace_array *tr)
8560 free_trace_buffer(&tr->array_buffer);
8562 #ifdef CONFIG_TRACER_MAX_TRACE
8563 free_trace_buffer(&tr->max_buffer);
8567 static void init_trace_flags_index(struct trace_array *tr)
8571 /* Used by the trace options files */
8572 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8573 tr->trace_flags_index[i] = i;
8576 static void __update_tracer_options(struct trace_array *tr)
8580 for (t = trace_types; t; t = t->next)
8581 add_tracer_options(tr, t);
8584 static void update_tracer_options(struct trace_array *tr)
8586 mutex_lock(&trace_types_lock);
8587 __update_tracer_options(tr);
8588 mutex_unlock(&trace_types_lock);
8591 /* Must have trace_types_lock held */
8592 struct trace_array *trace_array_find(const char *instance)
8594 struct trace_array *tr, *found = NULL;
8596 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8597 if (tr->name && strcmp(tr->name, instance) == 0) {
8606 struct trace_array *trace_array_find_get(const char *instance)
8608 struct trace_array *tr;
8610 mutex_lock(&trace_types_lock);
8611 tr = trace_array_find(instance);
8614 mutex_unlock(&trace_types_lock);
8619 static struct trace_array *trace_array_create(const char *name)
8621 struct trace_array *tr;
8625 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8627 return ERR_PTR(ret);
8629 tr->name = kstrdup(name, GFP_KERNEL);
8633 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8636 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8638 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8640 raw_spin_lock_init(&tr->start_lock);
8642 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8644 tr->current_trace = &nop_trace;
8646 INIT_LIST_HEAD(&tr->systems);
8647 INIT_LIST_HEAD(&tr->events);
8648 INIT_LIST_HEAD(&tr->hist_vars);
8649 INIT_LIST_HEAD(&tr->err_log);
8651 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8654 tr->dir = tracefs_create_dir(name, trace_instance_dir);
8658 ret = event_trace_add_tracer(tr->dir, tr);
8660 tracefs_remove(tr->dir);
8664 ftrace_init_trace_array(tr);
8666 init_tracer_tracefs(tr, tr->dir);
8667 init_trace_flags_index(tr);
8668 __update_tracer_options(tr);
8670 list_add(&tr->list, &ftrace_trace_arrays);
8678 free_trace_buffers(tr);
8679 free_cpumask_var(tr->tracing_cpumask);
8683 return ERR_PTR(ret);
8686 static int instance_mkdir(const char *name)
8688 struct trace_array *tr;
8691 mutex_lock(&event_mutex);
8692 mutex_lock(&trace_types_lock);
8695 if (trace_array_find(name))
8698 tr = trace_array_create(name);
8700 ret = PTR_ERR_OR_ZERO(tr);
8703 mutex_unlock(&trace_types_lock);
8704 mutex_unlock(&event_mutex);
8709 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
8710 * @name: The name of the trace array to be looked up/created.
8712 * Returns pointer to trace array with given name.
8713 * NULL, if it cannot be created.
8715 * NOTE: This function increments the reference counter associated with the
8716 * trace array returned. This makes sure it cannot be freed while in use.
8717 * Use trace_array_put() once the trace array is no longer needed.
8718 * If the trace_array is to be freed, trace_array_destroy() needs to
8719 * be called after the trace_array_put(), or simply let user space delete
8720 * it from the tracefs instances directory. But until the
8721 * trace_array_put() is called, user space can not delete it.
8724 struct trace_array *trace_array_get_by_name(const char *name)
8726 struct trace_array *tr;
8728 mutex_lock(&event_mutex);
8729 mutex_lock(&trace_types_lock);
8731 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8732 if (tr->name && strcmp(tr->name, name) == 0)
8736 tr = trace_array_create(name);
8744 mutex_unlock(&trace_types_lock);
8745 mutex_unlock(&event_mutex);
8748 EXPORT_SYMBOL_GPL(trace_array_get_by_name);
8750 static int __remove_instance(struct trace_array *tr)
8754 /* Reference counter for a newly created trace array = 1. */
8755 if (tr->ref > 1 || (tr->current_trace && tr->current_trace->ref))
8758 list_del(&tr->list);
8760 /* Disable all the flags that were enabled coming in */
8761 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8762 if ((1 << i) & ZEROED_TRACE_FLAGS)
8763 set_tracer_flag(tr, 1 << i, 0);
8766 tracing_set_nop(tr);
8767 clear_ftrace_function_probes(tr);
8768 event_trace_del_tracer(tr);
8769 ftrace_clear_pids(tr);
8770 ftrace_destroy_function_files(tr);
8771 tracefs_remove(tr->dir);
8772 free_trace_buffers(tr);
8774 for (i = 0; i < tr->nr_topts; i++) {
8775 kfree(tr->topts[i].topts);
8779 free_cpumask_var(tr->tracing_cpumask);
8787 int trace_array_destroy(struct trace_array *this_tr)
8789 struct trace_array *tr;
8795 mutex_lock(&event_mutex);
8796 mutex_lock(&trace_types_lock);
8800 /* Making sure trace array exists before destroying it. */
8801 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8802 if (tr == this_tr) {
8803 ret = __remove_instance(tr);
8808 mutex_unlock(&trace_types_lock);
8809 mutex_unlock(&event_mutex);
8813 EXPORT_SYMBOL_GPL(trace_array_destroy);
8815 static int instance_rmdir(const char *name)
8817 struct trace_array *tr;
8820 mutex_lock(&event_mutex);
8821 mutex_lock(&trace_types_lock);
8824 tr = trace_array_find(name);
8826 ret = __remove_instance(tr);
8828 mutex_unlock(&trace_types_lock);
8829 mutex_unlock(&event_mutex);
8834 static __init void create_trace_instances(struct dentry *d_tracer)
8836 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8839 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
8844 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8846 struct trace_event_file *file;
8849 trace_create_file("available_tracers", 0444, d_tracer,
8850 tr, &show_traces_fops);
8852 trace_create_file("current_tracer", 0644, d_tracer,
8853 tr, &set_tracer_fops);
8855 trace_create_file("tracing_cpumask", 0644, d_tracer,
8856 tr, &tracing_cpumask_fops);
8858 trace_create_file("trace_options", 0644, d_tracer,
8859 tr, &tracing_iter_fops);
8861 trace_create_file("trace", 0644, d_tracer,
8864 trace_create_file("trace_pipe", 0444, d_tracer,
8865 tr, &tracing_pipe_fops);
8867 trace_create_file("buffer_size_kb", 0644, d_tracer,
8868 tr, &tracing_entries_fops);
8870 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
8871 tr, &tracing_total_entries_fops);
8873 trace_create_file("free_buffer", 0200, d_tracer,
8874 tr, &tracing_free_buffer_fops);
8876 trace_create_file("trace_marker", 0220, d_tracer,
8877 tr, &tracing_mark_fops);
8879 file = __find_event_file(tr, "ftrace", "print");
8880 if (file && file->dir)
8881 trace_create_file("trigger", 0644, file->dir, file,
8882 &event_trigger_fops);
8883 tr->trace_marker_file = file;
8885 trace_create_file("trace_marker_raw", 0220, d_tracer,
8886 tr, &tracing_mark_raw_fops);
8888 trace_create_file("trace_clock", 0644, d_tracer, tr,
8891 trace_create_file("tracing_on", 0644, d_tracer,
8892 tr, &rb_simple_fops);
8894 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8895 &trace_time_stamp_mode_fops);
8897 tr->buffer_percent = 50;
8899 trace_create_file("buffer_percent", 0444, d_tracer,
8900 tr, &buffer_percent_fops);
8902 create_trace_options_dir(tr);
8904 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
8905 trace_create_maxlat_file(tr, d_tracer);
8908 if (ftrace_create_function_files(tr, d_tracer))
8909 MEM_FAIL(1, "Could not allocate function filter files");
8911 #ifdef CONFIG_TRACER_SNAPSHOT
8912 trace_create_file("snapshot", 0644, d_tracer,
8913 tr, &snapshot_fops);
8916 trace_create_file("error_log", 0644, d_tracer,
8917 tr, &tracing_err_log_fops);
8919 for_each_tracing_cpu(cpu)
8920 tracing_init_tracefs_percpu(tr, cpu);
8922 ftrace_init_tracefs(tr, d_tracer);
8925 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
8927 struct vfsmount *mnt;
8928 struct file_system_type *type;
8931 * To maintain backward compatibility for tools that mount
8932 * debugfs to get to the tracing facility, tracefs is automatically
8933 * mounted to the debugfs/tracing directory.
8935 type = get_fs_type("tracefs");
8938 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
8939 put_filesystem(type);
8948 * tracing_init_dentry - initialize top level trace array
8950 * This is called when creating files or directories in the tracing
8951 * directory. It is called via fs_initcall() by any of the boot up code
8952 * and expects to return the dentry of the top level tracing directory.
8954 struct dentry *tracing_init_dentry(void)
8956 struct trace_array *tr = &global_trace;
8958 if (security_locked_down(LOCKDOWN_TRACEFS)) {
8959 pr_warn("Tracing disabled due to lockdown\n");
8960 return ERR_PTR(-EPERM);
8963 /* The top level trace array uses NULL as parent */
8967 if (WARN_ON(!tracefs_initialized()) ||
8968 (IS_ENABLED(CONFIG_DEBUG_FS) &&
8969 WARN_ON(!debugfs_initialized())))
8970 return ERR_PTR(-ENODEV);
8973 * As there may still be users that expect the tracing
8974 * files to exist in debugfs/tracing, we must automount
8975 * the tracefs file system there, so older tools still
8976 * work with the newer kerenl.
8978 tr->dir = debugfs_create_automount("tracing", NULL,
8979 trace_automount, NULL);
8984 extern struct trace_eval_map *__start_ftrace_eval_maps[];
8985 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8987 static void __init trace_eval_init(void)
8991 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8992 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8995 #ifdef CONFIG_MODULES
8996 static void trace_module_add_evals(struct module *mod)
8998 if (!mod->num_trace_evals)
9002 * Modules with bad taint do not have events created, do
9003 * not bother with enums either.
9005 if (trace_module_has_bad_taint(mod))
9008 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
9011 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
9012 static void trace_module_remove_evals(struct module *mod)
9014 union trace_eval_map_item *map;
9015 union trace_eval_map_item **last = &trace_eval_maps;
9017 if (!mod->num_trace_evals)
9020 mutex_lock(&trace_eval_mutex);
9022 map = trace_eval_maps;
9025 if (map->head.mod == mod)
9027 map = trace_eval_jmp_to_tail(map);
9028 last = &map->tail.next;
9029 map = map->tail.next;
9034 *last = trace_eval_jmp_to_tail(map)->tail.next;
9037 mutex_unlock(&trace_eval_mutex);
9040 static inline void trace_module_remove_evals(struct module *mod) { }
9041 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9043 static int trace_module_notify(struct notifier_block *self,
9044 unsigned long val, void *data)
9046 struct module *mod = data;
9049 case MODULE_STATE_COMING:
9050 trace_module_add_evals(mod);
9052 case MODULE_STATE_GOING:
9053 trace_module_remove_evals(mod);
9060 static struct notifier_block trace_module_nb = {
9061 .notifier_call = trace_module_notify,
9064 #endif /* CONFIG_MODULES */
9066 static __init int tracer_init_tracefs(void)
9068 struct dentry *d_tracer;
9070 trace_access_lock_init();
9072 d_tracer = tracing_init_dentry();
9073 if (IS_ERR(d_tracer))
9078 init_tracer_tracefs(&global_trace, d_tracer);
9079 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
9081 trace_create_file("tracing_thresh", 0644, d_tracer,
9082 &global_trace, &tracing_thresh_fops);
9084 trace_create_file("README", 0444, d_tracer,
9085 NULL, &tracing_readme_fops);
9087 trace_create_file("saved_cmdlines", 0444, d_tracer,
9088 NULL, &tracing_saved_cmdlines_fops);
9090 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
9091 NULL, &tracing_saved_cmdlines_size_fops);
9093 trace_create_file("saved_tgids", 0444, d_tracer,
9094 NULL, &tracing_saved_tgids_fops);
9098 trace_create_eval_file(d_tracer);
9100 #ifdef CONFIG_MODULES
9101 register_module_notifier(&trace_module_nb);
9104 #ifdef CONFIG_DYNAMIC_FTRACE
9105 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
9106 NULL, &tracing_dyn_info_fops);
9109 create_trace_instances(d_tracer);
9111 update_tracer_options(&global_trace);
9116 static int trace_panic_handler(struct notifier_block *this,
9117 unsigned long event, void *unused)
9119 if (ftrace_dump_on_oops)
9120 ftrace_dump(ftrace_dump_on_oops);
9124 static struct notifier_block trace_panic_notifier = {
9125 .notifier_call = trace_panic_handler,
9127 .priority = 150 /* priority: INT_MAX >= x >= 0 */
9130 static int trace_die_handler(struct notifier_block *self,
9136 if (ftrace_dump_on_oops)
9137 ftrace_dump(ftrace_dump_on_oops);
9145 static struct notifier_block trace_die_notifier = {
9146 .notifier_call = trace_die_handler,
9151 * printk is set to max of 1024, we really don't need it that big.
9152 * Nothing should be printing 1000 characters anyway.
9154 #define TRACE_MAX_PRINT 1000
9157 * Define here KERN_TRACE so that we have one place to modify
9158 * it if we decide to change what log level the ftrace dump
9161 #define KERN_TRACE KERN_EMERG
9164 trace_printk_seq(struct trace_seq *s)
9166 /* Probably should print a warning here. */
9167 if (s->seq.len >= TRACE_MAX_PRINT)
9168 s->seq.len = TRACE_MAX_PRINT;
9171 * More paranoid code. Although the buffer size is set to
9172 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9173 * an extra layer of protection.
9175 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9176 s->seq.len = s->seq.size - 1;
9178 /* should be zero ended, but we are paranoid. */
9179 s->buffer[s->seq.len] = 0;
9181 printk(KERN_TRACE "%s", s->buffer);
9186 void trace_init_global_iter(struct trace_iterator *iter)
9188 iter->tr = &global_trace;
9189 iter->trace = iter->tr->current_trace;
9190 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9191 iter->array_buffer = &global_trace.array_buffer;
9193 if (iter->trace && iter->trace->open)
9194 iter->trace->open(iter);
9196 /* Annotate start of buffers if we had overruns */
9197 if (ring_buffer_overruns(iter->array_buffer->buffer))
9198 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9200 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9201 if (trace_clocks[iter->tr->clock_id].in_ns)
9202 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9205 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9207 /* use static because iter can be a bit big for the stack */
9208 static struct trace_iterator iter;
9209 static atomic_t dump_running;
9210 struct trace_array *tr = &global_trace;
9211 unsigned int old_userobj;
9212 unsigned long flags;
9215 /* Only allow one dump user at a time. */
9216 if (atomic_inc_return(&dump_running) != 1) {
9217 atomic_dec(&dump_running);
9222 * Always turn off tracing when we dump.
9223 * We don't need to show trace output of what happens
9224 * between multiple crashes.
9226 * If the user does a sysrq-z, then they can re-enable
9227 * tracing with echo 1 > tracing_on.
9231 local_irq_save(flags);
9232 printk_nmi_direct_enter();
9234 /* Simulate the iterator */
9235 trace_init_global_iter(&iter);
9236 /* Can not use kmalloc for iter.temp */
9237 iter.temp = static_temp_buf;
9238 iter.temp_size = STATIC_TEMP_BUF_SIZE;
9240 for_each_tracing_cpu(cpu) {
9241 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9244 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9246 /* don't look at user memory in panic mode */
9247 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9249 switch (oops_dump_mode) {
9251 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9254 iter.cpu_file = raw_smp_processor_id();
9259 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9260 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9263 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9265 /* Did function tracer already get disabled? */
9266 if (ftrace_is_dead()) {
9267 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9268 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9272 * We need to stop all tracing on all CPUS to read the
9273 * the next buffer. This is a bit expensive, but is
9274 * not done often. We fill all what we can read,
9275 * and then release the locks again.
9278 while (!trace_empty(&iter)) {
9281 printk(KERN_TRACE "---------------------------------\n");
9285 trace_iterator_reset(&iter);
9286 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9288 if (trace_find_next_entry_inc(&iter) != NULL) {
9291 ret = print_trace_line(&iter);
9292 if (ret != TRACE_TYPE_NO_CONSUME)
9293 trace_consume(&iter);
9295 touch_nmi_watchdog();
9297 trace_printk_seq(&iter.seq);
9301 printk(KERN_TRACE " (ftrace buffer empty)\n");
9303 printk(KERN_TRACE "---------------------------------\n");
9306 tr->trace_flags |= old_userobj;
9308 for_each_tracing_cpu(cpu) {
9309 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9311 atomic_dec(&dump_running);
9312 printk_nmi_direct_exit();
9313 local_irq_restore(flags);
9315 EXPORT_SYMBOL_GPL(ftrace_dump);
9317 int trace_run_command(const char *buf, int (*createfn)(int, char **))
9324 argv = argv_split(GFP_KERNEL, buf, &argc);
9329 ret = createfn(argc, argv);
9336 #define WRITE_BUFSIZE 4096
9338 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9339 size_t count, loff_t *ppos,
9340 int (*createfn)(int, char **))
9342 char *kbuf, *buf, *tmp;
9347 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9351 while (done < count) {
9352 size = count - done;
9354 if (size >= WRITE_BUFSIZE)
9355 size = WRITE_BUFSIZE - 1;
9357 if (copy_from_user(kbuf, buffer + done, size)) {
9364 tmp = strchr(buf, '\n');
9367 size = tmp - buf + 1;
9370 if (done + size < count) {
9373 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9374 pr_warn("Line length is too long: Should be less than %d\n",
9382 /* Remove comments */
9383 tmp = strchr(buf, '#');
9388 ret = trace_run_command(buf, createfn);
9393 } while (done < count);
9403 __init static int tracer_alloc_buffers(void)
9409 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9410 pr_warn("Tracing disabled due to lockdown\n");
9415 * Make sure we don't accidently add more trace options
9416 * than we have bits for.
9418 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9420 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9423 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9424 goto out_free_buffer_mask;
9426 /* Only allocate trace_printk buffers if a trace_printk exists */
9427 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
9428 /* Must be called before global_trace.buffer is allocated */
9429 trace_printk_init_buffers();
9431 /* To save memory, keep the ring buffer size to its minimum */
9432 if (ring_buffer_expanded)
9433 ring_buf_size = trace_buf_size;
9437 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9438 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9440 raw_spin_lock_init(&global_trace.start_lock);
9443 * The prepare callbacks allocates some memory for the ring buffer. We
9444 * don't free the buffer if the if the CPU goes down. If we were to free
9445 * the buffer, then the user would lose any trace that was in the
9446 * buffer. The memory will be removed once the "instance" is removed.
9448 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9449 "trace/RB:preapre", trace_rb_cpu_prepare,
9452 goto out_free_cpumask;
9453 /* Used for event triggers */
9455 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9457 goto out_rm_hp_state;
9459 if (trace_create_savedcmd() < 0)
9460 goto out_free_temp_buffer;
9462 /* TODO: make the number of buffers hot pluggable with CPUS */
9463 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9464 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
9465 goto out_free_savedcmd;
9468 if (global_trace.buffer_disabled)
9471 if (trace_boot_clock) {
9472 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9474 pr_warn("Trace clock %s not defined, going back to default\n",
9479 * register_tracer() might reference current_trace, so it
9480 * needs to be set before we register anything. This is
9481 * just a bootstrap of current_trace anyway.
9483 global_trace.current_trace = &nop_trace;
9485 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9487 ftrace_init_global_array_ops(&global_trace);
9489 init_trace_flags_index(&global_trace);
9491 register_tracer(&nop_trace);
9493 /* Function tracing may start here (via kernel command line) */
9494 init_function_trace();
9496 /* All seems OK, enable tracing */
9497 tracing_disabled = 0;
9499 atomic_notifier_chain_register(&panic_notifier_list,
9500 &trace_panic_notifier);
9502 register_die_notifier(&trace_die_notifier);
9504 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9506 INIT_LIST_HEAD(&global_trace.systems);
9507 INIT_LIST_HEAD(&global_trace.events);
9508 INIT_LIST_HEAD(&global_trace.hist_vars);
9509 INIT_LIST_HEAD(&global_trace.err_log);
9510 list_add(&global_trace.list, &ftrace_trace_arrays);
9512 apply_trace_boot_options();
9514 register_snapshot_cmd();
9519 free_saved_cmdlines_buffer(savedcmd);
9520 out_free_temp_buffer:
9521 ring_buffer_free(temp_buffer);
9523 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9525 free_cpumask_var(global_trace.tracing_cpumask);
9526 out_free_buffer_mask:
9527 free_cpumask_var(tracing_buffer_mask);
9532 void __init early_trace_init(void)
9534 if (tracepoint_printk) {
9535 tracepoint_print_iter =
9536 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
9537 if (MEM_FAIL(!tracepoint_print_iter,
9538 "Failed to allocate trace iterator\n"))
9539 tracepoint_printk = 0;
9541 static_key_enable(&tracepoint_printk_key.key);
9543 tracer_alloc_buffers();
9546 void __init trace_init(void)
9551 __init static int clear_boot_tracer(void)
9554 * The default tracer at boot buffer is an init section.
9555 * This function is called in lateinit. If we did not
9556 * find the boot tracer, then clear it out, to prevent
9557 * later registration from accessing the buffer that is
9558 * about to be freed.
9560 if (!default_bootup_tracer)
9563 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
9564 default_bootup_tracer);
9565 default_bootup_tracer = NULL;
9570 fs_initcall(tracer_init_tracefs);
9571 late_initcall_sync(clear_boot_tracer);
9573 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9574 __init static int tracing_set_default_clock(void)
9576 /* sched_clock_stable() is determined in late_initcall */
9577 if (!trace_boot_clock && !sched_clock_stable()) {
9578 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9579 pr_warn("Can not set tracing clock due to lockdown\n");
9584 "Unstable clock detected, switching default tracing clock to \"global\"\n"
9585 "If you want to keep using the local clock, then add:\n"
9586 " \"trace_clock=local\"\n"
9587 "on the kernel command line\n");
9588 tracing_set_clock(&global_trace, "global");
9593 late_initcall_sync(tracing_set_default_clock);