1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer based function tracer
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/security.h>
21 #include <linux/seq_file.h>
22 #include <linux/notifier.h>
23 #include <linux/irqflags.h>
24 #include <linux/debugfs.h>
25 #include <linux/tracefs.h>
26 #include <linux/pagemap.h>
27 #include <linux/hardirq.h>
28 #include <linux/linkage.h>
29 #include <linux/uaccess.h>
30 #include <linux/vmalloc.h>
31 #include <linux/ftrace.h>
32 #include <linux/module.h>
33 #include <linux/percpu.h>
34 #include <linux/splice.h>
35 #include <linux/kdebug.h>
36 #include <linux/string.h>
37 #include <linux/mount.h>
38 #include <linux/rwsem.h>
39 #include <linux/slab.h>
40 #include <linux/ctype.h>
41 #include <linux/init.h>
42 #include <linux/poll.h>
43 #include <linux/nmi.h>
45 #include <linux/trace.h>
46 #include <linux/sched/clock.h>
47 #include <linux/sched/rt.h>
48 #include <linux/fsnotify.h>
49 #include <linux/irq_work.h>
50 #include <linux/workqueue.h>
53 #include "trace_output.h"
56 * On boot up, the ring buffer is set to the minimum size, so that
57 * we do not waste memory on systems that are not using tracing.
59 bool ring_buffer_expanded;
62 * We need to change this state when a selftest is running.
63 * A selftest will lurk into the ring-buffer to count the
64 * entries inserted during the selftest although some concurrent
65 * insertions into the ring-buffer such as trace_printk could occurred
66 * at the same time, giving false positive or negative results.
68 static bool __read_mostly tracing_selftest_running;
71 * If a tracer is running, we do not want to run SELFTEST.
73 bool __read_mostly tracing_selftest_disabled;
75 /* Pipe tracepoints to printk */
76 struct trace_iterator *tracepoint_print_iter;
77 int tracepoint_printk;
78 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
80 /* For tracers that don't implement custom flags */
81 static struct tracer_opt dummy_tracer_opt[] = {
86 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
92 * To prevent the comm cache from being overwritten when no
93 * tracing is active, only save the comm when a trace event
96 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
99 * Kill all tracing for good (never come back).
100 * It is initialized to 1 but will turn to zero if the initialization
101 * of the tracer is successful. But that is the only place that sets
104 static int tracing_disabled = 1;
106 cpumask_var_t __read_mostly tracing_buffer_mask;
109 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
112 * is set, then ftrace_dump is called. This will output the contents
113 * of the ftrace buffers to the console. This is very useful for
114 * capturing traces that lead to crashes and outputing it to a
117 * It is default off, but you can enable it with either specifying
118 * "ftrace_dump_on_oops" in the kernel command line, or setting
119 * /proc/sys/kernel/ftrace_dump_on_oops
120 * Set 1 if you want to dump buffers of all CPUs
121 * Set 2 if you want to dump the buffer of the CPU that triggered oops
124 enum ftrace_dump_mode ftrace_dump_on_oops;
126 /* When set, tracing will stop when a WARN*() is hit */
127 int __disable_trace_on_warning;
129 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
130 /* Map of enums to their values, for "eval_map" file */
131 struct trace_eval_map_head {
133 unsigned long length;
136 union trace_eval_map_item;
138 struct trace_eval_map_tail {
140 * "end" is first and points to NULL as it must be different
141 * than "mod" or "eval_string"
143 union trace_eval_map_item *next;
144 const char *end; /* points to NULL */
147 static DEFINE_MUTEX(trace_eval_mutex);
150 * The trace_eval_maps are saved in an array with two extra elements,
151 * one at the beginning, and one at the end. The beginning item contains
152 * the count of the saved maps (head.length), and the module they
153 * belong to if not built in (head.mod). The ending item contains a
154 * pointer to the next array of saved eval_map items.
156 union trace_eval_map_item {
157 struct trace_eval_map map;
158 struct trace_eval_map_head head;
159 struct trace_eval_map_tail tail;
162 static union trace_eval_map_item *trace_eval_maps;
163 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
165 int tracing_set_tracer(struct trace_array *tr, const char *buf);
166 static void ftrace_trace_userstack(struct trace_buffer *buffer,
167 unsigned long flags, int pc);
169 #define MAX_TRACER_SIZE 100
170 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
171 static char *default_bootup_tracer;
173 static bool allocate_snapshot;
175 static int __init set_cmdline_ftrace(char *str)
177 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
178 default_bootup_tracer = bootup_tracer_buf;
179 /* We are using ftrace early, expand it */
180 ring_buffer_expanded = true;
183 __setup("ftrace=", set_cmdline_ftrace);
185 static int __init set_ftrace_dump_on_oops(char *str)
187 if (*str++ != '=' || !*str) {
188 ftrace_dump_on_oops = DUMP_ALL;
192 if (!strcmp("orig_cpu", str)) {
193 ftrace_dump_on_oops = DUMP_ORIG;
199 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
201 static int __init stop_trace_on_warning(char *str)
203 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
204 __disable_trace_on_warning = 1;
207 __setup("traceoff_on_warning", stop_trace_on_warning);
209 static int __init boot_alloc_snapshot(char *str)
211 allocate_snapshot = true;
212 /* We also need the main ring buffer expanded */
213 ring_buffer_expanded = true;
216 __setup("alloc_snapshot", boot_alloc_snapshot);
219 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
221 static int __init set_trace_boot_options(char *str)
223 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
226 __setup("trace_options=", set_trace_boot_options);
228 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
229 static char *trace_boot_clock __initdata;
231 static int __init set_trace_boot_clock(char *str)
233 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
234 trace_boot_clock = trace_boot_clock_buf;
237 __setup("trace_clock=", set_trace_boot_clock);
239 static int __init set_tracepoint_printk(char *str)
241 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
242 tracepoint_printk = 1;
245 __setup("tp_printk", set_tracepoint_printk);
247 unsigned long long ns2usecs(u64 nsec)
254 /* trace_flags holds trace_options default values */
255 #define TRACE_DEFAULT_FLAGS \
256 (FUNCTION_DEFAULT_FLAGS | \
257 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
258 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
259 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
260 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
262 /* trace_options that are only supported by global_trace */
263 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
264 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
266 /* trace_flags that are default zero for instances */
267 #define ZEROED_TRACE_FLAGS \
268 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
271 * The global_trace is the descriptor that holds the top-level tracing
272 * buffers for the live tracing.
274 static struct trace_array global_trace = {
275 .trace_flags = TRACE_DEFAULT_FLAGS,
278 LIST_HEAD(ftrace_trace_arrays);
280 int trace_array_get(struct trace_array *this_tr)
282 struct trace_array *tr;
285 mutex_lock(&trace_types_lock);
286 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
293 mutex_unlock(&trace_types_lock);
298 static void __trace_array_put(struct trace_array *this_tr)
300 WARN_ON(!this_tr->ref);
305 * trace_array_put - Decrement the reference counter for this trace array.
307 * NOTE: Use this when we no longer need the trace array returned by
308 * trace_array_get_by_name(). This ensures the trace array can be later
312 void trace_array_put(struct trace_array *this_tr)
317 mutex_lock(&trace_types_lock);
318 __trace_array_put(this_tr);
319 mutex_unlock(&trace_types_lock);
321 EXPORT_SYMBOL_GPL(trace_array_put);
323 int tracing_check_open_get_tr(struct trace_array *tr)
327 ret = security_locked_down(LOCKDOWN_TRACEFS);
331 if (tracing_disabled)
334 if (tr && trace_array_get(tr) < 0)
340 int call_filter_check_discard(struct trace_event_call *call, void *rec,
341 struct trace_buffer *buffer,
342 struct ring_buffer_event *event)
344 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
345 !filter_match_preds(call->filter, rec)) {
346 __trace_event_discard_commit(buffer, event);
353 void trace_free_pid_list(struct trace_pid_list *pid_list)
355 vfree(pid_list->pids);
360 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
361 * @filtered_pids: The list of pids to check
362 * @search_pid: The PID to find in @filtered_pids
364 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
367 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
370 * If pid_max changed after filtered_pids was created, we
371 * by default ignore all pids greater than the previous pid_max.
373 if (search_pid >= filtered_pids->pid_max)
376 return test_bit(search_pid, filtered_pids->pids);
380 * trace_ignore_this_task - should a task be ignored for tracing
381 * @filtered_pids: The list of pids to check
382 * @task: The task that should be ignored if not filtered
384 * Checks if @task should be traced or not from @filtered_pids.
385 * Returns true if @task should *NOT* be traced.
386 * Returns false if @task should be traced.
389 trace_ignore_this_task(struct trace_pid_list *filtered_pids,
390 struct trace_pid_list *filtered_no_pids,
391 struct task_struct *task)
394 * If filterd_no_pids is not empty, and the task's pid is listed
395 * in filtered_no_pids, then return true.
396 * Otherwise, if filtered_pids is empty, that means we can
397 * trace all tasks. If it has content, then only trace pids
398 * within filtered_pids.
401 return (filtered_pids &&
402 !trace_find_filtered_pid(filtered_pids, task->pid)) ||
404 trace_find_filtered_pid(filtered_no_pids, task->pid));
408 * trace_filter_add_remove_task - Add or remove a task from a pid_list
409 * @pid_list: The list to modify
410 * @self: The current task for fork or NULL for exit
411 * @task: The task to add or remove
413 * If adding a task, if @self is defined, the task is only added if @self
414 * is also included in @pid_list. This happens on fork and tasks should
415 * only be added when the parent is listed. If @self is NULL, then the
416 * @task pid will be removed from the list, which would happen on exit
419 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
420 struct task_struct *self,
421 struct task_struct *task)
426 /* For forks, we only add if the forking task is listed */
428 if (!trace_find_filtered_pid(pid_list, self->pid))
432 /* Sorry, but we don't support pid_max changing after setting */
433 if (task->pid >= pid_list->pid_max)
436 /* "self" is set for forks, and NULL for exits */
438 set_bit(task->pid, pid_list->pids);
440 clear_bit(task->pid, pid_list->pids);
444 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
445 * @pid_list: The pid list to show
446 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
447 * @pos: The position of the file
449 * This is used by the seq_file "next" operation to iterate the pids
450 * listed in a trace_pid_list structure.
452 * Returns the pid+1 as we want to display pid of zero, but NULL would
453 * stop the iteration.
455 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
457 unsigned long pid = (unsigned long)v;
461 /* pid already is +1 of the actual prevous bit */
462 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
464 /* Return pid + 1 to allow zero to be represented */
465 if (pid < pid_list->pid_max)
466 return (void *)(pid + 1);
472 * trace_pid_start - Used for seq_file to start reading pid lists
473 * @pid_list: The pid list to show
474 * @pos: The position of the file
476 * This is used by seq_file "start" operation to start the iteration
479 * Returns the pid+1 as we want to display pid of zero, but NULL would
480 * stop the iteration.
482 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
487 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
488 if (pid >= pid_list->pid_max)
491 /* Return pid + 1 so that zero can be the exit value */
492 for (pid++; pid && l < *pos;
493 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
499 * trace_pid_show - show the current pid in seq_file processing
500 * @m: The seq_file structure to write into
501 * @v: A void pointer of the pid (+1) value to display
503 * Can be directly used by seq_file operations to display the current
506 int trace_pid_show(struct seq_file *m, void *v)
508 unsigned long pid = (unsigned long)v - 1;
510 seq_printf(m, "%lu\n", pid);
514 /* 128 should be much more than enough */
515 #define PID_BUF_SIZE 127
517 int trace_pid_write(struct trace_pid_list *filtered_pids,
518 struct trace_pid_list **new_pid_list,
519 const char __user *ubuf, size_t cnt)
521 struct trace_pid_list *pid_list;
522 struct trace_parser parser;
530 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
534 * Always recreate a new array. The write is an all or nothing
535 * operation. Always create a new array when adding new pids by
536 * the user. If the operation fails, then the current list is
539 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
541 trace_parser_put(&parser);
545 pid_list->pid_max = READ_ONCE(pid_max);
547 /* Only truncating will shrink pid_max */
548 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
549 pid_list->pid_max = filtered_pids->pid_max;
551 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
552 if (!pid_list->pids) {
553 trace_parser_put(&parser);
559 /* copy the current bits to the new max */
560 for_each_set_bit(pid, filtered_pids->pids,
561 filtered_pids->pid_max) {
562 set_bit(pid, pid_list->pids);
571 ret = trace_get_user(&parser, ubuf, cnt, &pos);
572 if (ret < 0 || !trace_parser_loaded(&parser))
580 if (kstrtoul(parser.buffer, 0, &val))
582 if (val >= pid_list->pid_max)
587 set_bit(pid, pid_list->pids);
590 trace_parser_clear(&parser);
593 trace_parser_put(&parser);
596 trace_free_pid_list(pid_list);
601 /* Cleared the list of pids */
602 trace_free_pid_list(pid_list);
607 *new_pid_list = pid_list;
612 static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
616 /* Early boot up does not have a buffer yet */
618 return trace_clock_local();
620 ts = ring_buffer_time_stamp(buf->buffer, cpu);
621 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
626 u64 ftrace_now(int cpu)
628 return buffer_ftrace_now(&global_trace.array_buffer, cpu);
632 * tracing_is_enabled - Show if global_trace has been disabled
634 * Shows if the global trace has been enabled or not. It uses the
635 * mirror flag "buffer_disabled" to be used in fast paths such as for
636 * the irqsoff tracer. But it may be inaccurate due to races. If you
637 * need to know the accurate state, use tracing_is_on() which is a little
638 * slower, but accurate.
640 int tracing_is_enabled(void)
643 * For quick access (irqsoff uses this in fast path), just
644 * return the mirror variable of the state of the ring buffer.
645 * It's a little racy, but we don't really care.
648 return !global_trace.buffer_disabled;
652 * trace_buf_size is the size in bytes that is allocated
653 * for a buffer. Note, the number of bytes is always rounded
656 * This number is purposely set to a low number of 16384.
657 * If the dump on oops happens, it will be much appreciated
658 * to not have to wait for all that output. Anyway this can be
659 * boot time and run time configurable.
661 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
663 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
665 /* trace_types holds a link list of available tracers. */
666 static struct tracer *trace_types __read_mostly;
669 * trace_types_lock is used to protect the trace_types list.
671 DEFINE_MUTEX(trace_types_lock);
674 * serialize the access of the ring buffer
676 * ring buffer serializes readers, but it is low level protection.
677 * The validity of the events (which returns by ring_buffer_peek() ..etc)
678 * are not protected by ring buffer.
680 * The content of events may become garbage if we allow other process consumes
681 * these events concurrently:
682 * A) the page of the consumed events may become a normal page
683 * (not reader page) in ring buffer, and this page will be rewrited
684 * by events producer.
685 * B) The page of the consumed events may become a page for splice_read,
686 * and this page will be returned to system.
688 * These primitives allow multi process access to different cpu ring buffer
691 * These primitives don't distinguish read-only and read-consume access.
692 * Multi read-only access are also serialized.
696 static DECLARE_RWSEM(all_cpu_access_lock);
697 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
699 static inline void trace_access_lock(int cpu)
701 if (cpu == RING_BUFFER_ALL_CPUS) {
702 /* gain it for accessing the whole ring buffer. */
703 down_write(&all_cpu_access_lock);
705 /* gain it for accessing a cpu ring buffer. */
707 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
708 down_read(&all_cpu_access_lock);
710 /* Secondly block other access to this @cpu ring buffer. */
711 mutex_lock(&per_cpu(cpu_access_lock, cpu));
715 static inline void trace_access_unlock(int cpu)
717 if (cpu == RING_BUFFER_ALL_CPUS) {
718 up_write(&all_cpu_access_lock);
720 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
721 up_read(&all_cpu_access_lock);
725 static inline void trace_access_lock_init(void)
729 for_each_possible_cpu(cpu)
730 mutex_init(&per_cpu(cpu_access_lock, cpu));
735 static DEFINE_MUTEX(access_lock);
737 static inline void trace_access_lock(int cpu)
740 mutex_lock(&access_lock);
743 static inline void trace_access_unlock(int cpu)
746 mutex_unlock(&access_lock);
749 static inline void trace_access_lock_init(void)
755 #ifdef CONFIG_STACKTRACE
756 static void __ftrace_trace_stack(struct trace_buffer *buffer,
758 int skip, int pc, struct pt_regs *regs);
759 static inline void ftrace_trace_stack(struct trace_array *tr,
760 struct trace_buffer *buffer,
762 int skip, int pc, struct pt_regs *regs);
765 static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
767 int skip, int pc, struct pt_regs *regs)
770 static inline void ftrace_trace_stack(struct trace_array *tr,
771 struct trace_buffer *buffer,
773 int skip, int pc, struct pt_regs *regs)
779 static __always_inline void
780 trace_event_setup(struct ring_buffer_event *event,
781 int type, unsigned long flags, int pc)
783 struct trace_entry *ent = ring_buffer_event_data(event);
785 tracing_generic_entry_update(ent, type, flags, pc);
788 static __always_inline struct ring_buffer_event *
789 __trace_buffer_lock_reserve(struct trace_buffer *buffer,
792 unsigned long flags, int pc)
794 struct ring_buffer_event *event;
796 event = ring_buffer_lock_reserve(buffer, len);
798 trace_event_setup(event, type, flags, pc);
803 void tracer_tracing_on(struct trace_array *tr)
805 if (tr->array_buffer.buffer)
806 ring_buffer_record_on(tr->array_buffer.buffer);
808 * This flag is looked at when buffers haven't been allocated
809 * yet, or by some tracers (like irqsoff), that just want to
810 * know if the ring buffer has been disabled, but it can handle
811 * races of where it gets disabled but we still do a record.
812 * As the check is in the fast path of the tracers, it is more
813 * important to be fast than accurate.
815 tr->buffer_disabled = 0;
816 /* Make the flag seen by readers */
821 * tracing_on - enable tracing buffers
823 * This function enables tracing buffers that may have been
824 * disabled with tracing_off.
826 void tracing_on(void)
828 tracer_tracing_on(&global_trace);
830 EXPORT_SYMBOL_GPL(tracing_on);
833 static __always_inline void
834 __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
836 __this_cpu_write(trace_taskinfo_save, true);
838 /* If this is the temp buffer, we need to commit fully */
839 if (this_cpu_read(trace_buffered_event) == event) {
840 /* Length is in event->array[0] */
841 ring_buffer_write(buffer, event->array[0], &event->array[1]);
842 /* Release the temp buffer */
843 this_cpu_dec(trace_buffered_event_cnt);
845 ring_buffer_unlock_commit(buffer, event);
849 * __trace_puts - write a constant string into the trace buffer.
850 * @ip: The address of the caller
851 * @str: The constant string to write
852 * @size: The size of the string.
854 int __trace_puts(unsigned long ip, const char *str, int size)
856 struct ring_buffer_event *event;
857 struct trace_buffer *buffer;
858 struct print_entry *entry;
859 unsigned long irq_flags;
863 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
866 pc = preempt_count();
868 if (unlikely(tracing_selftest_running || tracing_disabled))
871 alloc = sizeof(*entry) + size + 2; /* possible \n added */
873 local_save_flags(irq_flags);
874 buffer = global_trace.array_buffer.buffer;
875 ring_buffer_nest_start(buffer);
876 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
883 entry = ring_buffer_event_data(event);
886 memcpy(&entry->buf, str, size);
888 /* Add a newline if necessary */
889 if (entry->buf[size - 1] != '\n') {
890 entry->buf[size] = '\n';
891 entry->buf[size + 1] = '\0';
893 entry->buf[size] = '\0';
895 __buffer_unlock_commit(buffer, event);
896 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
898 ring_buffer_nest_end(buffer);
901 EXPORT_SYMBOL_GPL(__trace_puts);
904 * __trace_bputs - write the pointer to a constant string into trace buffer
905 * @ip: The address of the caller
906 * @str: The constant string to write to the buffer to
908 int __trace_bputs(unsigned long ip, const char *str)
910 struct ring_buffer_event *event;
911 struct trace_buffer *buffer;
912 struct bputs_entry *entry;
913 unsigned long irq_flags;
914 int size = sizeof(struct bputs_entry);
918 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
921 pc = preempt_count();
923 if (unlikely(tracing_selftest_running || tracing_disabled))
926 local_save_flags(irq_flags);
927 buffer = global_trace.array_buffer.buffer;
929 ring_buffer_nest_start(buffer);
930 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
935 entry = ring_buffer_event_data(event);
939 __buffer_unlock_commit(buffer, event);
940 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
944 ring_buffer_nest_end(buffer);
947 EXPORT_SYMBOL_GPL(__trace_bputs);
949 #ifdef CONFIG_TRACER_SNAPSHOT
950 static void tracing_snapshot_instance_cond(struct trace_array *tr,
953 struct tracer *tracer = tr->current_trace;
957 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
958 internal_trace_puts("*** snapshot is being ignored ***\n");
962 if (!tr->allocated_snapshot) {
963 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
964 internal_trace_puts("*** stopping trace here! ***\n");
969 /* Note, snapshot can not be used when the tracer uses it */
970 if (tracer->use_max_tr) {
971 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
972 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
976 local_irq_save(flags);
977 update_max_tr(tr, current, smp_processor_id(), cond_data);
978 local_irq_restore(flags);
981 void tracing_snapshot_instance(struct trace_array *tr)
983 tracing_snapshot_instance_cond(tr, NULL);
987 * tracing_snapshot - take a snapshot of the current buffer.
989 * This causes a swap between the snapshot buffer and the current live
990 * tracing buffer. You can use this to take snapshots of the live
991 * trace when some condition is triggered, but continue to trace.
993 * Note, make sure to allocate the snapshot with either
994 * a tracing_snapshot_alloc(), or by doing it manually
995 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
997 * If the snapshot buffer is not allocated, it will stop tracing.
998 * Basically making a permanent snapshot.
1000 void tracing_snapshot(void)
1002 struct trace_array *tr = &global_trace;
1004 tracing_snapshot_instance(tr);
1006 EXPORT_SYMBOL_GPL(tracing_snapshot);
1009 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1010 * @tr: The tracing instance to snapshot
1011 * @cond_data: The data to be tested conditionally, and possibly saved
1013 * This is the same as tracing_snapshot() except that the snapshot is
1014 * conditional - the snapshot will only happen if the
1015 * cond_snapshot.update() implementation receiving the cond_data
1016 * returns true, which means that the trace array's cond_snapshot
1017 * update() operation used the cond_data to determine whether the
1018 * snapshot should be taken, and if it was, presumably saved it along
1019 * with the snapshot.
1021 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1023 tracing_snapshot_instance_cond(tr, cond_data);
1025 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1028 * tracing_snapshot_cond_data - get the user data associated with a snapshot
1029 * @tr: The tracing instance
1031 * When the user enables a conditional snapshot using
1032 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1033 * with the snapshot. This accessor is used to retrieve it.
1035 * Should not be called from cond_snapshot.update(), since it takes
1036 * the tr->max_lock lock, which the code calling
1037 * cond_snapshot.update() has already done.
1039 * Returns the cond_data associated with the trace array's snapshot.
1041 void *tracing_cond_snapshot_data(struct trace_array *tr)
1043 void *cond_data = NULL;
1045 arch_spin_lock(&tr->max_lock);
1047 if (tr->cond_snapshot)
1048 cond_data = tr->cond_snapshot->cond_data;
1050 arch_spin_unlock(&tr->max_lock);
1054 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1056 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1057 struct array_buffer *size_buf, int cpu_id);
1058 static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1060 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1064 if (!tr->allocated_snapshot) {
1066 /* allocate spare buffer */
1067 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1068 &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1072 tr->allocated_snapshot = true;
1078 static void free_snapshot(struct trace_array *tr)
1081 * We don't free the ring buffer. instead, resize it because
1082 * The max_tr ring buffer has some state (e.g. ring->clock) and
1083 * we want preserve it.
1085 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1086 set_buffer_entries(&tr->max_buffer, 1);
1087 tracing_reset_online_cpus(&tr->max_buffer);
1088 tr->allocated_snapshot = false;
1092 * tracing_alloc_snapshot - allocate snapshot buffer.
1094 * This only allocates the snapshot buffer if it isn't already
1095 * allocated - it doesn't also take a snapshot.
1097 * This is meant to be used in cases where the snapshot buffer needs
1098 * to be set up for events that can't sleep but need to be able to
1099 * trigger a snapshot.
1101 int tracing_alloc_snapshot(void)
1103 struct trace_array *tr = &global_trace;
1106 ret = tracing_alloc_snapshot_instance(tr);
1111 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1114 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1116 * This is similar to tracing_snapshot(), but it will allocate the
1117 * snapshot buffer if it isn't already allocated. Use this only
1118 * where it is safe to sleep, as the allocation may sleep.
1120 * This causes a swap between the snapshot buffer and the current live
1121 * tracing buffer. You can use this to take snapshots of the live
1122 * trace when some condition is triggered, but continue to trace.
1124 void tracing_snapshot_alloc(void)
1128 ret = tracing_alloc_snapshot();
1134 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1137 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1138 * @tr: The tracing instance
1139 * @cond_data: User data to associate with the snapshot
1140 * @update: Implementation of the cond_snapshot update function
1142 * Check whether the conditional snapshot for the given instance has
1143 * already been enabled, or if the current tracer is already using a
1144 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1145 * save the cond_data and update function inside.
1147 * Returns 0 if successful, error otherwise.
1149 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1150 cond_update_fn_t update)
1152 struct cond_snapshot *cond_snapshot;
1155 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1159 cond_snapshot->cond_data = cond_data;
1160 cond_snapshot->update = update;
1162 mutex_lock(&trace_types_lock);
1164 ret = tracing_alloc_snapshot_instance(tr);
1168 if (tr->current_trace->use_max_tr) {
1174 * The cond_snapshot can only change to NULL without the
1175 * trace_types_lock. We don't care if we race with it going
1176 * to NULL, but we want to make sure that it's not set to
1177 * something other than NULL when we get here, which we can
1178 * do safely with only holding the trace_types_lock and not
1179 * having to take the max_lock.
1181 if (tr->cond_snapshot) {
1186 arch_spin_lock(&tr->max_lock);
1187 tr->cond_snapshot = cond_snapshot;
1188 arch_spin_unlock(&tr->max_lock);
1190 mutex_unlock(&trace_types_lock);
1195 mutex_unlock(&trace_types_lock);
1196 kfree(cond_snapshot);
1199 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1202 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1203 * @tr: The tracing instance
1205 * Check whether the conditional snapshot for the given instance is
1206 * enabled; if so, free the cond_snapshot associated with it,
1207 * otherwise return -EINVAL.
1209 * Returns 0 if successful, error otherwise.
1211 int tracing_snapshot_cond_disable(struct trace_array *tr)
1215 arch_spin_lock(&tr->max_lock);
1217 if (!tr->cond_snapshot)
1220 kfree(tr->cond_snapshot);
1221 tr->cond_snapshot = NULL;
1224 arch_spin_unlock(&tr->max_lock);
1228 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1230 void tracing_snapshot(void)
1232 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1234 EXPORT_SYMBOL_GPL(tracing_snapshot);
1235 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1237 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1239 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1240 int tracing_alloc_snapshot(void)
1242 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1245 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1246 void tracing_snapshot_alloc(void)
1251 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1252 void *tracing_cond_snapshot_data(struct trace_array *tr)
1256 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1257 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1261 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1262 int tracing_snapshot_cond_disable(struct trace_array *tr)
1266 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1267 #endif /* CONFIG_TRACER_SNAPSHOT */
1269 void tracer_tracing_off(struct trace_array *tr)
1271 if (tr->array_buffer.buffer)
1272 ring_buffer_record_off(tr->array_buffer.buffer);
1274 * This flag is looked at when buffers haven't been allocated
1275 * yet, or by some tracers (like irqsoff), that just want to
1276 * know if the ring buffer has been disabled, but it can handle
1277 * races of where it gets disabled but we still do a record.
1278 * As the check is in the fast path of the tracers, it is more
1279 * important to be fast than accurate.
1281 tr->buffer_disabled = 1;
1282 /* Make the flag seen by readers */
1287 * tracing_off - turn off tracing buffers
1289 * This function stops the tracing buffers from recording data.
1290 * It does not disable any overhead the tracers themselves may
1291 * be causing. This function simply causes all recording to
1292 * the ring buffers to fail.
1294 void tracing_off(void)
1296 tracer_tracing_off(&global_trace);
1298 EXPORT_SYMBOL_GPL(tracing_off);
1300 void disable_trace_on_warning(void)
1302 if (__disable_trace_on_warning) {
1303 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
1304 "Disabling tracing due to warning\n");
1310 * tracer_tracing_is_on - show real state of ring buffer enabled
1311 * @tr : the trace array to know if ring buffer is enabled
1313 * Shows real state of the ring buffer if it is enabled or not.
1315 bool tracer_tracing_is_on(struct trace_array *tr)
1317 if (tr->array_buffer.buffer)
1318 return ring_buffer_record_is_on(tr->array_buffer.buffer);
1319 return !tr->buffer_disabled;
1323 * tracing_is_on - show state of ring buffers enabled
1325 int tracing_is_on(void)
1327 return tracer_tracing_is_on(&global_trace);
1329 EXPORT_SYMBOL_GPL(tracing_is_on);
1331 static int __init set_buf_size(char *str)
1333 unsigned long buf_size;
1337 buf_size = memparse(str, &str);
1338 /* nr_entries can not be zero */
1341 trace_buf_size = buf_size;
1344 __setup("trace_buf_size=", set_buf_size);
1346 static int __init set_tracing_thresh(char *str)
1348 unsigned long threshold;
1353 ret = kstrtoul(str, 0, &threshold);
1356 tracing_thresh = threshold * 1000;
1359 __setup("tracing_thresh=", set_tracing_thresh);
1361 unsigned long nsecs_to_usecs(unsigned long nsecs)
1363 return nsecs / 1000;
1367 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1368 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1369 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1370 * of strings in the order that the evals (enum) were defined.
1375 /* These must match the bit postions in trace_iterator_flags */
1376 static const char *trace_options[] = {
1384 int in_ns; /* is this clock in nanoseconds? */
1385 } trace_clocks[] = {
1386 { trace_clock_local, "local", 1 },
1387 { trace_clock_global, "global", 1 },
1388 { trace_clock_counter, "counter", 0 },
1389 { trace_clock_jiffies, "uptime", 0 },
1390 { trace_clock, "perf", 1 },
1391 { ktime_get_mono_fast_ns, "mono", 1 },
1392 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1393 { ktime_get_boot_fast_ns, "boot", 1 },
1397 bool trace_clock_in_ns(struct trace_array *tr)
1399 if (trace_clocks[tr->clock_id].in_ns)
1406 * trace_parser_get_init - gets the buffer for trace parser
1408 int trace_parser_get_init(struct trace_parser *parser, int size)
1410 memset(parser, 0, sizeof(*parser));
1412 parser->buffer = kmalloc(size, GFP_KERNEL);
1413 if (!parser->buffer)
1416 parser->size = size;
1421 * trace_parser_put - frees the buffer for trace parser
1423 void trace_parser_put(struct trace_parser *parser)
1425 kfree(parser->buffer);
1426 parser->buffer = NULL;
1430 * trace_get_user - reads the user input string separated by space
1431 * (matched by isspace(ch))
1433 * For each string found the 'struct trace_parser' is updated,
1434 * and the function returns.
1436 * Returns number of bytes read.
1438 * See kernel/trace/trace.h for 'struct trace_parser' details.
1440 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1441 size_t cnt, loff_t *ppos)
1448 trace_parser_clear(parser);
1450 ret = get_user(ch, ubuf++);
1458 * The parser is not finished with the last write,
1459 * continue reading the user input without skipping spaces.
1461 if (!parser->cont) {
1462 /* skip white space */
1463 while (cnt && isspace(ch)) {
1464 ret = get_user(ch, ubuf++);
1473 /* only spaces were written */
1474 if (isspace(ch) || !ch) {
1481 /* read the non-space input */
1482 while (cnt && !isspace(ch) && ch) {
1483 if (parser->idx < parser->size - 1)
1484 parser->buffer[parser->idx++] = ch;
1489 ret = get_user(ch, ubuf++);
1496 /* We either got finished input or we have to wait for another call. */
1497 if (isspace(ch) || !ch) {
1498 parser->buffer[parser->idx] = 0;
1499 parser->cont = false;
1500 } else if (parser->idx < parser->size - 1) {
1501 parser->cont = true;
1502 parser->buffer[parser->idx++] = ch;
1503 /* Make sure the parsed string always terminates with '\0'. */
1504 parser->buffer[parser->idx] = 0;
1517 /* TODO add a seq_buf_to_buffer() */
1518 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1522 if (trace_seq_used(s) <= s->seq.readpos)
1525 len = trace_seq_used(s) - s->seq.readpos;
1528 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1530 s->seq.readpos += cnt;
1534 unsigned long __read_mostly tracing_thresh;
1535 static const struct file_operations tracing_max_lat_fops;
1537 #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1538 defined(CONFIG_FSNOTIFY)
1540 static struct workqueue_struct *fsnotify_wq;
1542 static void latency_fsnotify_workfn(struct work_struct *work)
1544 struct trace_array *tr = container_of(work, struct trace_array,
1546 fsnotify(tr->d_max_latency->d_inode, FS_MODIFY,
1547 tr->d_max_latency->d_inode, FSNOTIFY_EVENT_INODE, NULL, 0);
1550 static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1552 struct trace_array *tr = container_of(iwork, struct trace_array,
1554 queue_work(fsnotify_wq, &tr->fsnotify_work);
1557 static void trace_create_maxlat_file(struct trace_array *tr,
1558 struct dentry *d_tracer)
1560 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1561 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
1562 tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
1563 d_tracer, &tr->max_latency,
1564 &tracing_max_lat_fops);
1567 __init static int latency_fsnotify_init(void)
1569 fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
1570 WQ_UNBOUND | WQ_HIGHPRI, 0);
1572 pr_err("Unable to allocate tr_max_lat_wq\n");
1578 late_initcall_sync(latency_fsnotify_init);
1580 void latency_fsnotify(struct trace_array *tr)
1585 * We cannot call queue_work(&tr->fsnotify_work) from here because it's
1586 * possible that we are called from __schedule() or do_idle(), which
1587 * could cause a deadlock.
1589 irq_work_queue(&tr->fsnotify_irqwork);
1593 * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
1594 * defined(CONFIG_FSNOTIFY)
1598 #define trace_create_maxlat_file(tr, d_tracer) \
1599 trace_create_file("tracing_max_latency", 0644, d_tracer, \
1600 &tr->max_latency, &tracing_max_lat_fops)
1604 #ifdef CONFIG_TRACER_MAX_TRACE
1606 * Copy the new maximum trace into the separate maximum-trace
1607 * structure. (this way the maximum trace is permanently saved,
1608 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1611 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1613 struct array_buffer *trace_buf = &tr->array_buffer;
1614 struct array_buffer *max_buf = &tr->max_buffer;
1615 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1616 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1619 max_buf->time_start = data->preempt_timestamp;
1621 max_data->saved_latency = tr->max_latency;
1622 max_data->critical_start = data->critical_start;
1623 max_data->critical_end = data->critical_end;
1625 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1626 max_data->pid = tsk->pid;
1628 * If tsk == current, then use current_uid(), as that does not use
1629 * RCU. The irq tracer can be called out of RCU scope.
1632 max_data->uid = current_uid();
1634 max_data->uid = task_uid(tsk);
1636 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1637 max_data->policy = tsk->policy;
1638 max_data->rt_priority = tsk->rt_priority;
1640 /* record this tasks comm */
1641 tracing_record_cmdline(tsk);
1642 latency_fsnotify(tr);
1646 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1648 * @tsk: the task with the latency
1649 * @cpu: The cpu that initiated the trace.
1650 * @cond_data: User data associated with a conditional snapshot
1652 * Flip the buffers between the @tr and the max_tr and record information
1653 * about which task was the cause of this latency.
1656 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1662 WARN_ON_ONCE(!irqs_disabled());
1664 if (!tr->allocated_snapshot) {
1665 /* Only the nop tracer should hit this when disabling */
1666 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1670 arch_spin_lock(&tr->max_lock);
1672 /* Inherit the recordable setting from array_buffer */
1673 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1674 ring_buffer_record_on(tr->max_buffer.buffer);
1676 ring_buffer_record_off(tr->max_buffer.buffer);
1678 #ifdef CONFIG_TRACER_SNAPSHOT
1679 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1682 swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1684 __update_max_tr(tr, tsk, cpu);
1687 arch_spin_unlock(&tr->max_lock);
1691 * update_max_tr_single - only copy one trace over, and reset the rest
1693 * @tsk: task with the latency
1694 * @cpu: the cpu of the buffer to copy.
1696 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1699 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1706 WARN_ON_ONCE(!irqs_disabled());
1707 if (!tr->allocated_snapshot) {
1708 /* Only the nop tracer should hit this when disabling */
1709 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1713 arch_spin_lock(&tr->max_lock);
1715 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1717 if (ret == -EBUSY) {
1719 * We failed to swap the buffer due to a commit taking
1720 * place on this CPU. We fail to record, but we reset
1721 * the max trace buffer (no one writes directly to it)
1722 * and flag that it failed.
1724 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1725 "Failed to swap buffers due to commit in progress\n");
1728 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1730 __update_max_tr(tr, tsk, cpu);
1731 arch_spin_unlock(&tr->max_lock);
1733 #endif /* CONFIG_TRACER_MAX_TRACE */
1735 static int wait_on_pipe(struct trace_iterator *iter, int full)
1737 /* Iterators are static, they should be filled or empty */
1738 if (trace_buffer_iter(iter, iter->cpu_file))
1741 return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1745 #ifdef CONFIG_FTRACE_STARTUP_TEST
1746 static bool selftests_can_run;
1748 struct trace_selftests {
1749 struct list_head list;
1750 struct tracer *type;
1753 static LIST_HEAD(postponed_selftests);
1755 static int save_selftest(struct tracer *type)
1757 struct trace_selftests *selftest;
1759 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1763 selftest->type = type;
1764 list_add(&selftest->list, &postponed_selftests);
1768 static int run_tracer_selftest(struct tracer *type)
1770 struct trace_array *tr = &global_trace;
1771 struct tracer *saved_tracer = tr->current_trace;
1774 if (!type->selftest || tracing_selftest_disabled)
1778 * If a tracer registers early in boot up (before scheduling is
1779 * initialized and such), then do not run its selftests yet.
1780 * Instead, run it a little later in the boot process.
1782 if (!selftests_can_run)
1783 return save_selftest(type);
1786 * Run a selftest on this tracer.
1787 * Here we reset the trace buffer, and set the current
1788 * tracer to be this tracer. The tracer can then run some
1789 * internal tracing to verify that everything is in order.
1790 * If we fail, we do not register this tracer.
1792 tracing_reset_online_cpus(&tr->array_buffer);
1794 tr->current_trace = type;
1796 #ifdef CONFIG_TRACER_MAX_TRACE
1797 if (type->use_max_tr) {
1798 /* If we expanded the buffers, make sure the max is expanded too */
1799 if (ring_buffer_expanded)
1800 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1801 RING_BUFFER_ALL_CPUS);
1802 tr->allocated_snapshot = true;
1806 /* the test is responsible for initializing and enabling */
1807 pr_info("Testing tracer %s: ", type->name);
1808 ret = type->selftest(type, tr);
1809 /* the test is responsible for resetting too */
1810 tr->current_trace = saved_tracer;
1812 printk(KERN_CONT "FAILED!\n");
1813 /* Add the warning after printing 'FAILED' */
1817 /* Only reset on passing, to avoid touching corrupted buffers */
1818 tracing_reset_online_cpus(&tr->array_buffer);
1820 #ifdef CONFIG_TRACER_MAX_TRACE
1821 if (type->use_max_tr) {
1822 tr->allocated_snapshot = false;
1824 /* Shrink the max buffer again */
1825 if (ring_buffer_expanded)
1826 ring_buffer_resize(tr->max_buffer.buffer, 1,
1827 RING_BUFFER_ALL_CPUS);
1831 printk(KERN_CONT "PASSED\n");
1835 static __init int init_trace_selftests(void)
1837 struct trace_selftests *p, *n;
1838 struct tracer *t, **last;
1841 selftests_can_run = true;
1843 mutex_lock(&trace_types_lock);
1845 if (list_empty(&postponed_selftests))
1848 pr_info("Running postponed tracer tests:\n");
1850 tracing_selftest_running = true;
1851 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1852 /* This loop can take minutes when sanitizers are enabled, so
1853 * lets make sure we allow RCU processing.
1856 ret = run_tracer_selftest(p->type);
1857 /* If the test fails, then warn and remove from available_tracers */
1859 WARN(1, "tracer: %s failed selftest, disabling\n",
1861 last = &trace_types;
1862 for (t = trace_types; t; t = t->next) {
1873 tracing_selftest_running = false;
1876 mutex_unlock(&trace_types_lock);
1880 core_initcall(init_trace_selftests);
1882 static inline int run_tracer_selftest(struct tracer *type)
1886 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1888 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1890 static void __init apply_trace_boot_options(void);
1893 * register_tracer - register a tracer with the ftrace system.
1894 * @type: the plugin for the tracer
1896 * Register a new plugin tracer.
1898 int __init register_tracer(struct tracer *type)
1904 pr_info("Tracer must have a name\n");
1908 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1909 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1913 if (security_locked_down(LOCKDOWN_TRACEFS)) {
1914 pr_warn("Can not register tracer %s due to lockdown\n",
1919 mutex_lock(&trace_types_lock);
1921 tracing_selftest_running = true;
1923 for (t = trace_types; t; t = t->next) {
1924 if (strcmp(type->name, t->name) == 0) {
1926 pr_info("Tracer %s already registered\n",
1933 if (!type->set_flag)
1934 type->set_flag = &dummy_set_flag;
1936 /*allocate a dummy tracer_flags*/
1937 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1942 type->flags->val = 0;
1943 type->flags->opts = dummy_tracer_opt;
1945 if (!type->flags->opts)
1946 type->flags->opts = dummy_tracer_opt;
1948 /* store the tracer for __set_tracer_option */
1949 type->flags->trace = type;
1951 ret = run_tracer_selftest(type);
1955 type->next = trace_types;
1957 add_tracer_options(&global_trace, type);
1960 tracing_selftest_running = false;
1961 mutex_unlock(&trace_types_lock);
1963 if (ret || !default_bootup_tracer)
1966 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1969 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1970 /* Do we want this tracer to start on bootup? */
1971 tracing_set_tracer(&global_trace, type->name);
1972 default_bootup_tracer = NULL;
1974 apply_trace_boot_options();
1976 /* disable other selftests, since this will break it. */
1977 tracing_selftest_disabled = true;
1978 #ifdef CONFIG_FTRACE_STARTUP_TEST
1979 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1987 static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
1989 struct trace_buffer *buffer = buf->buffer;
1994 ring_buffer_record_disable(buffer);
1996 /* Make sure all commits have finished */
1998 ring_buffer_reset_cpu(buffer, cpu);
2000 ring_buffer_record_enable(buffer);
2003 void tracing_reset_online_cpus(struct array_buffer *buf)
2005 struct trace_buffer *buffer = buf->buffer;
2010 ring_buffer_record_disable(buffer);
2012 /* Make sure all commits have finished */
2015 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
2017 ring_buffer_reset_online_cpus(buffer);
2019 ring_buffer_record_enable(buffer);
2022 /* Must have trace_types_lock held */
2023 void tracing_reset_all_online_cpus(void)
2025 struct trace_array *tr;
2027 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2028 if (!tr->clear_trace)
2030 tr->clear_trace = false;
2031 tracing_reset_online_cpus(&tr->array_buffer);
2032 #ifdef CONFIG_TRACER_MAX_TRACE
2033 tracing_reset_online_cpus(&tr->max_buffer);
2038 static int *tgid_map;
2040 #define SAVED_CMDLINES_DEFAULT 128
2041 #define NO_CMDLINE_MAP UINT_MAX
2042 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
2043 struct saved_cmdlines_buffer {
2044 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
2045 unsigned *map_cmdline_to_pid;
2046 unsigned cmdline_num;
2048 char *saved_cmdlines;
2050 static struct saved_cmdlines_buffer *savedcmd;
2052 /* temporary disable recording */
2053 static atomic_t trace_record_taskinfo_disabled __read_mostly;
2055 static inline char *get_saved_cmdlines(int idx)
2057 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
2060 static inline void set_cmdline(int idx, const char *cmdline)
2062 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
2065 static int allocate_cmdlines_buffer(unsigned int val,
2066 struct saved_cmdlines_buffer *s)
2068 s->map_cmdline_to_pid = kmalloc_array(val,
2069 sizeof(*s->map_cmdline_to_pid),
2071 if (!s->map_cmdline_to_pid)
2074 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
2075 if (!s->saved_cmdlines) {
2076 kfree(s->map_cmdline_to_pid);
2081 s->cmdline_num = val;
2082 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
2083 sizeof(s->map_pid_to_cmdline));
2084 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
2085 val * sizeof(*s->map_cmdline_to_pid));
2090 static int trace_create_savedcmd(void)
2094 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
2098 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
2108 int is_tracing_stopped(void)
2110 return global_trace.stop_count;
2114 * tracing_start - quick start of the tracer
2116 * If tracing is enabled but was stopped by tracing_stop,
2117 * this will start the tracer back up.
2119 void tracing_start(void)
2121 struct trace_buffer *buffer;
2122 unsigned long flags;
2124 if (tracing_disabled)
2127 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2128 if (--global_trace.stop_count) {
2129 if (global_trace.stop_count < 0) {
2130 /* Someone screwed up their debugging */
2132 global_trace.stop_count = 0;
2137 /* Prevent the buffers from switching */
2138 arch_spin_lock(&global_trace.max_lock);
2140 buffer = global_trace.array_buffer.buffer;
2142 ring_buffer_record_enable(buffer);
2144 #ifdef CONFIG_TRACER_MAX_TRACE
2145 buffer = global_trace.max_buffer.buffer;
2147 ring_buffer_record_enable(buffer);
2150 arch_spin_unlock(&global_trace.max_lock);
2153 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2156 static void tracing_start_tr(struct trace_array *tr)
2158 struct trace_buffer *buffer;
2159 unsigned long flags;
2161 if (tracing_disabled)
2164 /* If global, we need to also start the max tracer */
2165 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2166 return tracing_start();
2168 raw_spin_lock_irqsave(&tr->start_lock, flags);
2170 if (--tr->stop_count) {
2171 if (tr->stop_count < 0) {
2172 /* Someone screwed up their debugging */
2179 buffer = tr->array_buffer.buffer;
2181 ring_buffer_record_enable(buffer);
2184 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2188 * tracing_stop - quick stop of the tracer
2190 * Light weight way to stop tracing. Use in conjunction with
2193 void tracing_stop(void)
2195 struct trace_buffer *buffer;
2196 unsigned long flags;
2198 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2199 if (global_trace.stop_count++)
2202 /* Prevent the buffers from switching */
2203 arch_spin_lock(&global_trace.max_lock);
2205 buffer = global_trace.array_buffer.buffer;
2207 ring_buffer_record_disable(buffer);
2209 #ifdef CONFIG_TRACER_MAX_TRACE
2210 buffer = global_trace.max_buffer.buffer;
2212 ring_buffer_record_disable(buffer);
2215 arch_spin_unlock(&global_trace.max_lock);
2218 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2221 static void tracing_stop_tr(struct trace_array *tr)
2223 struct trace_buffer *buffer;
2224 unsigned long flags;
2226 /* If global, we need to also stop the max tracer */
2227 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2228 return tracing_stop();
2230 raw_spin_lock_irqsave(&tr->start_lock, flags);
2231 if (tr->stop_count++)
2234 buffer = tr->array_buffer.buffer;
2236 ring_buffer_record_disable(buffer);
2239 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2242 static int trace_save_cmdline(struct task_struct *tsk)
2246 /* treat recording of idle task as a success */
2250 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
2254 * It's not the end of the world if we don't get
2255 * the lock, but we also don't want to spin
2256 * nor do we want to disable interrupts,
2257 * so if we miss here, then better luck next time.
2259 if (!arch_spin_trylock(&trace_cmdline_lock))
2262 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2263 if (idx == NO_CMDLINE_MAP) {
2264 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2267 * Check whether the cmdline buffer at idx has a pid
2268 * mapped. We are going to overwrite that entry so we
2269 * need to clear the map_pid_to_cmdline. Otherwise we
2270 * would read the new comm for the old pid.
2272 pid = savedcmd->map_cmdline_to_pid[idx];
2273 if (pid != NO_CMDLINE_MAP)
2274 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
2276 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2277 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
2279 savedcmd->cmdline_idx = idx;
2282 set_cmdline(idx, tsk->comm);
2284 arch_spin_unlock(&trace_cmdline_lock);
2289 static void __trace_find_cmdline(int pid, char comm[])
2294 strcpy(comm, "<idle>");
2298 if (WARN_ON_ONCE(pid < 0)) {
2299 strcpy(comm, "<XXX>");
2303 if (pid > PID_MAX_DEFAULT) {
2304 strcpy(comm, "<...>");
2308 map = savedcmd->map_pid_to_cmdline[pid];
2309 if (map != NO_CMDLINE_MAP)
2310 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2312 strcpy(comm, "<...>");
2315 void trace_find_cmdline(int pid, char comm[])
2318 arch_spin_lock(&trace_cmdline_lock);
2320 __trace_find_cmdline(pid, comm);
2322 arch_spin_unlock(&trace_cmdline_lock);
2326 int trace_find_tgid(int pid)
2328 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2331 return tgid_map[pid];
2334 static int trace_save_tgid(struct task_struct *tsk)
2336 /* treat recording of idle task as a success */
2340 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2343 tgid_map[tsk->pid] = tsk->tgid;
2347 static bool tracing_record_taskinfo_skip(int flags)
2349 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2351 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2353 if (!__this_cpu_read(trace_taskinfo_save))
2359 * tracing_record_taskinfo - record the task info of a task
2361 * @task: task to record
2362 * @flags: TRACE_RECORD_CMDLINE for recording comm
2363 * TRACE_RECORD_TGID for recording tgid
2365 void tracing_record_taskinfo(struct task_struct *task, int flags)
2369 if (tracing_record_taskinfo_skip(flags))
2373 * Record as much task information as possible. If some fail, continue
2374 * to try to record the others.
2376 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2377 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2379 /* If recording any information failed, retry again soon. */
2383 __this_cpu_write(trace_taskinfo_save, false);
2387 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2389 * @prev: previous task during sched_switch
2390 * @next: next task during sched_switch
2391 * @flags: TRACE_RECORD_CMDLINE for recording comm
2392 * TRACE_RECORD_TGID for recording tgid
2394 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2395 struct task_struct *next, int flags)
2399 if (tracing_record_taskinfo_skip(flags))
2403 * Record as much task information as possible. If some fail, continue
2404 * to try to record the others.
2406 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2407 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2408 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2409 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2411 /* If recording any information failed, retry again soon. */
2415 __this_cpu_write(trace_taskinfo_save, false);
2418 /* Helpers to record a specific task information */
2419 void tracing_record_cmdline(struct task_struct *task)
2421 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2424 void tracing_record_tgid(struct task_struct *task)
2426 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2430 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2431 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2432 * simplifies those functions and keeps them in sync.
2434 enum print_line_t trace_handle_return(struct trace_seq *s)
2436 return trace_seq_has_overflowed(s) ?
2437 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2439 EXPORT_SYMBOL_GPL(trace_handle_return);
2442 tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
2443 unsigned long flags, int pc)
2445 struct task_struct *tsk = current;
2447 entry->preempt_count = pc & 0xff;
2448 entry->pid = (tsk) ? tsk->pid : 0;
2451 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2452 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2454 TRACE_FLAG_IRQS_NOSUPPORT |
2456 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
2457 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2458 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2459 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2460 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2462 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2464 struct ring_buffer_event *
2465 trace_buffer_lock_reserve(struct trace_buffer *buffer,
2468 unsigned long flags, int pc)
2470 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2473 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2474 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2475 static int trace_buffered_event_ref;
2478 * trace_buffered_event_enable - enable buffering events
2480 * When events are being filtered, it is quicker to use a temporary
2481 * buffer to write the event data into if there's a likely chance
2482 * that it will not be committed. The discard of the ring buffer
2483 * is not as fast as committing, and is much slower than copying
2486 * When an event is to be filtered, allocate per cpu buffers to
2487 * write the event data into, and if the event is filtered and discarded
2488 * it is simply dropped, otherwise, the entire data is to be committed
2491 void trace_buffered_event_enable(void)
2493 struct ring_buffer_event *event;
2497 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2499 if (trace_buffered_event_ref++)
2502 for_each_tracing_cpu(cpu) {
2503 page = alloc_pages_node(cpu_to_node(cpu),
2504 GFP_KERNEL | __GFP_NORETRY, 0);
2508 event = page_address(page);
2509 memset(event, 0, sizeof(*event));
2511 per_cpu(trace_buffered_event, cpu) = event;
2514 if (cpu == smp_processor_id() &&
2515 this_cpu_read(trace_buffered_event) !=
2516 per_cpu(trace_buffered_event, cpu))
2523 trace_buffered_event_disable();
2526 static void enable_trace_buffered_event(void *data)
2528 /* Probably not needed, but do it anyway */
2530 this_cpu_dec(trace_buffered_event_cnt);
2533 static void disable_trace_buffered_event(void *data)
2535 this_cpu_inc(trace_buffered_event_cnt);
2539 * trace_buffered_event_disable - disable buffering events
2541 * When a filter is removed, it is faster to not use the buffered
2542 * events, and to commit directly into the ring buffer. Free up
2543 * the temp buffers when there are no more users. This requires
2544 * special synchronization with current events.
2546 void trace_buffered_event_disable(void)
2550 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2552 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2555 if (--trace_buffered_event_ref)
2559 /* For each CPU, set the buffer as used. */
2560 smp_call_function_many(tracing_buffer_mask,
2561 disable_trace_buffered_event, NULL, 1);
2564 /* Wait for all current users to finish */
2567 for_each_tracing_cpu(cpu) {
2568 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2569 per_cpu(trace_buffered_event, cpu) = NULL;
2572 * Make sure trace_buffered_event is NULL before clearing
2573 * trace_buffered_event_cnt.
2578 /* Do the work on each cpu */
2579 smp_call_function_many(tracing_buffer_mask,
2580 enable_trace_buffered_event, NULL, 1);
2584 static struct trace_buffer *temp_buffer;
2586 struct ring_buffer_event *
2587 trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2588 struct trace_event_file *trace_file,
2589 int type, unsigned long len,
2590 unsigned long flags, int pc)
2592 struct ring_buffer_event *entry;
2595 *current_rb = trace_file->tr->array_buffer.buffer;
2597 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2598 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2599 (entry = this_cpu_read(trace_buffered_event))) {
2600 /* Try to use the per cpu buffer first */
2601 val = this_cpu_inc_return(trace_buffered_event_cnt);
2603 trace_event_setup(entry, type, flags, pc);
2604 entry->array[0] = len;
2607 this_cpu_dec(trace_buffered_event_cnt);
2610 entry = __trace_buffer_lock_reserve(*current_rb,
2611 type, len, flags, pc);
2613 * If tracing is off, but we have triggers enabled
2614 * we still need to look at the event data. Use the temp_buffer
2615 * to store the trace event for the tigger to use. It's recusive
2616 * safe and will not be recorded anywhere.
2618 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2619 *current_rb = temp_buffer;
2620 entry = __trace_buffer_lock_reserve(*current_rb,
2621 type, len, flags, pc);
2625 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2627 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2628 static DEFINE_MUTEX(tracepoint_printk_mutex);
2630 static void output_printk(struct trace_event_buffer *fbuffer)
2632 struct trace_event_call *event_call;
2633 struct trace_event_file *file;
2634 struct trace_event *event;
2635 unsigned long flags;
2636 struct trace_iterator *iter = tracepoint_print_iter;
2638 /* We should never get here if iter is NULL */
2639 if (WARN_ON_ONCE(!iter))
2642 event_call = fbuffer->trace_file->event_call;
2643 if (!event_call || !event_call->event.funcs ||
2644 !event_call->event.funcs->trace)
2647 file = fbuffer->trace_file;
2648 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2649 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2650 !filter_match_preds(file->filter, fbuffer->entry)))
2653 event = &fbuffer->trace_file->event_call->event;
2655 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2656 trace_seq_init(&iter->seq);
2657 iter->ent = fbuffer->entry;
2658 event_call->event.funcs->trace(iter, 0, event);
2659 trace_seq_putc(&iter->seq, 0);
2660 printk("%s", iter->seq.buffer);
2662 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2665 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2666 void *buffer, size_t *lenp,
2669 int save_tracepoint_printk;
2672 mutex_lock(&tracepoint_printk_mutex);
2673 save_tracepoint_printk = tracepoint_printk;
2675 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2678 * This will force exiting early, as tracepoint_printk
2679 * is always zero when tracepoint_printk_iter is not allocated
2681 if (!tracepoint_print_iter)
2682 tracepoint_printk = 0;
2684 if (save_tracepoint_printk == tracepoint_printk)
2687 if (tracepoint_printk)
2688 static_key_enable(&tracepoint_printk_key.key);
2690 static_key_disable(&tracepoint_printk_key.key);
2693 mutex_unlock(&tracepoint_printk_mutex);
2698 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2700 if (static_key_false(&tracepoint_printk_key.key))
2701 output_printk(fbuffer);
2703 event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer,
2704 fbuffer->event, fbuffer->entry,
2705 fbuffer->flags, fbuffer->pc, fbuffer->regs);
2707 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2712 * trace_buffer_unlock_commit_regs()
2713 * trace_event_buffer_commit()
2714 * trace_event_raw_event_xxx()
2716 # define STACK_SKIP 3
2718 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2719 struct trace_buffer *buffer,
2720 struct ring_buffer_event *event,
2721 unsigned long flags, int pc,
2722 struct pt_regs *regs)
2724 __buffer_unlock_commit(buffer, event);
2727 * If regs is not set, then skip the necessary functions.
2728 * Note, we can still get here via blktrace, wakeup tracer
2729 * and mmiotrace, but that's ok if they lose a function or
2730 * two. They are not that meaningful.
2732 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2733 ftrace_trace_userstack(buffer, flags, pc);
2737 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2740 trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2741 struct ring_buffer_event *event)
2743 __buffer_unlock_commit(buffer, event);
2747 trace_process_export(struct trace_export *export,
2748 struct ring_buffer_event *event)
2750 struct trace_entry *entry;
2751 unsigned int size = 0;
2753 entry = ring_buffer_event_data(event);
2754 size = ring_buffer_event_length(event);
2755 export->write(export, entry, size);
2758 static DEFINE_MUTEX(ftrace_export_lock);
2760 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2762 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2764 static inline void ftrace_exports_enable(void)
2766 static_branch_enable(&ftrace_exports_enabled);
2769 static inline void ftrace_exports_disable(void)
2771 static_branch_disable(&ftrace_exports_enabled);
2774 static void ftrace_exports(struct ring_buffer_event *event)
2776 struct trace_export *export;
2778 preempt_disable_notrace();
2780 export = rcu_dereference_raw_check(ftrace_exports_list);
2782 trace_process_export(export, event);
2783 export = rcu_dereference_raw_check(export->next);
2786 preempt_enable_notrace();
2790 add_trace_export(struct trace_export **list, struct trace_export *export)
2792 rcu_assign_pointer(export->next, *list);
2794 * We are entering export into the list but another
2795 * CPU might be walking that list. We need to make sure
2796 * the export->next pointer is valid before another CPU sees
2797 * the export pointer included into the list.
2799 rcu_assign_pointer(*list, export);
2803 rm_trace_export(struct trace_export **list, struct trace_export *export)
2805 struct trace_export **p;
2807 for (p = list; *p != NULL; p = &(*p)->next)
2814 rcu_assign_pointer(*p, (*p)->next);
2820 add_ftrace_export(struct trace_export **list, struct trace_export *export)
2823 ftrace_exports_enable();
2825 add_trace_export(list, export);
2829 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2833 ret = rm_trace_export(list, export);
2835 ftrace_exports_disable();
2840 int register_ftrace_export(struct trace_export *export)
2842 if (WARN_ON_ONCE(!export->write))
2845 mutex_lock(&ftrace_export_lock);
2847 add_ftrace_export(&ftrace_exports_list, export);
2849 mutex_unlock(&ftrace_export_lock);
2853 EXPORT_SYMBOL_GPL(register_ftrace_export);
2855 int unregister_ftrace_export(struct trace_export *export)
2859 mutex_lock(&ftrace_export_lock);
2861 ret = rm_ftrace_export(&ftrace_exports_list, export);
2863 mutex_unlock(&ftrace_export_lock);
2867 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2870 trace_function(struct trace_array *tr,
2871 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2874 struct trace_event_call *call = &event_function;
2875 struct trace_buffer *buffer = tr->array_buffer.buffer;
2876 struct ring_buffer_event *event;
2877 struct ftrace_entry *entry;
2879 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2883 entry = ring_buffer_event_data(event);
2885 entry->parent_ip = parent_ip;
2887 if (!call_filter_check_discard(call, entry, buffer, event)) {
2888 if (static_branch_unlikely(&ftrace_exports_enabled))
2889 ftrace_exports(event);
2890 __buffer_unlock_commit(buffer, event);
2894 #ifdef CONFIG_STACKTRACE
2896 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2897 #define FTRACE_KSTACK_NESTING 4
2899 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2901 struct ftrace_stack {
2902 unsigned long calls[FTRACE_KSTACK_ENTRIES];
2906 struct ftrace_stacks {
2907 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2910 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2911 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2913 static void __ftrace_trace_stack(struct trace_buffer *buffer,
2914 unsigned long flags,
2915 int skip, int pc, struct pt_regs *regs)
2917 struct trace_event_call *call = &event_kernel_stack;
2918 struct ring_buffer_event *event;
2919 unsigned int size, nr_entries;
2920 struct ftrace_stack *fstack;
2921 struct stack_entry *entry;
2925 * Add one, for this function and the call to save_stack_trace()
2926 * If regs is set, then these functions will not be in the way.
2928 #ifndef CONFIG_UNWINDER_ORC
2934 * Since events can happen in NMIs there's no safe way to
2935 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2936 * or NMI comes in, it will just have to use the default
2937 * FTRACE_STACK_SIZE.
2939 preempt_disable_notrace();
2941 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2943 /* This should never happen. If it does, yell once and skip */
2944 if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING))
2948 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2949 * interrupt will either see the value pre increment or post
2950 * increment. If the interrupt happens pre increment it will have
2951 * restored the counter when it returns. We just need a barrier to
2952 * keep gcc from moving things around.
2956 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2957 size = ARRAY_SIZE(fstack->calls);
2960 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2963 nr_entries = stack_trace_save(fstack->calls, size, skip);
2966 size = nr_entries * sizeof(unsigned long);
2967 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2968 sizeof(*entry) + size, flags, pc);
2971 entry = ring_buffer_event_data(event);
2973 memcpy(&entry->caller, fstack->calls, size);
2974 entry->size = nr_entries;
2976 if (!call_filter_check_discard(call, entry, buffer, event))
2977 __buffer_unlock_commit(buffer, event);
2980 /* Again, don't let gcc optimize things here */
2982 __this_cpu_dec(ftrace_stack_reserve);
2983 preempt_enable_notrace();
2987 static inline void ftrace_trace_stack(struct trace_array *tr,
2988 struct trace_buffer *buffer,
2989 unsigned long flags,
2990 int skip, int pc, struct pt_regs *regs)
2992 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2995 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2998 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
3001 struct trace_buffer *buffer = tr->array_buffer.buffer;
3003 if (rcu_is_watching()) {
3004 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
3009 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
3010 * but if the above rcu_is_watching() failed, then the NMI
3011 * triggered someplace critical, and rcu_irq_enter() should
3012 * not be called from NMI.
3014 if (unlikely(in_nmi()))
3017 rcu_irq_enter_irqson();
3018 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
3019 rcu_irq_exit_irqson();
3023 * trace_dump_stack - record a stack back trace in the trace buffer
3024 * @skip: Number of functions to skip (helper handlers)
3026 void trace_dump_stack(int skip)
3028 unsigned long flags;
3030 if (tracing_disabled || tracing_selftest_running)
3033 local_save_flags(flags);
3035 #ifndef CONFIG_UNWINDER_ORC
3036 /* Skip 1 to skip this function. */
3039 __ftrace_trace_stack(global_trace.array_buffer.buffer,
3040 flags, skip, preempt_count(), NULL);
3042 EXPORT_SYMBOL_GPL(trace_dump_stack);
3044 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
3045 static DEFINE_PER_CPU(int, user_stack_count);
3048 ftrace_trace_userstack(struct trace_buffer *buffer, unsigned long flags, int pc)
3050 struct trace_event_call *call = &event_user_stack;
3051 struct ring_buffer_event *event;
3052 struct userstack_entry *entry;
3054 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
3058 * NMIs can not handle page faults, even with fix ups.
3059 * The save user stack can (and often does) fault.
3061 if (unlikely(in_nmi()))
3065 * prevent recursion, since the user stack tracing may
3066 * trigger other kernel events.
3069 if (__this_cpu_read(user_stack_count))
3072 __this_cpu_inc(user_stack_count);
3074 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
3075 sizeof(*entry), flags, pc);
3077 goto out_drop_count;
3078 entry = ring_buffer_event_data(event);
3080 entry->tgid = current->tgid;
3081 memset(&entry->caller, 0, sizeof(entry->caller));
3083 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
3084 if (!call_filter_check_discard(call, entry, buffer, event))
3085 __buffer_unlock_commit(buffer, event);
3088 __this_cpu_dec(user_stack_count);
3092 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
3093 static void ftrace_trace_userstack(struct trace_buffer *buffer,
3094 unsigned long flags, int pc)
3097 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3099 #endif /* CONFIG_STACKTRACE */
3101 /* created for use with alloc_percpu */
3102 struct trace_buffer_struct {
3104 char buffer[4][TRACE_BUF_SIZE];
3107 static struct trace_buffer_struct *trace_percpu_buffer;
3110 * Thise allows for lockless recording. If we're nested too deeply, then
3111 * this returns NULL.
3113 static char *get_trace_buf(void)
3115 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3117 if (!buffer || buffer->nesting >= 4)
3122 /* Interrupts must see nesting incremented before we use the buffer */
3124 return &buffer->buffer[buffer->nesting][0];
3127 static void put_trace_buf(void)
3129 /* Don't let the decrement of nesting leak before this */
3131 this_cpu_dec(trace_percpu_buffer->nesting);
3134 static int alloc_percpu_trace_buffer(void)
3136 struct trace_buffer_struct *buffers;
3138 buffers = alloc_percpu(struct trace_buffer_struct);
3139 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3142 trace_percpu_buffer = buffers;
3146 static int buffers_allocated;
3148 void trace_printk_init_buffers(void)
3150 if (buffers_allocated)
3153 if (alloc_percpu_trace_buffer())
3156 /* trace_printk() is for debug use only. Don't use it in production. */
3159 pr_warn("**********************************************************\n");
3160 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3162 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3164 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3165 pr_warn("** unsafe for production use. **\n");
3167 pr_warn("** If you see this message and you are not debugging **\n");
3168 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3170 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3171 pr_warn("**********************************************************\n");
3173 /* Expand the buffers to set size */
3174 tracing_update_buffers();
3176 buffers_allocated = 1;
3179 * trace_printk_init_buffers() can be called by modules.
3180 * If that happens, then we need to start cmdline recording
3181 * directly here. If the global_trace.buffer is already
3182 * allocated here, then this was called by module code.
3184 if (global_trace.array_buffer.buffer)
3185 tracing_start_cmdline_record();
3187 EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3189 void trace_printk_start_comm(void)
3191 /* Start tracing comms if trace printk is set */
3192 if (!buffers_allocated)
3194 tracing_start_cmdline_record();
3197 static void trace_printk_start_stop_comm(int enabled)
3199 if (!buffers_allocated)
3203 tracing_start_cmdline_record();
3205 tracing_stop_cmdline_record();
3209 * trace_vbprintk - write binary msg to tracing buffer
3210 * @ip: The address of the caller
3211 * @fmt: The string format to write to the buffer
3212 * @args: Arguments for @fmt
3214 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3216 struct trace_event_call *call = &event_bprint;
3217 struct ring_buffer_event *event;
3218 struct trace_buffer *buffer;
3219 struct trace_array *tr = &global_trace;
3220 struct bprint_entry *entry;
3221 unsigned long flags;
3223 int len = 0, size, pc;
3225 if (unlikely(tracing_selftest_running || tracing_disabled))
3228 /* Don't pollute graph traces with trace_vprintk internals */
3229 pause_graph_tracing();
3231 pc = preempt_count();
3232 preempt_disable_notrace();
3234 tbuffer = get_trace_buf();
3240 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3242 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3245 local_save_flags(flags);
3246 size = sizeof(*entry) + sizeof(u32) * len;
3247 buffer = tr->array_buffer.buffer;
3248 ring_buffer_nest_start(buffer);
3249 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3253 entry = ring_buffer_event_data(event);
3257 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3258 if (!call_filter_check_discard(call, entry, buffer, event)) {
3259 __buffer_unlock_commit(buffer, event);
3260 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
3264 ring_buffer_nest_end(buffer);
3269 preempt_enable_notrace();
3270 unpause_graph_tracing();
3274 EXPORT_SYMBOL_GPL(trace_vbprintk);
3278 __trace_array_vprintk(struct trace_buffer *buffer,
3279 unsigned long ip, const char *fmt, va_list args)
3281 struct trace_event_call *call = &event_print;
3282 struct ring_buffer_event *event;
3283 int len = 0, size, pc;
3284 struct print_entry *entry;
3285 unsigned long flags;
3288 if (tracing_disabled || tracing_selftest_running)
3291 /* Don't pollute graph traces with trace_vprintk internals */
3292 pause_graph_tracing();
3294 pc = preempt_count();
3295 preempt_disable_notrace();
3298 tbuffer = get_trace_buf();
3304 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3306 local_save_flags(flags);
3307 size = sizeof(*entry) + len + 1;
3308 ring_buffer_nest_start(buffer);
3309 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3313 entry = ring_buffer_event_data(event);
3316 memcpy(&entry->buf, tbuffer, len + 1);
3317 if (!call_filter_check_discard(call, entry, buffer, event)) {
3318 __buffer_unlock_commit(buffer, event);
3319 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3323 ring_buffer_nest_end(buffer);
3327 preempt_enable_notrace();
3328 unpause_graph_tracing();
3334 int trace_array_vprintk(struct trace_array *tr,
3335 unsigned long ip, const char *fmt, va_list args)
3337 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3341 int trace_array_printk(struct trace_array *tr,
3342 unsigned long ip, const char *fmt, ...)
3350 /* This is only allowed for created instances */
3351 if (tr == &global_trace)
3354 if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3358 ret = trace_array_vprintk(tr, ip, fmt, ap);
3362 EXPORT_SYMBOL_GPL(trace_array_printk);
3365 int trace_array_printk_buf(struct trace_buffer *buffer,
3366 unsigned long ip, const char *fmt, ...)
3371 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3375 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3381 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3383 return trace_array_vprintk(&global_trace, ip, fmt, args);
3385 EXPORT_SYMBOL_GPL(trace_vprintk);
3387 static void trace_iterator_increment(struct trace_iterator *iter)
3389 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3393 ring_buffer_iter_advance(buf_iter);
3396 static struct trace_entry *
3397 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3398 unsigned long *lost_events)
3400 struct ring_buffer_event *event;
3401 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3404 event = ring_buffer_iter_peek(buf_iter, ts);
3406 *lost_events = ring_buffer_iter_dropped(buf_iter) ?
3407 (unsigned long)-1 : 0;
3409 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3414 iter->ent_size = ring_buffer_event_length(event);
3415 return ring_buffer_event_data(event);
3421 static struct trace_entry *
3422 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3423 unsigned long *missing_events, u64 *ent_ts)
3425 struct trace_buffer *buffer = iter->array_buffer->buffer;
3426 struct trace_entry *ent, *next = NULL;
3427 unsigned long lost_events = 0, next_lost = 0;
3428 int cpu_file = iter->cpu_file;
3429 u64 next_ts = 0, ts;
3435 * If we are in a per_cpu trace file, don't bother by iterating over
3436 * all cpu and peek directly.
3438 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3439 if (ring_buffer_empty_cpu(buffer, cpu_file))
3441 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3443 *ent_cpu = cpu_file;
3448 for_each_tracing_cpu(cpu) {
3450 if (ring_buffer_empty_cpu(buffer, cpu))
3453 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3456 * Pick the entry with the smallest timestamp:
3458 if (ent && (!next || ts < next_ts)) {
3462 next_lost = lost_events;
3463 next_size = iter->ent_size;
3467 iter->ent_size = next_size;
3470 *ent_cpu = next_cpu;
3476 *missing_events = next_lost;
3481 #define STATIC_TEMP_BUF_SIZE 128
3482 static char static_temp_buf[STATIC_TEMP_BUF_SIZE];
3484 /* Find the next real entry, without updating the iterator itself */
3485 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3486 int *ent_cpu, u64 *ent_ts)
3488 /* __find_next_entry will reset ent_size */
3489 int ent_size = iter->ent_size;
3490 struct trace_entry *entry;
3493 * If called from ftrace_dump(), then the iter->temp buffer
3494 * will be the static_temp_buf and not created from kmalloc.
3495 * If the entry size is greater than the buffer, we can
3496 * not save it. Just return NULL in that case. This is only
3497 * used to add markers when two consecutive events' time
3498 * stamps have a large delta. See trace_print_lat_context()
3500 if (iter->temp == static_temp_buf &&
3501 STATIC_TEMP_BUF_SIZE < ent_size)
3505 * The __find_next_entry() may call peek_next_entry(), which may
3506 * call ring_buffer_peek() that may make the contents of iter->ent
3507 * undefined. Need to copy iter->ent now.
3509 if (iter->ent && iter->ent != iter->temp) {
3510 if ((!iter->temp || iter->temp_size < iter->ent_size) &&
3511 !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3513 iter->temp = kmalloc(iter->ent_size, GFP_KERNEL);
3517 memcpy(iter->temp, iter->ent, iter->ent_size);
3518 iter->temp_size = iter->ent_size;
3519 iter->ent = iter->temp;
3521 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3522 /* Put back the original ent_size */
3523 iter->ent_size = ent_size;
3528 /* Find the next real entry, and increment the iterator to the next entry */
3529 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3531 iter->ent = __find_next_entry(iter, &iter->cpu,
3532 &iter->lost_events, &iter->ts);
3535 trace_iterator_increment(iter);
3537 return iter->ent ? iter : NULL;
3540 static void trace_consume(struct trace_iterator *iter)
3542 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
3543 &iter->lost_events);
3546 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3548 struct trace_iterator *iter = m->private;
3552 WARN_ON_ONCE(iter->leftover);
3556 /* can't go backwards */
3561 ent = trace_find_next_entry_inc(iter);
3565 while (ent && iter->idx < i)
3566 ent = trace_find_next_entry_inc(iter);
3573 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3575 struct ring_buffer_iter *buf_iter;
3576 unsigned long entries = 0;
3579 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
3581 buf_iter = trace_buffer_iter(iter, cpu);
3585 ring_buffer_iter_reset(buf_iter);
3588 * We could have the case with the max latency tracers
3589 * that a reset never took place on a cpu. This is evident
3590 * by the timestamp being before the start of the buffer.
3592 while (ring_buffer_iter_peek(buf_iter, &ts)) {
3593 if (ts >= iter->array_buffer->time_start)
3596 ring_buffer_iter_advance(buf_iter);
3599 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
3603 * The current tracer is copied to avoid a global locking
3606 static void *s_start(struct seq_file *m, loff_t *pos)
3608 struct trace_iterator *iter = m->private;
3609 struct trace_array *tr = iter->tr;
3610 int cpu_file = iter->cpu_file;
3616 * copy the tracer to avoid using a global lock all around.
3617 * iter->trace is a copy of current_trace, the pointer to the
3618 * name may be used instead of a strcmp(), as iter->trace->name
3619 * will point to the same string as current_trace->name.
3621 mutex_lock(&trace_types_lock);
3622 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3623 *iter->trace = *tr->current_trace;
3624 mutex_unlock(&trace_types_lock);
3626 #ifdef CONFIG_TRACER_MAX_TRACE
3627 if (iter->snapshot && iter->trace->use_max_tr)
3628 return ERR_PTR(-EBUSY);
3631 if (!iter->snapshot)
3632 atomic_inc(&trace_record_taskinfo_disabled);
3634 if (*pos != iter->pos) {
3639 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3640 for_each_tracing_cpu(cpu)
3641 tracing_iter_reset(iter, cpu);
3643 tracing_iter_reset(iter, cpu_file);
3646 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3651 * If we overflowed the seq_file before, then we want
3652 * to just reuse the trace_seq buffer again.
3658 p = s_next(m, p, &l);
3662 trace_event_read_lock();
3663 trace_access_lock(cpu_file);
3667 static void s_stop(struct seq_file *m, void *p)
3669 struct trace_iterator *iter = m->private;
3671 #ifdef CONFIG_TRACER_MAX_TRACE
3672 if (iter->snapshot && iter->trace->use_max_tr)
3676 if (!iter->snapshot)
3677 atomic_dec(&trace_record_taskinfo_disabled);
3679 trace_access_unlock(iter->cpu_file);
3680 trace_event_read_unlock();
3684 get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
3685 unsigned long *entries, int cpu)
3687 unsigned long count;
3689 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3691 * If this buffer has skipped entries, then we hold all
3692 * entries for the trace and we need to ignore the
3693 * ones before the time stamp.
3695 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3696 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3697 /* total is the same as the entries */
3701 ring_buffer_overrun_cpu(buf->buffer, cpu);
3706 get_total_entries(struct array_buffer *buf,
3707 unsigned long *total, unsigned long *entries)
3715 for_each_tracing_cpu(cpu) {
3716 get_total_entries_cpu(buf, &t, &e, cpu);
3722 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
3724 unsigned long total, entries;
3729 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
3734 unsigned long trace_total_entries(struct trace_array *tr)
3736 unsigned long total, entries;
3741 get_total_entries(&tr->array_buffer, &total, &entries);
3746 static void print_lat_help_header(struct seq_file *m)
3748 seq_puts(m, "# _------=> CPU# \n"
3749 "# / _-----=> irqs-off \n"
3750 "# | / _----=> need-resched \n"
3751 "# || / _---=> hardirq/softirq \n"
3752 "# ||| / _--=> preempt-depth \n"
3754 "# cmd pid ||||| time | caller \n"
3755 "# \\ / ||||| \\ | / \n");
3758 static void print_event_info(struct array_buffer *buf, struct seq_file *m)
3760 unsigned long total;
3761 unsigned long entries;
3763 get_total_entries(buf, &total, &entries);
3764 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3765 entries, total, num_online_cpus());
3769 static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
3772 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3774 print_event_info(buf, m);
3776 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3777 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
3780 static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
3783 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3784 const char *space = " ";
3785 int prec = tgid ? 10 : 2;
3787 print_event_info(buf, m);
3789 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space);
3790 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
3791 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
3792 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
3793 seq_printf(m, "# %.*s||| / delay\n", prec, space);
3794 seq_printf(m, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID ");
3795 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | ");
3799 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3801 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3802 struct array_buffer *buf = iter->array_buffer;
3803 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3804 struct tracer *type = iter->trace;
3805 unsigned long entries;
3806 unsigned long total;
3807 const char *name = "preemption";
3811 get_total_entries(buf, &total, &entries);
3813 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3815 seq_puts(m, "# -----------------------------------"
3816 "---------------------------------\n");
3817 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3818 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3819 nsecs_to_usecs(data->saved_latency),
3823 #if defined(CONFIG_PREEMPT_NONE)
3825 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3827 #elif defined(CONFIG_PREEMPT)
3829 #elif defined(CONFIG_PREEMPT_RT)
3834 /* These are reserved for later use */
3837 seq_printf(m, " #P:%d)\n", num_online_cpus());
3841 seq_puts(m, "# -----------------\n");
3842 seq_printf(m, "# | task: %.16s-%d "
3843 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3844 data->comm, data->pid,
3845 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3846 data->policy, data->rt_priority);
3847 seq_puts(m, "# -----------------\n");
3849 if (data->critical_start) {
3850 seq_puts(m, "# => started at: ");
3851 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3852 trace_print_seq(m, &iter->seq);
3853 seq_puts(m, "\n# => ended at: ");
3854 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3855 trace_print_seq(m, &iter->seq);
3856 seq_puts(m, "\n#\n");
3862 static void test_cpu_buff_start(struct trace_iterator *iter)
3864 struct trace_seq *s = &iter->seq;
3865 struct trace_array *tr = iter->tr;
3867 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3870 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3873 if (cpumask_available(iter->started) &&
3874 cpumask_test_cpu(iter->cpu, iter->started))
3877 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
3880 if (cpumask_available(iter->started))
3881 cpumask_set_cpu(iter->cpu, iter->started);
3883 /* Don't print started cpu buffer for the first entry of the trace */
3885 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3889 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3891 struct trace_array *tr = iter->tr;
3892 struct trace_seq *s = &iter->seq;
3893 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3894 struct trace_entry *entry;
3895 struct trace_event *event;
3899 test_cpu_buff_start(iter);
3901 event = ftrace_find_event(entry->type);
3903 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3904 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3905 trace_print_lat_context(iter);
3907 trace_print_context(iter);
3910 if (trace_seq_has_overflowed(s))
3911 return TRACE_TYPE_PARTIAL_LINE;
3914 return event->funcs->trace(iter, sym_flags, event);
3916 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3918 return trace_handle_return(s);
3921 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3923 struct trace_array *tr = iter->tr;
3924 struct trace_seq *s = &iter->seq;
3925 struct trace_entry *entry;
3926 struct trace_event *event;
3930 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3931 trace_seq_printf(s, "%d %d %llu ",
3932 entry->pid, iter->cpu, iter->ts);
3934 if (trace_seq_has_overflowed(s))
3935 return TRACE_TYPE_PARTIAL_LINE;
3937 event = ftrace_find_event(entry->type);
3939 return event->funcs->raw(iter, 0, event);
3941 trace_seq_printf(s, "%d ?\n", entry->type);
3943 return trace_handle_return(s);
3946 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3948 struct trace_array *tr = iter->tr;
3949 struct trace_seq *s = &iter->seq;
3950 unsigned char newline = '\n';
3951 struct trace_entry *entry;
3952 struct trace_event *event;
3956 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3957 SEQ_PUT_HEX_FIELD(s, entry->pid);
3958 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3959 SEQ_PUT_HEX_FIELD(s, iter->ts);
3960 if (trace_seq_has_overflowed(s))
3961 return TRACE_TYPE_PARTIAL_LINE;
3964 event = ftrace_find_event(entry->type);
3966 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3967 if (ret != TRACE_TYPE_HANDLED)
3971 SEQ_PUT_FIELD(s, newline);
3973 return trace_handle_return(s);
3976 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3978 struct trace_array *tr = iter->tr;
3979 struct trace_seq *s = &iter->seq;
3980 struct trace_entry *entry;
3981 struct trace_event *event;
3985 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3986 SEQ_PUT_FIELD(s, entry->pid);
3987 SEQ_PUT_FIELD(s, iter->cpu);
3988 SEQ_PUT_FIELD(s, iter->ts);
3989 if (trace_seq_has_overflowed(s))
3990 return TRACE_TYPE_PARTIAL_LINE;
3993 event = ftrace_find_event(entry->type);
3994 return event ? event->funcs->binary(iter, 0, event) :
3998 int trace_empty(struct trace_iterator *iter)
4000 struct ring_buffer_iter *buf_iter;
4003 /* If we are looking at one CPU buffer, only check that one */
4004 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4005 cpu = iter->cpu_file;
4006 buf_iter = trace_buffer_iter(iter, cpu);
4008 if (!ring_buffer_iter_empty(buf_iter))
4011 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4017 for_each_tracing_cpu(cpu) {
4018 buf_iter = trace_buffer_iter(iter, cpu);
4020 if (!ring_buffer_iter_empty(buf_iter))
4023 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
4031 /* Called with trace_event_read_lock() held. */
4032 enum print_line_t print_trace_line(struct trace_iterator *iter)
4034 struct trace_array *tr = iter->tr;
4035 unsigned long trace_flags = tr->trace_flags;
4036 enum print_line_t ret;
4038 if (iter->lost_events) {
4039 if (iter->lost_events == (unsigned long)-1)
4040 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
4043 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
4044 iter->cpu, iter->lost_events);
4045 if (trace_seq_has_overflowed(&iter->seq))
4046 return TRACE_TYPE_PARTIAL_LINE;
4049 if (iter->trace && iter->trace->print_line) {
4050 ret = iter->trace->print_line(iter);
4051 if (ret != TRACE_TYPE_UNHANDLED)
4055 if (iter->ent->type == TRACE_BPUTS &&
4056 trace_flags & TRACE_ITER_PRINTK &&
4057 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4058 return trace_print_bputs_msg_only(iter);
4060 if (iter->ent->type == TRACE_BPRINT &&
4061 trace_flags & TRACE_ITER_PRINTK &&
4062 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4063 return trace_print_bprintk_msg_only(iter);
4065 if (iter->ent->type == TRACE_PRINT &&
4066 trace_flags & TRACE_ITER_PRINTK &&
4067 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4068 return trace_print_printk_msg_only(iter);
4070 if (trace_flags & TRACE_ITER_BIN)
4071 return print_bin_fmt(iter);
4073 if (trace_flags & TRACE_ITER_HEX)
4074 return print_hex_fmt(iter);
4076 if (trace_flags & TRACE_ITER_RAW)
4077 return print_raw_fmt(iter);
4079 return print_trace_fmt(iter);
4082 void trace_latency_header(struct seq_file *m)
4084 struct trace_iterator *iter = m->private;
4085 struct trace_array *tr = iter->tr;
4087 /* print nothing if the buffers are empty */
4088 if (trace_empty(iter))
4091 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4092 print_trace_header(m, iter);
4094 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4095 print_lat_help_header(m);
4098 void trace_default_header(struct seq_file *m)
4100 struct trace_iterator *iter = m->private;
4101 struct trace_array *tr = iter->tr;
4102 unsigned long trace_flags = tr->trace_flags;
4104 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4107 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4108 /* print nothing if the buffers are empty */
4109 if (trace_empty(iter))
4111 print_trace_header(m, iter);
4112 if (!(trace_flags & TRACE_ITER_VERBOSE))
4113 print_lat_help_header(m);
4115 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4116 if (trace_flags & TRACE_ITER_IRQ_INFO)
4117 print_func_help_header_irq(iter->array_buffer,
4120 print_func_help_header(iter->array_buffer, m,
4126 static void test_ftrace_alive(struct seq_file *m)
4128 if (!ftrace_is_dead())
4130 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4131 "# MAY BE MISSING FUNCTION EVENTS\n");
4134 #ifdef CONFIG_TRACER_MAX_TRACE
4135 static void show_snapshot_main_help(struct seq_file *m)
4137 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4138 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4139 "# Takes a snapshot of the main buffer.\n"
4140 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4141 "# (Doesn't have to be '2' works with any number that\n"
4142 "# is not a '0' or '1')\n");
4145 static void show_snapshot_percpu_help(struct seq_file *m)
4147 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4148 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4149 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4150 "# Takes a snapshot of the main buffer for this cpu.\n");
4152 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4153 "# Must use main snapshot file to allocate.\n");
4155 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4156 "# (Doesn't have to be '2' works with any number that\n"
4157 "# is not a '0' or '1')\n");
4160 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4162 if (iter->tr->allocated_snapshot)
4163 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
4165 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
4167 seq_puts(m, "# Snapshot commands:\n");
4168 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4169 show_snapshot_main_help(m);
4171 show_snapshot_percpu_help(m);
4174 /* Should never be called */
4175 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4178 static int s_show(struct seq_file *m, void *v)
4180 struct trace_iterator *iter = v;
4183 if (iter->ent == NULL) {
4185 seq_printf(m, "# tracer: %s\n", iter->trace->name);
4187 test_ftrace_alive(m);
4189 if (iter->snapshot && trace_empty(iter))
4190 print_snapshot_help(m, iter);
4191 else if (iter->trace && iter->trace->print_header)
4192 iter->trace->print_header(m);
4194 trace_default_header(m);
4196 } else if (iter->leftover) {
4198 * If we filled the seq_file buffer earlier, we
4199 * want to just show it now.
4201 ret = trace_print_seq(m, &iter->seq);
4203 /* ret should this time be zero, but you never know */
4204 iter->leftover = ret;
4207 print_trace_line(iter);
4208 ret = trace_print_seq(m, &iter->seq);
4210 * If we overflow the seq_file buffer, then it will
4211 * ask us for this data again at start up.
4213 * ret is 0 if seq_file write succeeded.
4216 iter->leftover = ret;
4223 * Should be used after trace_array_get(), trace_types_lock
4224 * ensures that i_cdev was already initialized.
4226 static inline int tracing_get_cpu(struct inode *inode)
4228 if (inode->i_cdev) /* See trace_create_cpu_file() */
4229 return (long)inode->i_cdev - 1;
4230 return RING_BUFFER_ALL_CPUS;
4233 static const struct seq_operations tracer_seq_ops = {
4240 static struct trace_iterator *
4241 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4243 struct trace_array *tr = inode->i_private;
4244 struct trace_iterator *iter;
4247 if (tracing_disabled)
4248 return ERR_PTR(-ENODEV);
4250 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4252 return ERR_PTR(-ENOMEM);
4254 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4256 if (!iter->buffer_iter)
4260 * trace_find_next_entry() may need to save off iter->ent.
4261 * It will place it into the iter->temp buffer. As most
4262 * events are less than 128, allocate a buffer of that size.
4263 * If one is greater, then trace_find_next_entry() will
4264 * allocate a new buffer to adjust for the bigger iter->ent.
4265 * It's not critical if it fails to get allocated here.
4267 iter->temp = kmalloc(128, GFP_KERNEL);
4269 iter->temp_size = 128;
4272 * We make a copy of the current tracer to avoid concurrent
4273 * changes on it while we are reading.
4275 mutex_lock(&trace_types_lock);
4276 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4280 *iter->trace = *tr->current_trace;
4282 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4287 #ifdef CONFIG_TRACER_MAX_TRACE
4288 /* Currently only the top directory has a snapshot */
4289 if (tr->current_trace->print_max || snapshot)
4290 iter->array_buffer = &tr->max_buffer;
4293 iter->array_buffer = &tr->array_buffer;
4294 iter->snapshot = snapshot;
4296 iter->cpu_file = tracing_get_cpu(inode);
4297 mutex_init(&iter->mutex);
4299 /* Notify the tracer early; before we stop tracing. */
4300 if (iter->trace->open)
4301 iter->trace->open(iter);
4303 /* Annotate start of buffers if we had overruns */
4304 if (ring_buffer_overruns(iter->array_buffer->buffer))
4305 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4307 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4308 if (trace_clocks[tr->clock_id].in_ns)
4309 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4312 * If pause-on-trace is enabled, then stop the trace while
4313 * dumping, unless this is the "snapshot" file
4315 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4316 tracing_stop_tr(tr);
4318 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4319 for_each_tracing_cpu(cpu) {
4320 iter->buffer_iter[cpu] =
4321 ring_buffer_read_prepare(iter->array_buffer->buffer,
4324 ring_buffer_read_prepare_sync();
4325 for_each_tracing_cpu(cpu) {
4326 ring_buffer_read_start(iter->buffer_iter[cpu]);
4327 tracing_iter_reset(iter, cpu);
4330 cpu = iter->cpu_file;
4331 iter->buffer_iter[cpu] =
4332 ring_buffer_read_prepare(iter->array_buffer->buffer,
4334 ring_buffer_read_prepare_sync();
4335 ring_buffer_read_start(iter->buffer_iter[cpu]);
4336 tracing_iter_reset(iter, cpu);
4339 mutex_unlock(&trace_types_lock);
4344 mutex_unlock(&trace_types_lock);
4347 kfree(iter->buffer_iter);
4349 seq_release_private(inode, file);
4350 return ERR_PTR(-ENOMEM);
4353 int tracing_open_generic(struct inode *inode, struct file *filp)
4357 ret = tracing_check_open_get_tr(NULL);
4361 filp->private_data = inode->i_private;
4365 bool tracing_is_disabled(void)
4367 return (tracing_disabled) ? true: false;
4371 * Open and update trace_array ref count.
4372 * Must have the current trace_array passed to it.
4374 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4376 struct trace_array *tr = inode->i_private;
4379 ret = tracing_check_open_get_tr(tr);
4383 filp->private_data = inode->i_private;
4388 static int tracing_release(struct inode *inode, struct file *file)
4390 struct trace_array *tr = inode->i_private;
4391 struct seq_file *m = file->private_data;
4392 struct trace_iterator *iter;
4395 if (!(file->f_mode & FMODE_READ)) {
4396 trace_array_put(tr);
4400 /* Writes do not use seq_file */
4402 mutex_lock(&trace_types_lock);
4404 for_each_tracing_cpu(cpu) {
4405 if (iter->buffer_iter[cpu])
4406 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4409 if (iter->trace && iter->trace->close)
4410 iter->trace->close(iter);
4412 if (!iter->snapshot && tr->stop_count)
4413 /* reenable tracing if it was previously enabled */
4414 tracing_start_tr(tr);
4416 __trace_array_put(tr);
4418 mutex_unlock(&trace_types_lock);
4420 mutex_destroy(&iter->mutex);
4421 free_cpumask_var(iter->started);
4424 kfree(iter->buffer_iter);
4425 seq_release_private(inode, file);
4430 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4432 struct trace_array *tr = inode->i_private;
4434 trace_array_put(tr);
4438 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4440 struct trace_array *tr = inode->i_private;
4442 trace_array_put(tr);
4444 return single_release(inode, file);
4447 static int tracing_open(struct inode *inode, struct file *file)
4449 struct trace_array *tr = inode->i_private;
4450 struct trace_iterator *iter;
4453 ret = tracing_check_open_get_tr(tr);
4457 /* If this file was open for write, then erase contents */
4458 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4459 int cpu = tracing_get_cpu(inode);
4460 struct array_buffer *trace_buf = &tr->array_buffer;
4462 #ifdef CONFIG_TRACER_MAX_TRACE
4463 if (tr->current_trace->print_max)
4464 trace_buf = &tr->max_buffer;
4467 if (cpu == RING_BUFFER_ALL_CPUS)
4468 tracing_reset_online_cpus(trace_buf);
4470 tracing_reset_cpu(trace_buf, cpu);
4473 if (file->f_mode & FMODE_READ) {
4474 iter = __tracing_open(inode, file, false);
4476 ret = PTR_ERR(iter);
4477 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4478 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4482 trace_array_put(tr);
4488 * Some tracers are not suitable for instance buffers.
4489 * A tracer is always available for the global array (toplevel)
4490 * or if it explicitly states that it is.
4493 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4495 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4498 /* Find the next tracer that this trace array may use */
4499 static struct tracer *
4500 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4502 while (t && !trace_ok_for_array(t, tr))
4509 t_next(struct seq_file *m, void *v, loff_t *pos)
4511 struct trace_array *tr = m->private;
4512 struct tracer *t = v;
4517 t = get_tracer_for_array(tr, t->next);
4522 static void *t_start(struct seq_file *m, loff_t *pos)
4524 struct trace_array *tr = m->private;
4528 mutex_lock(&trace_types_lock);
4530 t = get_tracer_for_array(tr, trace_types);
4531 for (; t && l < *pos; t = t_next(m, t, &l))
4537 static void t_stop(struct seq_file *m, void *p)
4539 mutex_unlock(&trace_types_lock);
4542 static int t_show(struct seq_file *m, void *v)
4544 struct tracer *t = v;
4549 seq_puts(m, t->name);
4558 static const struct seq_operations show_traces_seq_ops = {
4565 static int show_traces_open(struct inode *inode, struct file *file)
4567 struct trace_array *tr = inode->i_private;
4571 ret = tracing_check_open_get_tr(tr);
4575 ret = seq_open(file, &show_traces_seq_ops);
4577 trace_array_put(tr);
4581 m = file->private_data;
4587 static int show_traces_release(struct inode *inode, struct file *file)
4589 struct trace_array *tr = inode->i_private;
4591 trace_array_put(tr);
4592 return seq_release(inode, file);
4596 tracing_write_stub(struct file *filp, const char __user *ubuf,
4597 size_t count, loff_t *ppos)
4602 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4606 if (file->f_mode & FMODE_READ)
4607 ret = seq_lseek(file, offset, whence);
4609 file->f_pos = ret = 0;
4614 static const struct file_operations tracing_fops = {
4615 .open = tracing_open,
4617 .write = tracing_write_stub,
4618 .llseek = tracing_lseek,
4619 .release = tracing_release,
4622 static const struct file_operations show_traces_fops = {
4623 .open = show_traces_open,
4625 .llseek = seq_lseek,
4626 .release = show_traces_release,
4630 tracing_cpumask_read(struct file *filp, char __user *ubuf,
4631 size_t count, loff_t *ppos)
4633 struct trace_array *tr = file_inode(filp)->i_private;
4637 len = snprintf(NULL, 0, "%*pb\n",
4638 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4639 mask_str = kmalloc(len, GFP_KERNEL);
4643 len = snprintf(mask_str, len, "%*pb\n",
4644 cpumask_pr_args(tr->tracing_cpumask));
4649 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4657 int tracing_set_cpumask(struct trace_array *tr,
4658 cpumask_var_t tracing_cpumask_new)
4665 local_irq_disable();
4666 arch_spin_lock(&tr->max_lock);
4667 for_each_tracing_cpu(cpu) {
4669 * Increase/decrease the disabled counter if we are
4670 * about to flip a bit in the cpumask:
4672 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4673 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4674 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4675 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
4677 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4678 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4679 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
4680 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
4683 arch_spin_unlock(&tr->max_lock);
4686 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4692 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4693 size_t count, loff_t *ppos)
4695 struct trace_array *tr = file_inode(filp)->i_private;
4696 cpumask_var_t tracing_cpumask_new;
4699 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4702 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4706 err = tracing_set_cpumask(tr, tracing_cpumask_new);
4710 free_cpumask_var(tracing_cpumask_new);
4715 free_cpumask_var(tracing_cpumask_new);
4720 static const struct file_operations tracing_cpumask_fops = {
4721 .open = tracing_open_generic_tr,
4722 .read = tracing_cpumask_read,
4723 .write = tracing_cpumask_write,
4724 .release = tracing_release_generic_tr,
4725 .llseek = generic_file_llseek,
4728 static int tracing_trace_options_show(struct seq_file *m, void *v)
4730 struct tracer_opt *trace_opts;
4731 struct trace_array *tr = m->private;
4735 mutex_lock(&trace_types_lock);
4736 tracer_flags = tr->current_trace->flags->val;
4737 trace_opts = tr->current_trace->flags->opts;
4739 for (i = 0; trace_options[i]; i++) {
4740 if (tr->trace_flags & (1 << i))
4741 seq_printf(m, "%s\n", trace_options[i]);
4743 seq_printf(m, "no%s\n", trace_options[i]);
4746 for (i = 0; trace_opts[i].name; i++) {
4747 if (tracer_flags & trace_opts[i].bit)
4748 seq_printf(m, "%s\n", trace_opts[i].name);
4750 seq_printf(m, "no%s\n", trace_opts[i].name);
4752 mutex_unlock(&trace_types_lock);
4757 static int __set_tracer_option(struct trace_array *tr,
4758 struct tracer_flags *tracer_flags,
4759 struct tracer_opt *opts, int neg)
4761 struct tracer *trace = tracer_flags->trace;
4764 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4769 tracer_flags->val &= ~opts->bit;
4771 tracer_flags->val |= opts->bit;
4775 /* Try to assign a tracer specific option */
4776 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4778 struct tracer *trace = tr->current_trace;
4779 struct tracer_flags *tracer_flags = trace->flags;
4780 struct tracer_opt *opts = NULL;
4783 for (i = 0; tracer_flags->opts[i].name; i++) {
4784 opts = &tracer_flags->opts[i];
4786 if (strcmp(cmp, opts->name) == 0)
4787 return __set_tracer_option(tr, trace->flags, opts, neg);
4793 /* Some tracers require overwrite to stay enabled */
4794 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4796 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4802 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4804 if ((mask == TRACE_ITER_RECORD_TGID) ||
4805 (mask == TRACE_ITER_RECORD_CMD))
4806 lockdep_assert_held(&event_mutex);
4808 /* do nothing if flag is already set */
4809 if (!!(tr->trace_flags & mask) == !!enabled)
4812 /* Give the tracer a chance to approve the change */
4813 if (tr->current_trace->flag_changed)
4814 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4818 tr->trace_flags |= mask;
4820 tr->trace_flags &= ~mask;
4822 if (mask == TRACE_ITER_RECORD_CMD)
4823 trace_event_enable_cmd_record(enabled);
4825 if (mask == TRACE_ITER_RECORD_TGID) {
4827 tgid_map = kvcalloc(PID_MAX_DEFAULT + 1,
4831 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4835 trace_event_enable_tgid_record(enabled);
4838 if (mask == TRACE_ITER_EVENT_FORK)
4839 trace_event_follow_fork(tr, enabled);
4841 if (mask == TRACE_ITER_FUNC_FORK)
4842 ftrace_pid_follow_fork(tr, enabled);
4844 if (mask == TRACE_ITER_OVERWRITE) {
4845 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
4846 #ifdef CONFIG_TRACER_MAX_TRACE
4847 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4851 if (mask == TRACE_ITER_PRINTK) {
4852 trace_printk_start_stop_comm(enabled);
4853 trace_printk_control(enabled);
4859 int trace_set_options(struct trace_array *tr, char *option)
4864 size_t orig_len = strlen(option);
4867 cmp = strstrip(option);
4869 len = str_has_prefix(cmp, "no");
4875 mutex_lock(&event_mutex);
4876 mutex_lock(&trace_types_lock);
4878 ret = match_string(trace_options, -1, cmp);
4879 /* If no option could be set, test the specific tracer options */
4881 ret = set_tracer_option(tr, cmp, neg);
4883 ret = set_tracer_flag(tr, 1 << ret, !neg);
4885 mutex_unlock(&trace_types_lock);
4886 mutex_unlock(&event_mutex);
4889 * If the first trailing whitespace is replaced with '\0' by strstrip,
4890 * turn it back into a space.
4892 if (orig_len > strlen(option))
4893 option[strlen(option)] = ' ';
4898 static void __init apply_trace_boot_options(void)
4900 char *buf = trace_boot_options_buf;
4904 option = strsep(&buf, ",");
4910 trace_set_options(&global_trace, option);
4912 /* Put back the comma to allow this to be called again */
4919 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4920 size_t cnt, loff_t *ppos)
4922 struct seq_file *m = filp->private_data;
4923 struct trace_array *tr = m->private;
4927 if (cnt >= sizeof(buf))
4930 if (copy_from_user(buf, ubuf, cnt))
4935 ret = trace_set_options(tr, buf);
4944 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4946 struct trace_array *tr = inode->i_private;
4949 ret = tracing_check_open_get_tr(tr);
4953 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4955 trace_array_put(tr);
4960 static const struct file_operations tracing_iter_fops = {
4961 .open = tracing_trace_options_open,
4963 .llseek = seq_lseek,
4964 .release = tracing_single_release_tr,
4965 .write = tracing_trace_options_write,
4968 static const char readme_msg[] =
4969 "tracing mini-HOWTO:\n\n"
4970 "# echo 0 > tracing_on : quick way to disable tracing\n"
4971 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4972 " Important files:\n"
4973 " trace\t\t\t- The static contents of the buffer\n"
4974 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4975 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4976 " current_tracer\t- function and latency tracers\n"
4977 " available_tracers\t- list of configured tracers for current_tracer\n"
4978 " error_log\t- error log for failed commands (that support it)\n"
4979 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4980 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4981 " trace_clock\t\t-change the clock used to order events\n"
4982 " local: Per cpu clock but may not be synced across CPUs\n"
4983 " global: Synced across CPUs but slows tracing down.\n"
4984 " counter: Not a clock, but just an increment\n"
4985 " uptime: Jiffy counter from time of boot\n"
4986 " perf: Same clock that perf events use\n"
4987 #ifdef CONFIG_X86_64
4988 " x86-tsc: TSC cycle counter\n"
4990 "\n timestamp_mode\t-view the mode used to timestamp events\n"
4991 " delta: Delta difference against a buffer-wide timestamp\n"
4992 " absolute: Absolute (standalone) timestamp\n"
4993 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4994 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4995 " tracing_cpumask\t- Limit which CPUs to trace\n"
4996 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4997 "\t\t\t Remove sub-buffer with rmdir\n"
4998 " trace_options\t\t- Set format or modify how tracing happens\n"
4999 "\t\t\t Disable an option by prefixing 'no' to the\n"
5000 "\t\t\t option name\n"
5001 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5002 #ifdef CONFIG_DYNAMIC_FTRACE
5003 "\n available_filter_functions - list of functions that can be filtered on\n"
5004 " set_ftrace_filter\t- echo function name in here to only trace these\n"
5005 "\t\t\t functions\n"
5006 "\t accepts: func_full_name or glob-matching-pattern\n"
5007 "\t modules: Can select a group via module\n"
5008 "\t Format: :mod:<module-name>\n"
5009 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
5010 "\t triggers: a command to perform when function is hit\n"
5011 "\t Format: <function>:<trigger>[:count]\n"
5012 "\t trigger: traceon, traceoff\n"
5013 "\t\t enable_event:<system>:<event>\n"
5014 "\t\t disable_event:<system>:<event>\n"
5015 #ifdef CONFIG_STACKTRACE
5018 #ifdef CONFIG_TRACER_SNAPSHOT
5023 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5024 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5025 "\t The first one will disable tracing every time do_fault is hit\n"
5026 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
5027 "\t The first time do trap is hit and it disables tracing, the\n"
5028 "\t counter will decrement to 2. If tracing is already disabled,\n"
5029 "\t the counter will not decrement. It only decrements when the\n"
5030 "\t trigger did work\n"
5031 "\t To remove trigger without count:\n"
5032 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5033 "\t To remove trigger with a count:\n"
5034 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5035 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
5036 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5037 "\t modules: Can select a group via module command :mod:\n"
5038 "\t Does not accept triggers\n"
5039 #endif /* CONFIG_DYNAMIC_FTRACE */
5040 #ifdef CONFIG_FUNCTION_TRACER
5041 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5043 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5046 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5047 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5048 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5049 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5051 #ifdef CONFIG_TRACER_SNAPSHOT
5052 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5053 "\t\t\t snapshot buffer. Read the contents for more\n"
5054 "\t\t\t information\n"
5056 #ifdef CONFIG_STACK_TRACER
5057 " stack_trace\t\t- Shows the max stack trace when active\n"
5058 " stack_max_size\t- Shows current max stack size that was traced\n"
5059 "\t\t\t Write into this file to reset the max size (trigger a\n"
5060 "\t\t\t new trace)\n"
5061 #ifdef CONFIG_DYNAMIC_FTRACE
5062 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5065 #endif /* CONFIG_STACK_TRACER */
5066 #ifdef CONFIG_DYNAMIC_EVENTS
5067 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5068 "\t\t\t Write into this file to define/undefine new trace events.\n"
5070 #ifdef CONFIG_KPROBE_EVENTS
5071 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5072 "\t\t\t Write into this file to define/undefine new trace events.\n"
5074 #ifdef CONFIG_UPROBE_EVENTS
5075 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5076 "\t\t\t Write into this file to define/undefine new trace events.\n"
5078 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5079 "\t accepts: event-definitions (one definition per line)\n"
5080 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
5081 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
5082 #ifdef CONFIG_HIST_TRIGGERS
5083 "\t s:[synthetic/]<event> <field> [<field>]\n"
5085 "\t -:[<group>/]<event>\n"
5086 #ifdef CONFIG_KPROBE_EVENTS
5087 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5088 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5090 #ifdef CONFIG_UPROBE_EVENTS
5091 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
5093 "\t args: <name>=fetcharg[:type]\n"
5094 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
5095 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5096 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5098 "\t $stack<index>, $stack, $retval, $comm,\n"
5100 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5101 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
5102 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5103 "\t <type>\\[<array-size>\\]\n"
5104 #ifdef CONFIG_HIST_TRIGGERS
5105 "\t field: <stype> <name>;\n"
5106 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5107 "\t [unsigned] char/int/long\n"
5110 " events/\t\t- Directory containing all trace event subsystems:\n"
5111 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5112 " events/<system>/\t- Directory containing all trace events for <system>:\n"
5113 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5115 " filter\t\t- If set, only events passing filter are traced\n"
5116 " events/<system>/<event>/\t- Directory containing control files for\n"
5118 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5119 " filter\t\t- If set, only events passing filter are traced\n"
5120 " trigger\t\t- If set, a command to perform when event is hit\n"
5121 "\t Format: <trigger>[:count][if <filter>]\n"
5122 "\t trigger: traceon, traceoff\n"
5123 "\t enable_event:<system>:<event>\n"
5124 "\t disable_event:<system>:<event>\n"
5125 #ifdef CONFIG_HIST_TRIGGERS
5126 "\t enable_hist:<system>:<event>\n"
5127 "\t disable_hist:<system>:<event>\n"
5129 #ifdef CONFIG_STACKTRACE
5132 #ifdef CONFIG_TRACER_SNAPSHOT
5135 #ifdef CONFIG_HIST_TRIGGERS
5136 "\t\t hist (see below)\n"
5138 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
5139 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5140 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5141 "\t events/block/block_unplug/trigger\n"
5142 "\t The first disables tracing every time block_unplug is hit.\n"
5143 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
5144 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
5145 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5146 "\t Like function triggers, the counter is only decremented if it\n"
5147 "\t enabled or disabled tracing.\n"
5148 "\t To remove a trigger without a count:\n"
5149 "\t echo '!<trigger> > <system>/<event>/trigger\n"
5150 "\t To remove a trigger with a count:\n"
5151 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5152 "\t Filters can be ignored when removing a trigger.\n"
5153 #ifdef CONFIG_HIST_TRIGGERS
5154 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
5155 "\t Format: hist:keys=<field1[,field2,...]>\n"
5156 "\t [:values=<field1[,field2,...]>]\n"
5157 "\t [:sort=<field1[,field2,...]>]\n"
5158 "\t [:size=#entries]\n"
5159 "\t [:pause][:continue][:clear]\n"
5160 "\t [:name=histname1]\n"
5161 "\t [:<handler>.<action>]\n"
5162 "\t [if <filter>]\n\n"
5163 "\t When a matching event is hit, an entry is added to a hash\n"
5164 "\t table using the key(s) and value(s) named, and the value of a\n"
5165 "\t sum called 'hitcount' is incremented. Keys and values\n"
5166 "\t correspond to fields in the event's format description. Keys\n"
5167 "\t can be any field, or the special string 'stacktrace'.\n"
5168 "\t Compound keys consisting of up to two fields can be specified\n"
5169 "\t by the 'keys' keyword. Values must correspond to numeric\n"
5170 "\t fields. Sort keys consisting of up to two fields can be\n"
5171 "\t specified using the 'sort' keyword. The sort direction can\n"
5172 "\t be modified by appending '.descending' or '.ascending' to a\n"
5173 "\t sort field. The 'size' parameter can be used to specify more\n"
5174 "\t or fewer than the default 2048 entries for the hashtable size.\n"
5175 "\t If a hist trigger is given a name using the 'name' parameter,\n"
5176 "\t its histogram data will be shared with other triggers of the\n"
5177 "\t same name, and trigger hits will update this common data.\n\n"
5178 "\t Reading the 'hist' file for the event will dump the hash\n"
5179 "\t table in its entirety to stdout. If there are multiple hist\n"
5180 "\t triggers attached to an event, there will be a table for each\n"
5181 "\t trigger in the output. The table displayed for a named\n"
5182 "\t trigger will be the same as any other instance having the\n"
5183 "\t same name. The default format used to display a given field\n"
5184 "\t can be modified by appending any of the following modifiers\n"
5185 "\t to the field name, as applicable:\n\n"
5186 "\t .hex display a number as a hex value\n"
5187 "\t .sym display an address as a symbol\n"
5188 "\t .sym-offset display an address as a symbol and offset\n"
5189 "\t .execname display a common_pid as a program name\n"
5190 "\t .syscall display a syscall id as a syscall name\n"
5191 "\t .log2 display log2 value rather than raw number\n"
5192 "\t .usecs display a common_timestamp in microseconds\n\n"
5193 "\t The 'pause' parameter can be used to pause an existing hist\n"
5194 "\t trigger or to start a hist trigger but not log any events\n"
5195 "\t until told to do so. 'continue' can be used to start or\n"
5196 "\t restart a paused hist trigger.\n\n"
5197 "\t The 'clear' parameter will clear the contents of a running\n"
5198 "\t hist trigger and leave its current paused/active state\n"
5200 "\t The enable_hist and disable_hist triggers can be used to\n"
5201 "\t have one event conditionally start and stop another event's\n"
5202 "\t already-attached hist trigger. The syntax is analogous to\n"
5203 "\t the enable_event and disable_event triggers.\n\n"
5204 "\t Hist trigger handlers and actions are executed whenever a\n"
5205 "\t a histogram entry is added or updated. They take the form:\n\n"
5206 "\t <handler>.<action>\n\n"
5207 "\t The available handlers are:\n\n"
5208 "\t onmatch(matching.event) - invoke on addition or update\n"
5209 "\t onmax(var) - invoke if var exceeds current max\n"
5210 "\t onchange(var) - invoke action if var changes\n\n"
5211 "\t The available actions are:\n\n"
5212 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5213 "\t save(field,...) - save current event fields\n"
5214 #ifdef CONFIG_TRACER_SNAPSHOT
5215 "\t snapshot() - snapshot the trace buffer\n"
5221 tracing_readme_read(struct file *filp, char __user *ubuf,
5222 size_t cnt, loff_t *ppos)
5224 return simple_read_from_buffer(ubuf, cnt, ppos,
5225 readme_msg, strlen(readme_msg));
5228 static const struct file_operations tracing_readme_fops = {
5229 .open = tracing_open_generic,
5230 .read = tracing_readme_read,
5231 .llseek = generic_file_llseek,
5234 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
5238 if (*pos || m->count)
5243 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
5244 if (trace_find_tgid(*ptr))
5251 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5261 v = saved_tgids_next(m, v, &l);
5269 static void saved_tgids_stop(struct seq_file *m, void *v)
5273 static int saved_tgids_show(struct seq_file *m, void *v)
5275 int pid = (int *)v - tgid_map;
5277 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5281 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5282 .start = saved_tgids_start,
5283 .stop = saved_tgids_stop,
5284 .next = saved_tgids_next,
5285 .show = saved_tgids_show,
5288 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5292 ret = tracing_check_open_get_tr(NULL);
5296 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5300 static const struct file_operations tracing_saved_tgids_fops = {
5301 .open = tracing_saved_tgids_open,
5303 .llseek = seq_lseek,
5304 .release = seq_release,
5307 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5309 unsigned int *ptr = v;
5311 if (*pos || m->count)
5316 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5318 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5327 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5333 arch_spin_lock(&trace_cmdline_lock);
5335 v = &savedcmd->map_cmdline_to_pid[0];
5337 v = saved_cmdlines_next(m, v, &l);
5345 static void saved_cmdlines_stop(struct seq_file *m, void *v)
5347 arch_spin_unlock(&trace_cmdline_lock);
5351 static int saved_cmdlines_show(struct seq_file *m, void *v)
5353 char buf[TASK_COMM_LEN];
5354 unsigned int *pid = v;
5356 __trace_find_cmdline(*pid, buf);
5357 seq_printf(m, "%d %s\n", *pid, buf);
5361 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5362 .start = saved_cmdlines_start,
5363 .next = saved_cmdlines_next,
5364 .stop = saved_cmdlines_stop,
5365 .show = saved_cmdlines_show,
5368 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5372 ret = tracing_check_open_get_tr(NULL);
5376 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5379 static const struct file_operations tracing_saved_cmdlines_fops = {
5380 .open = tracing_saved_cmdlines_open,
5382 .llseek = seq_lseek,
5383 .release = seq_release,
5387 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5388 size_t cnt, loff_t *ppos)
5393 arch_spin_lock(&trace_cmdline_lock);
5394 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5395 arch_spin_unlock(&trace_cmdline_lock);
5397 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5400 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5402 kfree(s->saved_cmdlines);
5403 kfree(s->map_cmdline_to_pid);
5407 static int tracing_resize_saved_cmdlines(unsigned int val)
5409 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5411 s = kmalloc(sizeof(*s), GFP_KERNEL);
5415 if (allocate_cmdlines_buffer(val, s) < 0) {
5420 arch_spin_lock(&trace_cmdline_lock);
5421 savedcmd_temp = savedcmd;
5423 arch_spin_unlock(&trace_cmdline_lock);
5424 free_saved_cmdlines_buffer(savedcmd_temp);
5430 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5431 size_t cnt, loff_t *ppos)
5436 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5440 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5441 if (!val || val > PID_MAX_DEFAULT)
5444 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5453 static const struct file_operations tracing_saved_cmdlines_size_fops = {
5454 .open = tracing_open_generic,
5455 .read = tracing_saved_cmdlines_size_read,
5456 .write = tracing_saved_cmdlines_size_write,
5459 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5460 static union trace_eval_map_item *
5461 update_eval_map(union trace_eval_map_item *ptr)
5463 if (!ptr->map.eval_string) {
5464 if (ptr->tail.next) {
5465 ptr = ptr->tail.next;
5466 /* Set ptr to the next real item (skip head) */
5474 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5476 union trace_eval_map_item *ptr = v;
5479 * Paranoid! If ptr points to end, we don't want to increment past it.
5480 * This really should never happen.
5483 ptr = update_eval_map(ptr);
5484 if (WARN_ON_ONCE(!ptr))
5488 ptr = update_eval_map(ptr);
5493 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5495 union trace_eval_map_item *v;
5498 mutex_lock(&trace_eval_mutex);
5500 v = trace_eval_maps;
5504 while (v && l < *pos) {
5505 v = eval_map_next(m, v, &l);
5511 static void eval_map_stop(struct seq_file *m, void *v)
5513 mutex_unlock(&trace_eval_mutex);
5516 static int eval_map_show(struct seq_file *m, void *v)
5518 union trace_eval_map_item *ptr = v;
5520 seq_printf(m, "%s %ld (%s)\n",
5521 ptr->map.eval_string, ptr->map.eval_value,
5527 static const struct seq_operations tracing_eval_map_seq_ops = {
5528 .start = eval_map_start,
5529 .next = eval_map_next,
5530 .stop = eval_map_stop,
5531 .show = eval_map_show,
5534 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5538 ret = tracing_check_open_get_tr(NULL);
5542 return seq_open(filp, &tracing_eval_map_seq_ops);
5545 static const struct file_operations tracing_eval_map_fops = {
5546 .open = tracing_eval_map_open,
5548 .llseek = seq_lseek,
5549 .release = seq_release,
5552 static inline union trace_eval_map_item *
5553 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5555 /* Return tail of array given the head */
5556 return ptr + ptr->head.length + 1;
5560 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5563 struct trace_eval_map **stop;
5564 struct trace_eval_map **map;
5565 union trace_eval_map_item *map_array;
5566 union trace_eval_map_item *ptr;
5571 * The trace_eval_maps contains the map plus a head and tail item,
5572 * where the head holds the module and length of array, and the
5573 * tail holds a pointer to the next list.
5575 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5577 pr_warn("Unable to allocate trace eval mapping\n");
5581 mutex_lock(&trace_eval_mutex);
5583 if (!trace_eval_maps)
5584 trace_eval_maps = map_array;
5586 ptr = trace_eval_maps;
5588 ptr = trace_eval_jmp_to_tail(ptr);
5589 if (!ptr->tail.next)
5591 ptr = ptr->tail.next;
5594 ptr->tail.next = map_array;
5596 map_array->head.mod = mod;
5597 map_array->head.length = len;
5600 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5601 map_array->map = **map;
5604 memset(map_array, 0, sizeof(*map_array));
5606 mutex_unlock(&trace_eval_mutex);
5609 static void trace_create_eval_file(struct dentry *d_tracer)
5611 trace_create_file("eval_map", 0444, d_tracer,
5612 NULL, &tracing_eval_map_fops);
5615 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5616 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5617 static inline void trace_insert_eval_map_file(struct module *mod,
5618 struct trace_eval_map **start, int len) { }
5619 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5621 static void trace_insert_eval_map(struct module *mod,
5622 struct trace_eval_map **start, int len)
5624 struct trace_eval_map **map;
5631 trace_event_eval_update(map, len);
5633 trace_insert_eval_map_file(mod, start, len);
5637 tracing_set_trace_read(struct file *filp, char __user *ubuf,
5638 size_t cnt, loff_t *ppos)
5640 struct trace_array *tr = filp->private_data;
5641 char buf[MAX_TRACER_SIZE+2];
5644 mutex_lock(&trace_types_lock);
5645 r = sprintf(buf, "%s\n", tr->current_trace->name);
5646 mutex_unlock(&trace_types_lock);
5648 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5651 int tracer_init(struct tracer *t, struct trace_array *tr)
5653 tracing_reset_online_cpus(&tr->array_buffer);
5657 static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
5661 for_each_tracing_cpu(cpu)
5662 per_cpu_ptr(buf->data, cpu)->entries = val;
5665 #ifdef CONFIG_TRACER_MAX_TRACE
5666 /* resize @tr's buffer to the size of @size_tr's entries */
5667 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
5668 struct array_buffer *size_buf, int cpu_id)
5672 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5673 for_each_tracing_cpu(cpu) {
5674 ret = ring_buffer_resize(trace_buf->buffer,
5675 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5678 per_cpu_ptr(trace_buf->data, cpu)->entries =
5679 per_cpu_ptr(size_buf->data, cpu)->entries;
5682 ret = ring_buffer_resize(trace_buf->buffer,
5683 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5685 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5686 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5691 #endif /* CONFIG_TRACER_MAX_TRACE */
5693 static int __tracing_resize_ring_buffer(struct trace_array *tr,
5694 unsigned long size, int cpu)
5699 * If kernel or user changes the size of the ring buffer
5700 * we use the size that was given, and we can forget about
5701 * expanding it later.
5703 ring_buffer_expanded = true;
5705 /* May be called before buffers are initialized */
5706 if (!tr->array_buffer.buffer)
5709 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
5713 #ifdef CONFIG_TRACER_MAX_TRACE
5714 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5715 !tr->current_trace->use_max_tr)
5718 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5720 int r = resize_buffer_duplicate_size(&tr->array_buffer,
5721 &tr->array_buffer, cpu);
5724 * AARGH! We are left with different
5725 * size max buffer!!!!
5726 * The max buffer is our "snapshot" buffer.
5727 * When a tracer needs a snapshot (one of the
5728 * latency tracers), it swaps the max buffer
5729 * with the saved snap shot. We succeeded to
5730 * update the size of the main buffer, but failed to
5731 * update the size of the max buffer. But when we tried
5732 * to reset the main buffer to the original size, we
5733 * failed there too. This is very unlikely to
5734 * happen, but if it does, warn and kill all
5738 tracing_disabled = 1;
5743 if (cpu == RING_BUFFER_ALL_CPUS)
5744 set_buffer_entries(&tr->max_buffer, size);
5746 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5749 #endif /* CONFIG_TRACER_MAX_TRACE */
5751 if (cpu == RING_BUFFER_ALL_CPUS)
5752 set_buffer_entries(&tr->array_buffer, size);
5754 per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
5759 ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5760 unsigned long size, int cpu_id)
5764 mutex_lock(&trace_types_lock);
5766 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5767 /* make sure, this cpu is enabled in the mask */
5768 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5774 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5779 mutex_unlock(&trace_types_lock);
5786 * tracing_update_buffers - used by tracing facility to expand ring buffers
5788 * To save on memory when the tracing is never used on a system with it
5789 * configured in. The ring buffers are set to a minimum size. But once
5790 * a user starts to use the tracing facility, then they need to grow
5791 * to their default size.
5793 * This function is to be called when a tracer is about to be used.
5795 int tracing_update_buffers(void)
5799 mutex_lock(&trace_types_lock);
5800 if (!ring_buffer_expanded)
5801 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5802 RING_BUFFER_ALL_CPUS);
5803 mutex_unlock(&trace_types_lock);
5808 struct trace_option_dentry;
5811 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5814 * Used to clear out the tracer before deletion of an instance.
5815 * Must have trace_types_lock held.
5817 static void tracing_set_nop(struct trace_array *tr)
5819 if (tr->current_trace == &nop_trace)
5822 tr->current_trace->enabled--;
5824 if (tr->current_trace->reset)
5825 tr->current_trace->reset(tr);
5827 tr->current_trace = &nop_trace;
5830 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5832 /* Only enable if the directory has been created already. */
5836 create_trace_option_files(tr, t);
5839 int tracing_set_tracer(struct trace_array *tr, const char *buf)
5842 #ifdef CONFIG_TRACER_MAX_TRACE
5847 mutex_lock(&trace_types_lock);
5849 if (!ring_buffer_expanded) {
5850 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5851 RING_BUFFER_ALL_CPUS);
5857 for (t = trace_types; t; t = t->next) {
5858 if (strcmp(t->name, buf) == 0)
5865 if (t == tr->current_trace)
5868 #ifdef CONFIG_TRACER_SNAPSHOT
5869 if (t->use_max_tr) {
5870 arch_spin_lock(&tr->max_lock);
5871 if (tr->cond_snapshot)
5873 arch_spin_unlock(&tr->max_lock);
5878 /* Some tracers won't work on kernel command line */
5879 if (system_state < SYSTEM_RUNNING && t->noboot) {
5880 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5885 /* Some tracers are only allowed for the top level buffer */
5886 if (!trace_ok_for_array(t, tr)) {
5891 /* If trace pipe files are being read, we can't change the tracer */
5892 if (tr->trace_ref) {
5897 trace_branch_disable();
5899 tr->current_trace->enabled--;
5901 if (tr->current_trace->reset)
5902 tr->current_trace->reset(tr);
5904 /* Current trace needs to be nop_trace before synchronize_rcu */
5905 tr->current_trace = &nop_trace;
5907 #ifdef CONFIG_TRACER_MAX_TRACE
5908 had_max_tr = tr->allocated_snapshot;
5910 if (had_max_tr && !t->use_max_tr) {
5912 * We need to make sure that the update_max_tr sees that
5913 * current_trace changed to nop_trace to keep it from
5914 * swapping the buffers after we resize it.
5915 * The update_max_tr is called from interrupts disabled
5916 * so a synchronized_sched() is sufficient.
5923 #ifdef CONFIG_TRACER_MAX_TRACE
5924 if (t->use_max_tr && !had_max_tr) {
5925 ret = tracing_alloc_snapshot_instance(tr);
5932 ret = tracer_init(t, tr);
5937 tr->current_trace = t;
5938 tr->current_trace->enabled++;
5939 trace_branch_enable(tr);
5941 mutex_unlock(&trace_types_lock);
5947 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5948 size_t cnt, loff_t *ppos)
5950 struct trace_array *tr = filp->private_data;
5951 char buf[MAX_TRACER_SIZE+1];
5958 if (cnt > MAX_TRACER_SIZE)
5959 cnt = MAX_TRACER_SIZE;
5961 if (copy_from_user(buf, ubuf, cnt))
5966 /* strip ending whitespace. */
5967 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5970 err = tracing_set_tracer(tr, buf);
5980 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5981 size_t cnt, loff_t *ppos)
5986 r = snprintf(buf, sizeof(buf), "%ld\n",
5987 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5988 if (r > sizeof(buf))
5990 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5994 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5995 size_t cnt, loff_t *ppos)
6000 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6010 tracing_thresh_read(struct file *filp, char __user *ubuf,
6011 size_t cnt, loff_t *ppos)
6013 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
6017 tracing_thresh_write(struct file *filp, const char __user *ubuf,
6018 size_t cnt, loff_t *ppos)
6020 struct trace_array *tr = filp->private_data;
6023 mutex_lock(&trace_types_lock);
6024 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
6028 if (tr->current_trace->update_thresh) {
6029 ret = tr->current_trace->update_thresh(tr);
6036 mutex_unlock(&trace_types_lock);
6041 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6044 tracing_max_lat_read(struct file *filp, char __user *ubuf,
6045 size_t cnt, loff_t *ppos)
6047 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
6051 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6052 size_t cnt, loff_t *ppos)
6054 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
6059 static int tracing_open_pipe(struct inode *inode, struct file *filp)
6061 struct trace_array *tr = inode->i_private;
6062 struct trace_iterator *iter;
6065 ret = tracing_check_open_get_tr(tr);
6069 mutex_lock(&trace_types_lock);
6071 /* create a buffer to store the information to pass to userspace */
6072 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6075 __trace_array_put(tr);
6079 trace_seq_init(&iter->seq);
6080 iter->trace = tr->current_trace;
6082 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
6087 /* trace pipe does not show start of buffer */
6088 cpumask_setall(iter->started);
6090 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6091 iter->iter_flags |= TRACE_FILE_LAT_FMT;
6093 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6094 if (trace_clocks[tr->clock_id].in_ns)
6095 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6098 iter->array_buffer = &tr->array_buffer;
6099 iter->cpu_file = tracing_get_cpu(inode);
6100 mutex_init(&iter->mutex);
6101 filp->private_data = iter;
6103 if (iter->trace->pipe_open)
6104 iter->trace->pipe_open(iter);
6106 nonseekable_open(inode, filp);
6110 mutex_unlock(&trace_types_lock);
6115 __trace_array_put(tr);
6116 mutex_unlock(&trace_types_lock);
6120 static int tracing_release_pipe(struct inode *inode, struct file *file)
6122 struct trace_iterator *iter = file->private_data;
6123 struct trace_array *tr = inode->i_private;
6125 mutex_lock(&trace_types_lock);
6129 if (iter->trace->pipe_close)
6130 iter->trace->pipe_close(iter);
6132 mutex_unlock(&trace_types_lock);
6134 free_cpumask_var(iter->started);
6135 mutex_destroy(&iter->mutex);
6138 trace_array_put(tr);
6144 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6146 struct trace_array *tr = iter->tr;
6148 /* Iterators are static, they should be filled or empty */
6149 if (trace_buffer_iter(iter, iter->cpu_file))
6150 return EPOLLIN | EPOLLRDNORM;
6152 if (tr->trace_flags & TRACE_ITER_BLOCK)
6154 * Always select as readable when in blocking mode
6156 return EPOLLIN | EPOLLRDNORM;
6158 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
6163 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6165 struct trace_iterator *iter = filp->private_data;
6167 return trace_poll(iter, filp, poll_table);
6170 /* Must be called with iter->mutex held. */
6171 static int tracing_wait_pipe(struct file *filp)
6173 struct trace_iterator *iter = filp->private_data;
6176 while (trace_empty(iter)) {
6178 if ((filp->f_flags & O_NONBLOCK)) {
6183 * We block until we read something and tracing is disabled.
6184 * We still block if tracing is disabled, but we have never
6185 * read anything. This allows a user to cat this file, and
6186 * then enable tracing. But after we have read something,
6187 * we give an EOF when tracing is again disabled.
6189 * iter->pos will be 0 if we haven't read anything.
6191 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
6194 mutex_unlock(&iter->mutex);
6196 ret = wait_on_pipe(iter, 0);
6198 mutex_lock(&iter->mutex);
6211 tracing_read_pipe(struct file *filp, char __user *ubuf,
6212 size_t cnt, loff_t *ppos)
6214 struct trace_iterator *iter = filp->private_data;
6218 * Avoid more than one consumer on a single file descriptor
6219 * This is just a matter of traces coherency, the ring buffer itself
6222 mutex_lock(&iter->mutex);
6224 /* return any leftover data */
6225 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6229 trace_seq_init(&iter->seq);
6231 if (iter->trace->read) {
6232 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6238 sret = tracing_wait_pipe(filp);
6242 /* stop when tracing is finished */
6243 if (trace_empty(iter)) {
6248 if (cnt >= PAGE_SIZE)
6249 cnt = PAGE_SIZE - 1;
6251 /* reset all but tr, trace, and overruns */
6252 memset(&iter->seq, 0,
6253 sizeof(struct trace_iterator) -
6254 offsetof(struct trace_iterator, seq));
6255 cpumask_clear(iter->started);
6256 trace_seq_init(&iter->seq);
6259 trace_event_read_lock();
6260 trace_access_lock(iter->cpu_file);
6261 while (trace_find_next_entry_inc(iter) != NULL) {
6262 enum print_line_t ret;
6263 int save_len = iter->seq.seq.len;
6265 ret = print_trace_line(iter);
6266 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6267 /* don't print partial lines */
6268 iter->seq.seq.len = save_len;
6271 if (ret != TRACE_TYPE_NO_CONSUME)
6272 trace_consume(iter);
6274 if (trace_seq_used(&iter->seq) >= cnt)
6278 * Setting the full flag means we reached the trace_seq buffer
6279 * size and we should leave by partial output condition above.
6280 * One of the trace_seq_* functions is not used properly.
6282 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6285 trace_access_unlock(iter->cpu_file);
6286 trace_event_read_unlock();
6288 /* Now copy what we have to the user */
6289 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6290 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6291 trace_seq_init(&iter->seq);
6294 * If there was nothing to send to user, in spite of consuming trace
6295 * entries, go back to wait for more entries.
6301 mutex_unlock(&iter->mutex);
6306 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6309 __free_page(spd->pages[idx]);
6313 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6319 /* Seq buffer is page-sized, exactly what we need. */
6321 save_len = iter->seq.seq.len;
6322 ret = print_trace_line(iter);
6324 if (trace_seq_has_overflowed(&iter->seq)) {
6325 iter->seq.seq.len = save_len;
6330 * This should not be hit, because it should only
6331 * be set if the iter->seq overflowed. But check it
6332 * anyway to be safe.
6334 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6335 iter->seq.seq.len = save_len;
6339 count = trace_seq_used(&iter->seq) - save_len;
6342 iter->seq.seq.len = save_len;
6346 if (ret != TRACE_TYPE_NO_CONSUME)
6347 trace_consume(iter);
6349 if (!trace_find_next_entry_inc(iter)) {
6359 static ssize_t tracing_splice_read_pipe(struct file *filp,
6361 struct pipe_inode_info *pipe,
6365 struct page *pages_def[PIPE_DEF_BUFFERS];
6366 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6367 struct trace_iterator *iter = filp->private_data;
6368 struct splice_pipe_desc spd = {
6370 .partial = partial_def,
6371 .nr_pages = 0, /* This gets updated below. */
6372 .nr_pages_max = PIPE_DEF_BUFFERS,
6373 .ops = &default_pipe_buf_ops,
6374 .spd_release = tracing_spd_release_pipe,
6380 if (splice_grow_spd(pipe, &spd))
6383 mutex_lock(&iter->mutex);
6385 if (iter->trace->splice_read) {
6386 ret = iter->trace->splice_read(iter, filp,
6387 ppos, pipe, len, flags);
6392 ret = tracing_wait_pipe(filp);
6396 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6401 trace_event_read_lock();
6402 trace_access_lock(iter->cpu_file);
6404 /* Fill as many pages as possible. */
6405 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6406 spd.pages[i] = alloc_page(GFP_KERNEL);
6410 rem = tracing_fill_pipe_page(rem, iter);
6412 /* Copy the data into the page, so we can start over. */
6413 ret = trace_seq_to_buffer(&iter->seq,
6414 page_address(spd.pages[i]),
6415 trace_seq_used(&iter->seq));
6417 __free_page(spd.pages[i]);
6420 spd.partial[i].offset = 0;
6421 spd.partial[i].len = trace_seq_used(&iter->seq);
6423 trace_seq_init(&iter->seq);
6426 trace_access_unlock(iter->cpu_file);
6427 trace_event_read_unlock();
6428 mutex_unlock(&iter->mutex);
6433 ret = splice_to_pipe(pipe, &spd);
6437 splice_shrink_spd(&spd);
6441 mutex_unlock(&iter->mutex);
6446 tracing_entries_read(struct file *filp, char __user *ubuf,
6447 size_t cnt, loff_t *ppos)
6449 struct inode *inode = file_inode(filp);
6450 struct trace_array *tr = inode->i_private;
6451 int cpu = tracing_get_cpu(inode);
6456 mutex_lock(&trace_types_lock);
6458 if (cpu == RING_BUFFER_ALL_CPUS) {
6459 int cpu, buf_size_same;
6464 /* check if all cpu sizes are same */
6465 for_each_tracing_cpu(cpu) {
6466 /* fill in the size from first enabled cpu */
6468 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6469 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6475 if (buf_size_same) {
6476 if (!ring_buffer_expanded)
6477 r = sprintf(buf, "%lu (expanded: %lu)\n",
6479 trace_buf_size >> 10);
6481 r = sprintf(buf, "%lu\n", size >> 10);
6483 r = sprintf(buf, "X\n");
6485 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6487 mutex_unlock(&trace_types_lock);
6489 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6494 tracing_entries_write(struct file *filp, const char __user *ubuf,
6495 size_t cnt, loff_t *ppos)
6497 struct inode *inode = file_inode(filp);
6498 struct trace_array *tr = inode->i_private;
6502 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6506 /* must have at least 1 entry */
6510 /* value is in KB */
6512 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6522 tracing_total_entries_read(struct file *filp, char __user *ubuf,
6523 size_t cnt, loff_t *ppos)
6525 struct trace_array *tr = filp->private_data;
6528 unsigned long size = 0, expanded_size = 0;
6530 mutex_lock(&trace_types_lock);
6531 for_each_tracing_cpu(cpu) {
6532 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
6533 if (!ring_buffer_expanded)
6534 expanded_size += trace_buf_size >> 10;
6536 if (ring_buffer_expanded)
6537 r = sprintf(buf, "%lu\n", size);
6539 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6540 mutex_unlock(&trace_types_lock);
6542 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6546 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6547 size_t cnt, loff_t *ppos)
6550 * There is no need to read what the user has written, this function
6551 * is just to make sure that there is no error when "echo" is used
6560 tracing_free_buffer_release(struct inode *inode, struct file *filp)
6562 struct trace_array *tr = inode->i_private;
6564 /* disable tracing ? */
6565 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6566 tracer_tracing_off(tr);
6567 /* resize the ring buffer to 0 */
6568 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6570 trace_array_put(tr);
6576 tracing_mark_write(struct file *filp, const char __user *ubuf,
6577 size_t cnt, loff_t *fpos)
6579 struct trace_array *tr = filp->private_data;
6580 struct ring_buffer_event *event;
6581 enum event_trigger_type tt = ETT_NONE;
6582 struct trace_buffer *buffer;
6583 struct print_entry *entry;
6584 unsigned long irq_flags;
6589 /* Used in tracing_mark_raw_write() as well */
6590 #define FAULTED_STR "<faulted>"
6591 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6593 if (tracing_disabled)
6596 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6599 if (cnt > TRACE_BUF_SIZE)
6600 cnt = TRACE_BUF_SIZE;
6602 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6604 local_save_flags(irq_flags);
6605 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6607 /* If less than "<faulted>", then make sure we can still add that */
6608 if (cnt < FAULTED_SIZE)
6609 size += FAULTED_SIZE - cnt;
6611 buffer = tr->array_buffer.buffer;
6612 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6613 irq_flags, preempt_count());
6614 if (unlikely(!event))
6615 /* Ring buffer disabled, return as if not open for write */
6618 entry = ring_buffer_event_data(event);
6619 entry->ip = _THIS_IP_;
6621 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6623 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6630 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6631 /* do not add \n before testing triggers, but add \0 */
6632 entry->buf[cnt] = '\0';
6633 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6636 if (entry->buf[cnt - 1] != '\n') {
6637 entry->buf[cnt] = '\n';
6638 entry->buf[cnt + 1] = '\0';
6640 entry->buf[cnt] = '\0';
6642 __buffer_unlock_commit(buffer, event);
6645 event_triggers_post_call(tr->trace_marker_file, tt);
6653 /* Limit it for now to 3K (including tag) */
6654 #define RAW_DATA_MAX_SIZE (1024*3)
6657 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6658 size_t cnt, loff_t *fpos)
6660 struct trace_array *tr = filp->private_data;
6661 struct ring_buffer_event *event;
6662 struct trace_buffer *buffer;
6663 struct raw_data_entry *entry;
6664 unsigned long irq_flags;
6669 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6671 if (tracing_disabled)
6674 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6677 /* The marker must at least have a tag id */
6678 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6681 if (cnt > TRACE_BUF_SIZE)
6682 cnt = TRACE_BUF_SIZE;
6684 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6686 local_save_flags(irq_flags);
6687 size = sizeof(*entry) + cnt;
6688 if (cnt < FAULT_SIZE_ID)
6689 size += FAULT_SIZE_ID - cnt;
6691 buffer = tr->array_buffer.buffer;
6692 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6693 irq_flags, preempt_count());
6695 /* Ring buffer disabled, return as if not open for write */
6698 entry = ring_buffer_event_data(event);
6700 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6703 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6708 __buffer_unlock_commit(buffer, event);
6716 static int tracing_clock_show(struct seq_file *m, void *v)
6718 struct trace_array *tr = m->private;
6721 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6723 "%s%s%s%s", i ? " " : "",
6724 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6725 i == tr->clock_id ? "]" : "");
6731 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6735 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6736 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6739 if (i == ARRAY_SIZE(trace_clocks))
6742 mutex_lock(&trace_types_lock);
6746 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
6749 * New clock may not be consistent with the previous clock.
6750 * Reset the buffer so that it doesn't have incomparable timestamps.
6752 tracing_reset_online_cpus(&tr->array_buffer);
6754 #ifdef CONFIG_TRACER_MAX_TRACE
6755 if (tr->max_buffer.buffer)
6756 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6757 tracing_reset_online_cpus(&tr->max_buffer);
6760 mutex_unlock(&trace_types_lock);
6765 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6766 size_t cnt, loff_t *fpos)
6768 struct seq_file *m = filp->private_data;
6769 struct trace_array *tr = m->private;
6771 const char *clockstr;
6774 if (cnt >= sizeof(buf))
6777 if (copy_from_user(buf, ubuf, cnt))
6782 clockstr = strstrip(buf);
6784 ret = tracing_set_clock(tr, clockstr);
6793 static int tracing_clock_open(struct inode *inode, struct file *file)
6795 struct trace_array *tr = inode->i_private;
6798 ret = tracing_check_open_get_tr(tr);
6802 ret = single_open(file, tracing_clock_show, inode->i_private);
6804 trace_array_put(tr);
6809 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6811 struct trace_array *tr = m->private;
6813 mutex_lock(&trace_types_lock);
6815 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
6816 seq_puts(m, "delta [absolute]\n");
6818 seq_puts(m, "[delta] absolute\n");
6820 mutex_unlock(&trace_types_lock);
6825 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6827 struct trace_array *tr = inode->i_private;
6830 ret = tracing_check_open_get_tr(tr);
6834 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6836 trace_array_put(tr);
6841 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6845 mutex_lock(&trace_types_lock);
6847 if (abs && tr->time_stamp_abs_ref++)
6851 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6856 if (--tr->time_stamp_abs_ref)
6860 ring_buffer_set_time_stamp_abs(tr->array_buffer.buffer, abs);
6862 #ifdef CONFIG_TRACER_MAX_TRACE
6863 if (tr->max_buffer.buffer)
6864 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6867 mutex_unlock(&trace_types_lock);
6872 struct ftrace_buffer_info {
6873 struct trace_iterator iter;
6875 unsigned int spare_cpu;
6879 #ifdef CONFIG_TRACER_SNAPSHOT
6880 static int tracing_snapshot_open(struct inode *inode, struct file *file)
6882 struct trace_array *tr = inode->i_private;
6883 struct trace_iterator *iter;
6887 ret = tracing_check_open_get_tr(tr);
6891 if (file->f_mode & FMODE_READ) {
6892 iter = __tracing_open(inode, file, true);
6894 ret = PTR_ERR(iter);
6896 /* Writes still need the seq_file to hold the private data */
6898 m = kzalloc(sizeof(*m), GFP_KERNEL);
6901 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6909 iter->array_buffer = &tr->max_buffer;
6910 iter->cpu_file = tracing_get_cpu(inode);
6912 file->private_data = m;
6916 trace_array_put(tr);
6922 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6925 struct seq_file *m = filp->private_data;
6926 struct trace_iterator *iter = m->private;
6927 struct trace_array *tr = iter->tr;
6931 ret = tracing_update_buffers();
6935 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6939 mutex_lock(&trace_types_lock);
6941 if (tr->current_trace->use_max_tr) {
6946 arch_spin_lock(&tr->max_lock);
6947 if (tr->cond_snapshot)
6949 arch_spin_unlock(&tr->max_lock);
6955 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6959 if (tr->allocated_snapshot)
6963 /* Only allow per-cpu swap if the ring buffer supports it */
6964 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6965 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6970 if (tr->allocated_snapshot)
6971 ret = resize_buffer_duplicate_size(&tr->max_buffer,
6972 &tr->array_buffer, iter->cpu_file);
6974 ret = tracing_alloc_snapshot_instance(tr);
6977 local_irq_disable();
6978 /* Now, we're going to swap */
6979 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6980 update_max_tr(tr, current, smp_processor_id(), NULL);
6982 update_max_tr_single(tr, current, iter->cpu_file);
6986 if (tr->allocated_snapshot) {
6987 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6988 tracing_reset_online_cpus(&tr->max_buffer);
6990 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
7000 mutex_unlock(&trace_types_lock);
7004 static int tracing_snapshot_release(struct inode *inode, struct file *file)
7006 struct seq_file *m = file->private_data;
7009 ret = tracing_release(inode, file);
7011 if (file->f_mode & FMODE_READ)
7014 /* If write only, the seq_file is just a stub */
7022 static int tracing_buffers_open(struct inode *inode, struct file *filp);
7023 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7024 size_t count, loff_t *ppos);
7025 static int tracing_buffers_release(struct inode *inode, struct file *file);
7026 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7027 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7029 static int snapshot_raw_open(struct inode *inode, struct file *filp)
7031 struct ftrace_buffer_info *info;
7034 /* The following checks for tracefs lockdown */
7035 ret = tracing_buffers_open(inode, filp);
7039 info = filp->private_data;
7041 if (info->iter.trace->use_max_tr) {
7042 tracing_buffers_release(inode, filp);
7046 info->iter.snapshot = true;
7047 info->iter.array_buffer = &info->iter.tr->max_buffer;
7052 #endif /* CONFIG_TRACER_SNAPSHOT */
7055 static const struct file_operations tracing_thresh_fops = {
7056 .open = tracing_open_generic,
7057 .read = tracing_thresh_read,
7058 .write = tracing_thresh_write,
7059 .llseek = generic_file_llseek,
7062 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7063 static const struct file_operations tracing_max_lat_fops = {
7064 .open = tracing_open_generic,
7065 .read = tracing_max_lat_read,
7066 .write = tracing_max_lat_write,
7067 .llseek = generic_file_llseek,
7071 static const struct file_operations set_tracer_fops = {
7072 .open = tracing_open_generic,
7073 .read = tracing_set_trace_read,
7074 .write = tracing_set_trace_write,
7075 .llseek = generic_file_llseek,
7078 static const struct file_operations tracing_pipe_fops = {
7079 .open = tracing_open_pipe,
7080 .poll = tracing_poll_pipe,
7081 .read = tracing_read_pipe,
7082 .splice_read = tracing_splice_read_pipe,
7083 .release = tracing_release_pipe,
7084 .llseek = no_llseek,
7087 static const struct file_operations tracing_entries_fops = {
7088 .open = tracing_open_generic_tr,
7089 .read = tracing_entries_read,
7090 .write = tracing_entries_write,
7091 .llseek = generic_file_llseek,
7092 .release = tracing_release_generic_tr,
7095 static const struct file_operations tracing_total_entries_fops = {
7096 .open = tracing_open_generic_tr,
7097 .read = tracing_total_entries_read,
7098 .llseek = generic_file_llseek,
7099 .release = tracing_release_generic_tr,
7102 static const struct file_operations tracing_free_buffer_fops = {
7103 .open = tracing_open_generic_tr,
7104 .write = tracing_free_buffer_write,
7105 .release = tracing_free_buffer_release,
7108 static const struct file_operations tracing_mark_fops = {
7109 .open = tracing_open_generic_tr,
7110 .write = tracing_mark_write,
7111 .llseek = generic_file_llseek,
7112 .release = tracing_release_generic_tr,
7115 static const struct file_operations tracing_mark_raw_fops = {
7116 .open = tracing_open_generic_tr,
7117 .write = tracing_mark_raw_write,
7118 .llseek = generic_file_llseek,
7119 .release = tracing_release_generic_tr,
7122 static const struct file_operations trace_clock_fops = {
7123 .open = tracing_clock_open,
7125 .llseek = seq_lseek,
7126 .release = tracing_single_release_tr,
7127 .write = tracing_clock_write,
7130 static const struct file_operations trace_time_stamp_mode_fops = {
7131 .open = tracing_time_stamp_mode_open,
7133 .llseek = seq_lseek,
7134 .release = tracing_single_release_tr,
7137 #ifdef CONFIG_TRACER_SNAPSHOT
7138 static const struct file_operations snapshot_fops = {
7139 .open = tracing_snapshot_open,
7141 .write = tracing_snapshot_write,
7142 .llseek = tracing_lseek,
7143 .release = tracing_snapshot_release,
7146 static const struct file_operations snapshot_raw_fops = {
7147 .open = snapshot_raw_open,
7148 .read = tracing_buffers_read,
7149 .release = tracing_buffers_release,
7150 .splice_read = tracing_buffers_splice_read,
7151 .llseek = no_llseek,
7154 #endif /* CONFIG_TRACER_SNAPSHOT */
7156 #define TRACING_LOG_ERRS_MAX 8
7157 #define TRACING_LOG_LOC_MAX 128
7159 #define CMD_PREFIX " Command: "
7162 const char **errs; /* ptr to loc-specific array of err strings */
7163 u8 type; /* index into errs -> specific err string */
7164 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
7168 struct tracing_log_err {
7169 struct list_head list;
7170 struct err_info info;
7171 char loc[TRACING_LOG_LOC_MAX]; /* err location */
7172 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
7175 static DEFINE_MUTEX(tracing_err_log_lock);
7177 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
7179 struct tracing_log_err *err;
7181 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7182 err = kzalloc(sizeof(*err), GFP_KERNEL);
7184 err = ERR_PTR(-ENOMEM);
7185 tr->n_err_log_entries++;
7190 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7191 list_del(&err->list);
7197 * err_pos - find the position of a string within a command for error careting
7198 * @cmd: The tracing command that caused the error
7199 * @str: The string to position the caret at within @cmd
7201 * Finds the position of the first occurence of @str within @cmd. The
7202 * return value can be passed to tracing_log_err() for caret placement
7205 * Returns the index within @cmd of the first occurence of @str or 0
7206 * if @str was not found.
7208 unsigned int err_pos(char *cmd, const char *str)
7212 if (WARN_ON(!strlen(cmd)))
7215 found = strstr(cmd, str);
7223 * tracing_log_err - write an error to the tracing error log
7224 * @tr: The associated trace array for the error (NULL for top level array)
7225 * @loc: A string describing where the error occurred
7226 * @cmd: The tracing command that caused the error
7227 * @errs: The array of loc-specific static error strings
7228 * @type: The index into errs[], which produces the specific static err string
7229 * @pos: The position the caret should be placed in the cmd
7231 * Writes an error into tracing/error_log of the form:
7233 * <loc>: error: <text>
7237 * tracing/error_log is a small log file containing the last
7238 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7239 * unless there has been a tracing error, and the error log can be
7240 * cleared and have its memory freed by writing the empty string in
7241 * truncation mode to it i.e. echo > tracing/error_log.
7243 * NOTE: the @errs array along with the @type param are used to
7244 * produce a static error string - this string is not copied and saved
7245 * when the error is logged - only a pointer to it is saved. See
7246 * existing callers for examples of how static strings are typically
7247 * defined for use with tracing_log_err().
7249 void tracing_log_err(struct trace_array *tr,
7250 const char *loc, const char *cmd,
7251 const char **errs, u8 type, u8 pos)
7253 struct tracing_log_err *err;
7258 mutex_lock(&tracing_err_log_lock);
7259 err = get_tracing_log_err(tr);
7260 if (PTR_ERR(err) == -ENOMEM) {
7261 mutex_unlock(&tracing_err_log_lock);
7265 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7266 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7268 err->info.errs = errs;
7269 err->info.type = type;
7270 err->info.pos = pos;
7271 err->info.ts = local_clock();
7273 list_add_tail(&err->list, &tr->err_log);
7274 mutex_unlock(&tracing_err_log_lock);
7277 static void clear_tracing_err_log(struct trace_array *tr)
7279 struct tracing_log_err *err, *next;
7281 mutex_lock(&tracing_err_log_lock);
7282 list_for_each_entry_safe(err, next, &tr->err_log, list) {
7283 list_del(&err->list);
7287 tr->n_err_log_entries = 0;
7288 mutex_unlock(&tracing_err_log_lock);
7291 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7293 struct trace_array *tr = m->private;
7295 mutex_lock(&tracing_err_log_lock);
7297 return seq_list_start(&tr->err_log, *pos);
7300 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7302 struct trace_array *tr = m->private;
7304 return seq_list_next(v, &tr->err_log, pos);
7307 static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7309 mutex_unlock(&tracing_err_log_lock);
7312 static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7316 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7318 for (i = 0; i < pos; i++)
7323 static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7325 struct tracing_log_err *err = v;
7328 const char *err_text = err->info.errs[err->info.type];
7329 u64 sec = err->info.ts;
7332 nsec = do_div(sec, NSEC_PER_SEC);
7333 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7334 err->loc, err_text);
7335 seq_printf(m, "%s", err->cmd);
7336 tracing_err_log_show_pos(m, err->info.pos);
7342 static const struct seq_operations tracing_err_log_seq_ops = {
7343 .start = tracing_err_log_seq_start,
7344 .next = tracing_err_log_seq_next,
7345 .stop = tracing_err_log_seq_stop,
7346 .show = tracing_err_log_seq_show
7349 static int tracing_err_log_open(struct inode *inode, struct file *file)
7351 struct trace_array *tr = inode->i_private;
7354 ret = tracing_check_open_get_tr(tr);
7358 /* If this file was opened for write, then erase contents */
7359 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7360 clear_tracing_err_log(tr);
7362 if (file->f_mode & FMODE_READ) {
7363 ret = seq_open(file, &tracing_err_log_seq_ops);
7365 struct seq_file *m = file->private_data;
7368 trace_array_put(tr);
7374 static ssize_t tracing_err_log_write(struct file *file,
7375 const char __user *buffer,
7376 size_t count, loff_t *ppos)
7381 static int tracing_err_log_release(struct inode *inode, struct file *file)
7383 struct trace_array *tr = inode->i_private;
7385 trace_array_put(tr);
7387 if (file->f_mode & FMODE_READ)
7388 seq_release(inode, file);
7393 static const struct file_operations tracing_err_log_fops = {
7394 .open = tracing_err_log_open,
7395 .write = tracing_err_log_write,
7397 .llseek = seq_lseek,
7398 .release = tracing_err_log_release,
7401 static int tracing_buffers_open(struct inode *inode, struct file *filp)
7403 struct trace_array *tr = inode->i_private;
7404 struct ftrace_buffer_info *info;
7407 ret = tracing_check_open_get_tr(tr);
7411 info = kzalloc(sizeof(*info), GFP_KERNEL);
7413 trace_array_put(tr);
7417 mutex_lock(&trace_types_lock);
7420 info->iter.cpu_file = tracing_get_cpu(inode);
7421 info->iter.trace = tr->current_trace;
7422 info->iter.array_buffer = &tr->array_buffer;
7424 /* Force reading ring buffer for first read */
7425 info->read = (unsigned int)-1;
7427 filp->private_data = info;
7431 mutex_unlock(&trace_types_lock);
7433 ret = nonseekable_open(inode, filp);
7435 trace_array_put(tr);
7441 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7443 struct ftrace_buffer_info *info = filp->private_data;
7444 struct trace_iterator *iter = &info->iter;
7446 return trace_poll(iter, filp, poll_table);
7450 tracing_buffers_read(struct file *filp, char __user *ubuf,
7451 size_t count, loff_t *ppos)
7453 struct ftrace_buffer_info *info = filp->private_data;
7454 struct trace_iterator *iter = &info->iter;
7461 #ifdef CONFIG_TRACER_MAX_TRACE
7462 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7467 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
7469 if (IS_ERR(info->spare)) {
7470 ret = PTR_ERR(info->spare);
7473 info->spare_cpu = iter->cpu_file;
7479 /* Do we have previous read data to read? */
7480 if (info->read < PAGE_SIZE)
7484 trace_access_lock(iter->cpu_file);
7485 ret = ring_buffer_read_page(iter->array_buffer->buffer,
7489 trace_access_unlock(iter->cpu_file);
7492 if (trace_empty(iter)) {
7493 if ((filp->f_flags & O_NONBLOCK))
7496 ret = wait_on_pipe(iter, 0);
7507 size = PAGE_SIZE - info->read;
7511 ret = copy_to_user(ubuf, info->spare + info->read, size);
7523 static int tracing_buffers_release(struct inode *inode, struct file *file)
7525 struct ftrace_buffer_info *info = file->private_data;
7526 struct trace_iterator *iter = &info->iter;
7528 mutex_lock(&trace_types_lock);
7530 iter->tr->trace_ref--;
7532 __trace_array_put(iter->tr);
7535 ring_buffer_free_read_page(iter->array_buffer->buffer,
7536 info->spare_cpu, info->spare);
7539 mutex_unlock(&trace_types_lock);
7545 struct trace_buffer *buffer;
7548 refcount_t refcount;
7551 static void buffer_ref_release(struct buffer_ref *ref)
7553 if (!refcount_dec_and_test(&ref->refcount))
7555 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7559 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7560 struct pipe_buffer *buf)
7562 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7564 buffer_ref_release(ref);
7568 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7569 struct pipe_buffer *buf)
7571 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7573 if (refcount_read(&ref->refcount) > INT_MAX/2)
7576 refcount_inc(&ref->refcount);
7580 /* Pipe buffer operations for a buffer. */
7581 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7582 .release = buffer_pipe_buf_release,
7583 .get = buffer_pipe_buf_get,
7587 * Callback from splice_to_pipe(), if we need to release some pages
7588 * at the end of the spd in case we error'ed out in filling the pipe.
7590 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7592 struct buffer_ref *ref =
7593 (struct buffer_ref *)spd->partial[i].private;
7595 buffer_ref_release(ref);
7596 spd->partial[i].private = 0;
7600 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7601 struct pipe_inode_info *pipe, size_t len,
7604 struct ftrace_buffer_info *info = file->private_data;
7605 struct trace_iterator *iter = &info->iter;
7606 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7607 struct page *pages_def[PIPE_DEF_BUFFERS];
7608 struct splice_pipe_desc spd = {
7610 .partial = partial_def,
7611 .nr_pages_max = PIPE_DEF_BUFFERS,
7612 .ops = &buffer_pipe_buf_ops,
7613 .spd_release = buffer_spd_release,
7615 struct buffer_ref *ref;
7619 #ifdef CONFIG_TRACER_MAX_TRACE
7620 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7624 if (*ppos & (PAGE_SIZE - 1))
7627 if (len & (PAGE_SIZE - 1)) {
7628 if (len < PAGE_SIZE)
7633 if (splice_grow_spd(pipe, &spd))
7637 trace_access_lock(iter->cpu_file);
7638 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7640 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
7644 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
7650 refcount_set(&ref->refcount, 1);
7651 ref->buffer = iter->array_buffer->buffer;
7652 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
7653 if (IS_ERR(ref->page)) {
7654 ret = PTR_ERR(ref->page);
7659 ref->cpu = iter->cpu_file;
7661 r = ring_buffer_read_page(ref->buffer, &ref->page,
7662 len, iter->cpu_file, 1);
7664 ring_buffer_free_read_page(ref->buffer, ref->cpu,
7670 page = virt_to_page(ref->page);
7672 spd.pages[i] = page;
7673 spd.partial[i].len = PAGE_SIZE;
7674 spd.partial[i].offset = 0;
7675 spd.partial[i].private = (unsigned long)ref;
7679 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
7682 trace_access_unlock(iter->cpu_file);
7685 /* did we read anything? */
7686 if (!spd.nr_pages) {
7691 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7694 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
7701 ret = splice_to_pipe(pipe, &spd);
7703 splice_shrink_spd(&spd);
7708 static const struct file_operations tracing_buffers_fops = {
7709 .open = tracing_buffers_open,
7710 .read = tracing_buffers_read,
7711 .poll = tracing_buffers_poll,
7712 .release = tracing_buffers_release,
7713 .splice_read = tracing_buffers_splice_read,
7714 .llseek = no_llseek,
7718 tracing_stats_read(struct file *filp, char __user *ubuf,
7719 size_t count, loff_t *ppos)
7721 struct inode *inode = file_inode(filp);
7722 struct trace_array *tr = inode->i_private;
7723 struct array_buffer *trace_buf = &tr->array_buffer;
7724 int cpu = tracing_get_cpu(inode);
7725 struct trace_seq *s;
7727 unsigned long long t;
7728 unsigned long usec_rem;
7730 s = kmalloc(sizeof(*s), GFP_KERNEL);
7736 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7737 trace_seq_printf(s, "entries: %ld\n", cnt);
7739 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7740 trace_seq_printf(s, "overrun: %ld\n", cnt);
7742 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7743 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7745 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7746 trace_seq_printf(s, "bytes: %ld\n", cnt);
7748 if (trace_clocks[tr->clock_id].in_ns) {
7749 /* local or global for trace_clock */
7750 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7751 usec_rem = do_div(t, USEC_PER_SEC);
7752 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7755 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
7756 usec_rem = do_div(t, USEC_PER_SEC);
7757 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7759 /* counter or tsc mode for trace_clock */
7760 trace_seq_printf(s, "oldest event ts: %llu\n",
7761 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7763 trace_seq_printf(s, "now ts: %llu\n",
7764 ring_buffer_time_stamp(trace_buf->buffer, cpu));
7767 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7768 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7770 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7771 trace_seq_printf(s, "read events: %ld\n", cnt);
7773 count = simple_read_from_buffer(ubuf, count, ppos,
7774 s->buffer, trace_seq_used(s));
7781 static const struct file_operations tracing_stats_fops = {
7782 .open = tracing_open_generic_tr,
7783 .read = tracing_stats_read,
7784 .llseek = generic_file_llseek,
7785 .release = tracing_release_generic_tr,
7788 #ifdef CONFIG_DYNAMIC_FTRACE
7791 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7792 size_t cnt, loff_t *ppos)
7798 /* 256 should be plenty to hold the amount needed */
7799 buf = kmalloc(256, GFP_KERNEL);
7803 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
7804 ftrace_update_tot_cnt,
7805 ftrace_number_of_pages,
7806 ftrace_number_of_groups);
7808 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7813 static const struct file_operations tracing_dyn_info_fops = {
7814 .open = tracing_open_generic,
7815 .read = tracing_read_dyn_info,
7816 .llseek = generic_file_llseek,
7818 #endif /* CONFIG_DYNAMIC_FTRACE */
7820 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7822 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7823 struct trace_array *tr, struct ftrace_probe_ops *ops,
7826 tracing_snapshot_instance(tr);
7830 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7831 struct trace_array *tr, struct ftrace_probe_ops *ops,
7834 struct ftrace_func_mapper *mapper = data;
7838 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7848 tracing_snapshot_instance(tr);
7852 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7853 struct ftrace_probe_ops *ops, void *data)
7855 struct ftrace_func_mapper *mapper = data;
7858 seq_printf(m, "%ps:", (void *)ip);
7860 seq_puts(m, "snapshot");
7863 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7866 seq_printf(m, ":count=%ld\n", *count);
7868 seq_puts(m, ":unlimited\n");
7874 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7875 unsigned long ip, void *init_data, void **data)
7877 struct ftrace_func_mapper *mapper = *data;
7880 mapper = allocate_ftrace_func_mapper();
7886 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7890 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7891 unsigned long ip, void *data)
7893 struct ftrace_func_mapper *mapper = data;
7898 free_ftrace_func_mapper(mapper, NULL);
7902 ftrace_func_mapper_remove_ip(mapper, ip);
7905 static struct ftrace_probe_ops snapshot_probe_ops = {
7906 .func = ftrace_snapshot,
7907 .print = ftrace_snapshot_print,
7910 static struct ftrace_probe_ops snapshot_count_probe_ops = {
7911 .func = ftrace_count_snapshot,
7912 .print = ftrace_snapshot_print,
7913 .init = ftrace_snapshot_init,
7914 .free = ftrace_snapshot_free,
7918 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7919 char *glob, char *cmd, char *param, int enable)
7921 struct ftrace_probe_ops *ops;
7922 void *count = (void *)-1;
7929 /* hash funcs only work with set_ftrace_filter */
7933 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7936 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
7941 number = strsep(¶m, ":");
7943 if (!strlen(number))
7947 * We use the callback data field (which is a pointer)
7950 ret = kstrtoul(number, 0, (unsigned long *)&count);
7955 ret = tracing_alloc_snapshot_instance(tr);
7959 ret = register_ftrace_function_probe(glob, tr, ops, count);
7962 return ret < 0 ? ret : 0;
7965 static struct ftrace_func_command ftrace_snapshot_cmd = {
7967 .func = ftrace_trace_snapshot_callback,
7970 static __init int register_snapshot_cmd(void)
7972 return register_ftrace_command(&ftrace_snapshot_cmd);
7975 static inline __init int register_snapshot_cmd(void) { return 0; }
7976 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7978 static struct dentry *tracing_get_dentry(struct trace_array *tr)
7980 if (WARN_ON(!tr->dir))
7981 return ERR_PTR(-ENODEV);
7983 /* Top directory uses NULL as the parent */
7984 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7987 /* All sub buffers have a descriptor */
7991 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7993 struct dentry *d_tracer;
7996 return tr->percpu_dir;
7998 d_tracer = tracing_get_dentry(tr);
7999 if (IS_ERR(d_tracer))
8002 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
8004 MEM_FAIL(!tr->percpu_dir,
8005 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8007 return tr->percpu_dir;
8010 static struct dentry *
8011 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8012 void *data, long cpu, const struct file_operations *fops)
8014 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8016 if (ret) /* See tracing_get_cpu() */
8017 d_inode(ret)->i_cdev = (void *)(cpu + 1);
8022 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8024 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8025 struct dentry *d_cpu;
8026 char cpu_dir[30]; /* 30 characters should be more than enough */
8031 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8032 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8034 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8038 /* per cpu trace_pipe */
8039 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
8040 tr, cpu, &tracing_pipe_fops);
8043 trace_create_cpu_file("trace", 0644, d_cpu,
8044 tr, cpu, &tracing_fops);
8046 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
8047 tr, cpu, &tracing_buffers_fops);
8049 trace_create_cpu_file("stats", 0444, d_cpu,
8050 tr, cpu, &tracing_stats_fops);
8052 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
8053 tr, cpu, &tracing_entries_fops);
8055 #ifdef CONFIG_TRACER_SNAPSHOT
8056 trace_create_cpu_file("snapshot", 0644, d_cpu,
8057 tr, cpu, &snapshot_fops);
8059 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
8060 tr, cpu, &snapshot_raw_fops);
8064 #ifdef CONFIG_FTRACE_SELFTEST
8065 /* Let selftest have access to static functions in this file */
8066 #include "trace_selftest.c"
8070 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8073 struct trace_option_dentry *topt = filp->private_data;
8076 if (topt->flags->val & topt->opt->bit)
8081 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8085 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8088 struct trace_option_dentry *topt = filp->private_data;
8092 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8096 if (val != 0 && val != 1)
8099 if (!!(topt->flags->val & topt->opt->bit) != val) {
8100 mutex_lock(&trace_types_lock);
8101 ret = __set_tracer_option(topt->tr, topt->flags,
8103 mutex_unlock(&trace_types_lock);
8114 static const struct file_operations trace_options_fops = {
8115 .open = tracing_open_generic,
8116 .read = trace_options_read,
8117 .write = trace_options_write,
8118 .llseek = generic_file_llseek,
8122 * In order to pass in both the trace_array descriptor as well as the index
8123 * to the flag that the trace option file represents, the trace_array
8124 * has a character array of trace_flags_index[], which holds the index
8125 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8126 * The address of this character array is passed to the flag option file
8127 * read/write callbacks.
8129 * In order to extract both the index and the trace_array descriptor,
8130 * get_tr_index() uses the following algorithm.
8134 * As the pointer itself contains the address of the index (remember
8137 * Then to get the trace_array descriptor, by subtracting that index
8138 * from the ptr, we get to the start of the index itself.
8140 * ptr - idx == &index[0]
8142 * Then a simple container_of() from that pointer gets us to the
8143 * trace_array descriptor.
8145 static void get_tr_index(void *data, struct trace_array **ptr,
8146 unsigned int *pindex)
8148 *pindex = *(unsigned char *)data;
8150 *ptr = container_of(data - *pindex, struct trace_array,
8155 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8158 void *tr_index = filp->private_data;
8159 struct trace_array *tr;
8163 get_tr_index(tr_index, &tr, &index);
8165 if (tr->trace_flags & (1 << index))
8170 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
8174 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8177 void *tr_index = filp->private_data;
8178 struct trace_array *tr;
8183 get_tr_index(tr_index, &tr, &index);
8185 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8189 if (val != 0 && val != 1)
8192 mutex_lock(&event_mutex);
8193 mutex_lock(&trace_types_lock);
8194 ret = set_tracer_flag(tr, 1 << index, val);
8195 mutex_unlock(&trace_types_lock);
8196 mutex_unlock(&event_mutex);
8206 static const struct file_operations trace_options_core_fops = {
8207 .open = tracing_open_generic,
8208 .read = trace_options_core_read,
8209 .write = trace_options_core_write,
8210 .llseek = generic_file_llseek,
8213 struct dentry *trace_create_file(const char *name,
8215 struct dentry *parent,
8217 const struct file_operations *fops)
8221 ret = tracefs_create_file(name, mode, parent, data, fops);
8223 pr_warn("Could not create tracefs '%s' entry\n", name);
8229 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8231 struct dentry *d_tracer;
8236 d_tracer = tracing_get_dentry(tr);
8237 if (IS_ERR(d_tracer))
8240 tr->options = tracefs_create_dir("options", d_tracer);
8242 pr_warn("Could not create tracefs directory 'options'\n");
8250 create_trace_option_file(struct trace_array *tr,
8251 struct trace_option_dentry *topt,
8252 struct tracer_flags *flags,
8253 struct tracer_opt *opt)
8255 struct dentry *t_options;
8257 t_options = trace_options_init_dentry(tr);
8261 topt->flags = flags;
8265 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
8266 &trace_options_fops);
8271 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8273 struct trace_option_dentry *topts;
8274 struct trace_options *tr_topts;
8275 struct tracer_flags *flags;
8276 struct tracer_opt *opts;
8283 flags = tracer->flags;
8285 if (!flags || !flags->opts)
8289 * If this is an instance, only create flags for tracers
8290 * the instance may have.
8292 if (!trace_ok_for_array(tracer, tr))
8295 for (i = 0; i < tr->nr_topts; i++) {
8296 /* Make sure there's no duplicate flags. */
8297 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8303 for (cnt = 0; opts[cnt].name; cnt++)
8306 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
8310 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8317 tr->topts = tr_topts;
8318 tr->topts[tr->nr_topts].tracer = tracer;
8319 tr->topts[tr->nr_topts].topts = topts;
8322 for (cnt = 0; opts[cnt].name; cnt++) {
8323 create_trace_option_file(tr, &topts[cnt], flags,
8325 MEM_FAIL(topts[cnt].entry == NULL,
8326 "Failed to create trace option: %s",
8331 static struct dentry *
8332 create_trace_option_core_file(struct trace_array *tr,
8333 const char *option, long index)
8335 struct dentry *t_options;
8337 t_options = trace_options_init_dentry(tr);
8341 return trace_create_file(option, 0644, t_options,
8342 (void *)&tr->trace_flags_index[index],
8343 &trace_options_core_fops);
8346 static void create_trace_options_dir(struct trace_array *tr)
8348 struct dentry *t_options;
8349 bool top_level = tr == &global_trace;
8352 t_options = trace_options_init_dentry(tr);
8356 for (i = 0; trace_options[i]; i++) {
8358 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8359 create_trace_option_core_file(tr, trace_options[i], i);
8364 rb_simple_read(struct file *filp, char __user *ubuf,
8365 size_t cnt, loff_t *ppos)
8367 struct trace_array *tr = filp->private_data;
8371 r = tracer_tracing_is_on(tr);
8372 r = sprintf(buf, "%d\n", r);
8374 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8378 rb_simple_write(struct file *filp, const char __user *ubuf,
8379 size_t cnt, loff_t *ppos)
8381 struct trace_array *tr = filp->private_data;
8382 struct trace_buffer *buffer = tr->array_buffer.buffer;
8386 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8391 mutex_lock(&trace_types_lock);
8392 if (!!val == tracer_tracing_is_on(tr)) {
8393 val = 0; /* do nothing */
8395 tracer_tracing_on(tr);
8396 if (tr->current_trace->start)
8397 tr->current_trace->start(tr);
8399 tracer_tracing_off(tr);
8400 if (tr->current_trace->stop)
8401 tr->current_trace->stop(tr);
8403 mutex_unlock(&trace_types_lock);
8411 static const struct file_operations rb_simple_fops = {
8412 .open = tracing_open_generic_tr,
8413 .read = rb_simple_read,
8414 .write = rb_simple_write,
8415 .release = tracing_release_generic_tr,
8416 .llseek = default_llseek,
8420 buffer_percent_read(struct file *filp, char __user *ubuf,
8421 size_t cnt, loff_t *ppos)
8423 struct trace_array *tr = filp->private_data;
8427 r = tr->buffer_percent;
8428 r = sprintf(buf, "%d\n", r);
8430 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8434 buffer_percent_write(struct file *filp, const char __user *ubuf,
8435 size_t cnt, loff_t *ppos)
8437 struct trace_array *tr = filp->private_data;
8441 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8451 tr->buffer_percent = val;
8458 static const struct file_operations buffer_percent_fops = {
8459 .open = tracing_open_generic_tr,
8460 .read = buffer_percent_read,
8461 .write = buffer_percent_write,
8462 .release = tracing_release_generic_tr,
8463 .llseek = default_llseek,
8466 static struct dentry *trace_instance_dir;
8469 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
8472 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
8474 enum ring_buffer_flags rb_flags;
8476 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
8480 buf->buffer = ring_buffer_alloc(size, rb_flags);
8484 buf->data = alloc_percpu(struct trace_array_cpu);
8486 ring_buffer_free(buf->buffer);
8491 /* Allocate the first page for all buffers */
8492 set_buffer_entries(&tr->array_buffer,
8493 ring_buffer_size(tr->array_buffer.buffer, 0));
8498 static int allocate_trace_buffers(struct trace_array *tr, int size)
8502 ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
8506 #ifdef CONFIG_TRACER_MAX_TRACE
8507 ret = allocate_trace_buffer(tr, &tr->max_buffer,
8508 allocate_snapshot ? size : 1);
8509 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
8510 ring_buffer_free(tr->array_buffer.buffer);
8511 tr->array_buffer.buffer = NULL;
8512 free_percpu(tr->array_buffer.data);
8513 tr->array_buffer.data = NULL;
8516 tr->allocated_snapshot = allocate_snapshot;
8519 * Only the top level trace array gets its snapshot allocated
8520 * from the kernel command line.
8522 allocate_snapshot = false;
8528 static void free_trace_buffer(struct array_buffer *buf)
8531 ring_buffer_free(buf->buffer);
8533 free_percpu(buf->data);
8538 static void free_trace_buffers(struct trace_array *tr)
8543 free_trace_buffer(&tr->array_buffer);
8545 #ifdef CONFIG_TRACER_MAX_TRACE
8546 free_trace_buffer(&tr->max_buffer);
8550 static void init_trace_flags_index(struct trace_array *tr)
8554 /* Used by the trace options files */
8555 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8556 tr->trace_flags_index[i] = i;
8559 static void __update_tracer_options(struct trace_array *tr)
8563 for (t = trace_types; t; t = t->next)
8564 add_tracer_options(tr, t);
8567 static void update_tracer_options(struct trace_array *tr)
8569 mutex_lock(&trace_types_lock);
8570 __update_tracer_options(tr);
8571 mutex_unlock(&trace_types_lock);
8574 /* Must have trace_types_lock held */
8575 struct trace_array *trace_array_find(const char *instance)
8577 struct trace_array *tr, *found = NULL;
8579 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8580 if (tr->name && strcmp(tr->name, instance) == 0) {
8589 struct trace_array *trace_array_find_get(const char *instance)
8591 struct trace_array *tr;
8593 mutex_lock(&trace_types_lock);
8594 tr = trace_array_find(instance);
8597 mutex_unlock(&trace_types_lock);
8602 static struct trace_array *trace_array_create(const char *name)
8604 struct trace_array *tr;
8608 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8610 return ERR_PTR(ret);
8612 tr->name = kstrdup(name, GFP_KERNEL);
8616 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8619 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8621 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8623 raw_spin_lock_init(&tr->start_lock);
8625 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8627 tr->current_trace = &nop_trace;
8629 INIT_LIST_HEAD(&tr->systems);
8630 INIT_LIST_HEAD(&tr->events);
8631 INIT_LIST_HEAD(&tr->hist_vars);
8632 INIT_LIST_HEAD(&tr->err_log);
8634 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8637 tr->dir = tracefs_create_dir(name, trace_instance_dir);
8641 ret = event_trace_add_tracer(tr->dir, tr);
8643 tracefs_remove(tr->dir);
8647 ftrace_init_trace_array(tr);
8649 init_tracer_tracefs(tr, tr->dir);
8650 init_trace_flags_index(tr);
8651 __update_tracer_options(tr);
8653 list_add(&tr->list, &ftrace_trace_arrays);
8661 free_trace_buffers(tr);
8662 free_cpumask_var(tr->tracing_cpumask);
8666 return ERR_PTR(ret);
8669 static int instance_mkdir(const char *name)
8671 struct trace_array *tr;
8674 mutex_lock(&event_mutex);
8675 mutex_lock(&trace_types_lock);
8678 if (trace_array_find(name))
8681 tr = trace_array_create(name);
8683 ret = PTR_ERR_OR_ZERO(tr);
8686 mutex_unlock(&trace_types_lock);
8687 mutex_unlock(&event_mutex);
8692 * trace_array_get_by_name - Create/Lookup a trace array, given its name.
8693 * @name: The name of the trace array to be looked up/created.
8695 * Returns pointer to trace array with given name.
8696 * NULL, if it cannot be created.
8698 * NOTE: This function increments the reference counter associated with the
8699 * trace array returned. This makes sure it cannot be freed while in use.
8700 * Use trace_array_put() once the trace array is no longer needed.
8701 * If the trace_array is to be freed, trace_array_destroy() needs to
8702 * be called after the trace_array_put(), or simply let user space delete
8703 * it from the tracefs instances directory. But until the
8704 * trace_array_put() is called, user space can not delete it.
8707 struct trace_array *trace_array_get_by_name(const char *name)
8709 struct trace_array *tr;
8711 mutex_lock(&event_mutex);
8712 mutex_lock(&trace_types_lock);
8714 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8715 if (tr->name && strcmp(tr->name, name) == 0)
8719 tr = trace_array_create(name);
8727 mutex_unlock(&trace_types_lock);
8728 mutex_unlock(&event_mutex);
8731 EXPORT_SYMBOL_GPL(trace_array_get_by_name);
8733 static int __remove_instance(struct trace_array *tr)
8737 /* Reference counter for a newly created trace array = 1. */
8738 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
8741 list_del(&tr->list);
8743 /* Disable all the flags that were enabled coming in */
8744 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8745 if ((1 << i) & ZEROED_TRACE_FLAGS)
8746 set_tracer_flag(tr, 1 << i, 0);
8749 tracing_set_nop(tr);
8750 clear_ftrace_function_probes(tr);
8751 event_trace_del_tracer(tr);
8752 ftrace_clear_pids(tr);
8753 ftrace_destroy_function_files(tr);
8754 tracefs_remove(tr->dir);
8755 free_trace_buffers(tr);
8757 for (i = 0; i < tr->nr_topts; i++) {
8758 kfree(tr->topts[i].topts);
8762 free_cpumask_var(tr->tracing_cpumask);
8770 int trace_array_destroy(struct trace_array *this_tr)
8772 struct trace_array *tr;
8778 mutex_lock(&event_mutex);
8779 mutex_lock(&trace_types_lock);
8783 /* Making sure trace array exists before destroying it. */
8784 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8785 if (tr == this_tr) {
8786 ret = __remove_instance(tr);
8791 mutex_unlock(&trace_types_lock);
8792 mutex_unlock(&event_mutex);
8796 EXPORT_SYMBOL_GPL(trace_array_destroy);
8798 static int instance_rmdir(const char *name)
8800 struct trace_array *tr;
8803 mutex_lock(&event_mutex);
8804 mutex_lock(&trace_types_lock);
8807 tr = trace_array_find(name);
8809 ret = __remove_instance(tr);
8811 mutex_unlock(&trace_types_lock);
8812 mutex_unlock(&event_mutex);
8817 static __init void create_trace_instances(struct dentry *d_tracer)
8819 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8822 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
8827 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8829 struct trace_event_file *file;
8832 trace_create_file("available_tracers", 0444, d_tracer,
8833 tr, &show_traces_fops);
8835 trace_create_file("current_tracer", 0644, d_tracer,
8836 tr, &set_tracer_fops);
8838 trace_create_file("tracing_cpumask", 0644, d_tracer,
8839 tr, &tracing_cpumask_fops);
8841 trace_create_file("trace_options", 0644, d_tracer,
8842 tr, &tracing_iter_fops);
8844 trace_create_file("trace", 0644, d_tracer,
8847 trace_create_file("trace_pipe", 0444, d_tracer,
8848 tr, &tracing_pipe_fops);
8850 trace_create_file("buffer_size_kb", 0644, d_tracer,
8851 tr, &tracing_entries_fops);
8853 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
8854 tr, &tracing_total_entries_fops);
8856 trace_create_file("free_buffer", 0200, d_tracer,
8857 tr, &tracing_free_buffer_fops);
8859 trace_create_file("trace_marker", 0220, d_tracer,
8860 tr, &tracing_mark_fops);
8862 file = __find_event_file(tr, "ftrace", "print");
8863 if (file && file->dir)
8864 trace_create_file("trigger", 0644, file->dir, file,
8865 &event_trigger_fops);
8866 tr->trace_marker_file = file;
8868 trace_create_file("trace_marker_raw", 0220, d_tracer,
8869 tr, &tracing_mark_raw_fops);
8871 trace_create_file("trace_clock", 0644, d_tracer, tr,
8874 trace_create_file("tracing_on", 0644, d_tracer,
8875 tr, &rb_simple_fops);
8877 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8878 &trace_time_stamp_mode_fops);
8880 tr->buffer_percent = 50;
8882 trace_create_file("buffer_percent", 0444, d_tracer,
8883 tr, &buffer_percent_fops);
8885 create_trace_options_dir(tr);
8887 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
8888 trace_create_maxlat_file(tr, d_tracer);
8891 if (ftrace_create_function_files(tr, d_tracer))
8892 MEM_FAIL(1, "Could not allocate function filter files");
8894 #ifdef CONFIG_TRACER_SNAPSHOT
8895 trace_create_file("snapshot", 0644, d_tracer,
8896 tr, &snapshot_fops);
8899 trace_create_file("error_log", 0644, d_tracer,
8900 tr, &tracing_err_log_fops);
8902 for_each_tracing_cpu(cpu)
8903 tracing_init_tracefs_percpu(tr, cpu);
8905 ftrace_init_tracefs(tr, d_tracer);
8908 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
8910 struct vfsmount *mnt;
8911 struct file_system_type *type;
8914 * To maintain backward compatibility for tools that mount
8915 * debugfs to get to the tracing facility, tracefs is automatically
8916 * mounted to the debugfs/tracing directory.
8918 type = get_fs_type("tracefs");
8921 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
8922 put_filesystem(type);
8931 * tracing_init_dentry - initialize top level trace array
8933 * This is called when creating files or directories in the tracing
8934 * directory. It is called via fs_initcall() by any of the boot up code
8935 * and expects to return the dentry of the top level tracing directory.
8937 struct dentry *tracing_init_dentry(void)
8939 struct trace_array *tr = &global_trace;
8941 if (security_locked_down(LOCKDOWN_TRACEFS)) {
8942 pr_warn("Tracing disabled due to lockdown\n");
8943 return ERR_PTR(-EPERM);
8946 /* The top level trace array uses NULL as parent */
8950 if (WARN_ON(!tracefs_initialized()) ||
8951 (IS_ENABLED(CONFIG_DEBUG_FS) &&
8952 WARN_ON(!debugfs_initialized())))
8953 return ERR_PTR(-ENODEV);
8956 * As there may still be users that expect the tracing
8957 * files to exist in debugfs/tracing, we must automount
8958 * the tracefs file system there, so older tools still
8959 * work with the newer kerenl.
8961 tr->dir = debugfs_create_automount("tracing", NULL,
8962 trace_automount, NULL);
8967 extern struct trace_eval_map *__start_ftrace_eval_maps[];
8968 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8970 static void __init trace_eval_init(void)
8974 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8975 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8978 #ifdef CONFIG_MODULES
8979 static void trace_module_add_evals(struct module *mod)
8981 if (!mod->num_trace_evals)
8985 * Modules with bad taint do not have events created, do
8986 * not bother with enums either.
8988 if (trace_module_has_bad_taint(mod))
8991 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
8994 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
8995 static void trace_module_remove_evals(struct module *mod)
8997 union trace_eval_map_item *map;
8998 union trace_eval_map_item **last = &trace_eval_maps;
9000 if (!mod->num_trace_evals)
9003 mutex_lock(&trace_eval_mutex);
9005 map = trace_eval_maps;
9008 if (map->head.mod == mod)
9010 map = trace_eval_jmp_to_tail(map);
9011 last = &map->tail.next;
9012 map = map->tail.next;
9017 *last = trace_eval_jmp_to_tail(map)->tail.next;
9020 mutex_unlock(&trace_eval_mutex);
9023 static inline void trace_module_remove_evals(struct module *mod) { }
9024 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9026 static int trace_module_notify(struct notifier_block *self,
9027 unsigned long val, void *data)
9029 struct module *mod = data;
9032 case MODULE_STATE_COMING:
9033 trace_module_add_evals(mod);
9035 case MODULE_STATE_GOING:
9036 trace_module_remove_evals(mod);
9043 static struct notifier_block trace_module_nb = {
9044 .notifier_call = trace_module_notify,
9047 #endif /* CONFIG_MODULES */
9049 static __init int tracer_init_tracefs(void)
9051 struct dentry *d_tracer;
9053 trace_access_lock_init();
9055 d_tracer = tracing_init_dentry();
9056 if (IS_ERR(d_tracer))
9061 init_tracer_tracefs(&global_trace, d_tracer);
9062 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
9064 trace_create_file("tracing_thresh", 0644, d_tracer,
9065 &global_trace, &tracing_thresh_fops);
9067 trace_create_file("README", 0444, d_tracer,
9068 NULL, &tracing_readme_fops);
9070 trace_create_file("saved_cmdlines", 0444, d_tracer,
9071 NULL, &tracing_saved_cmdlines_fops);
9073 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
9074 NULL, &tracing_saved_cmdlines_size_fops);
9076 trace_create_file("saved_tgids", 0444, d_tracer,
9077 NULL, &tracing_saved_tgids_fops);
9081 trace_create_eval_file(d_tracer);
9083 #ifdef CONFIG_MODULES
9084 register_module_notifier(&trace_module_nb);
9087 #ifdef CONFIG_DYNAMIC_FTRACE
9088 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
9089 NULL, &tracing_dyn_info_fops);
9092 create_trace_instances(d_tracer);
9094 update_tracer_options(&global_trace);
9099 static int trace_panic_handler(struct notifier_block *this,
9100 unsigned long event, void *unused)
9102 if (ftrace_dump_on_oops)
9103 ftrace_dump(ftrace_dump_on_oops);
9107 static struct notifier_block trace_panic_notifier = {
9108 .notifier_call = trace_panic_handler,
9110 .priority = 150 /* priority: INT_MAX >= x >= 0 */
9113 static int trace_die_handler(struct notifier_block *self,
9119 if (ftrace_dump_on_oops)
9120 ftrace_dump(ftrace_dump_on_oops);
9128 static struct notifier_block trace_die_notifier = {
9129 .notifier_call = trace_die_handler,
9134 * printk is set to max of 1024, we really don't need it that big.
9135 * Nothing should be printing 1000 characters anyway.
9137 #define TRACE_MAX_PRINT 1000
9140 * Define here KERN_TRACE so that we have one place to modify
9141 * it if we decide to change what log level the ftrace dump
9144 #define KERN_TRACE KERN_EMERG
9147 trace_printk_seq(struct trace_seq *s)
9149 /* Probably should print a warning here. */
9150 if (s->seq.len >= TRACE_MAX_PRINT)
9151 s->seq.len = TRACE_MAX_PRINT;
9154 * More paranoid code. Although the buffer size is set to
9155 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9156 * an extra layer of protection.
9158 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9159 s->seq.len = s->seq.size - 1;
9161 /* should be zero ended, but we are paranoid. */
9162 s->buffer[s->seq.len] = 0;
9164 printk(KERN_TRACE "%s", s->buffer);
9169 void trace_init_global_iter(struct trace_iterator *iter)
9171 iter->tr = &global_trace;
9172 iter->trace = iter->tr->current_trace;
9173 iter->cpu_file = RING_BUFFER_ALL_CPUS;
9174 iter->array_buffer = &global_trace.array_buffer;
9176 if (iter->trace && iter->trace->open)
9177 iter->trace->open(iter);
9179 /* Annotate start of buffers if we had overruns */
9180 if (ring_buffer_overruns(iter->array_buffer->buffer))
9181 iter->iter_flags |= TRACE_FILE_ANNOTATE;
9183 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
9184 if (trace_clocks[iter->tr->clock_id].in_ns)
9185 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9188 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
9190 /* use static because iter can be a bit big for the stack */
9191 static struct trace_iterator iter;
9192 static atomic_t dump_running;
9193 struct trace_array *tr = &global_trace;
9194 unsigned int old_userobj;
9195 unsigned long flags;
9198 /* Only allow one dump user at a time. */
9199 if (atomic_inc_return(&dump_running) != 1) {
9200 atomic_dec(&dump_running);
9205 * Always turn off tracing when we dump.
9206 * We don't need to show trace output of what happens
9207 * between multiple crashes.
9209 * If the user does a sysrq-z, then they can re-enable
9210 * tracing with echo 1 > tracing_on.
9214 local_irq_save(flags);
9215 printk_nmi_direct_enter();
9217 /* Simulate the iterator */
9218 trace_init_global_iter(&iter);
9219 /* Can not use kmalloc for iter.temp */
9220 iter.temp = static_temp_buf;
9221 iter.temp_size = STATIC_TEMP_BUF_SIZE;
9223 for_each_tracing_cpu(cpu) {
9224 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9227 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9229 /* don't look at user memory in panic mode */
9230 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9232 switch (oops_dump_mode) {
9234 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9237 iter.cpu_file = raw_smp_processor_id();
9242 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
9243 iter.cpu_file = RING_BUFFER_ALL_CPUS;
9246 printk(KERN_TRACE "Dumping ftrace buffer:\n");
9248 /* Did function tracer already get disabled? */
9249 if (ftrace_is_dead()) {
9250 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9251 printk("# MAY BE MISSING FUNCTION EVENTS\n");
9255 * We need to stop all tracing on all CPUS to read the
9256 * the next buffer. This is a bit expensive, but is
9257 * not done often. We fill all what we can read,
9258 * and then release the locks again.
9261 while (!trace_empty(&iter)) {
9264 printk(KERN_TRACE "---------------------------------\n");
9268 trace_iterator_reset(&iter);
9269 iter.iter_flags |= TRACE_FILE_LAT_FMT;
9271 if (trace_find_next_entry_inc(&iter) != NULL) {
9274 ret = print_trace_line(&iter);
9275 if (ret != TRACE_TYPE_NO_CONSUME)
9276 trace_consume(&iter);
9278 touch_nmi_watchdog();
9280 trace_printk_seq(&iter.seq);
9284 printk(KERN_TRACE " (ftrace buffer empty)\n");
9286 printk(KERN_TRACE "---------------------------------\n");
9289 tr->trace_flags |= old_userobj;
9291 for_each_tracing_cpu(cpu) {
9292 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9294 atomic_dec(&dump_running);
9295 printk_nmi_direct_exit();
9296 local_irq_restore(flags);
9298 EXPORT_SYMBOL_GPL(ftrace_dump);
9300 int trace_run_command(const char *buf, int (*createfn)(int, char **))
9307 argv = argv_split(GFP_KERNEL, buf, &argc);
9312 ret = createfn(argc, argv);
9319 #define WRITE_BUFSIZE 4096
9321 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
9322 size_t count, loff_t *ppos,
9323 int (*createfn)(int, char **))
9325 char *kbuf, *buf, *tmp;
9330 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
9334 while (done < count) {
9335 size = count - done;
9337 if (size >= WRITE_BUFSIZE)
9338 size = WRITE_BUFSIZE - 1;
9340 if (copy_from_user(kbuf, buffer + done, size)) {
9347 tmp = strchr(buf, '\n');
9350 size = tmp - buf + 1;
9353 if (done + size < count) {
9356 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9357 pr_warn("Line length is too long: Should be less than %d\n",
9365 /* Remove comments */
9366 tmp = strchr(buf, '#');
9371 ret = trace_run_command(buf, createfn);
9376 } while (done < count);
9386 __init static int tracer_alloc_buffers(void)
9392 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9393 pr_warn("Tracing disabled due to lockdown\n");
9398 * Make sure we don't accidently add more trace options
9399 * than we have bits for.
9401 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
9403 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9406 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9407 goto out_free_buffer_mask;
9409 /* Only allocate trace_printk buffers if a trace_printk exists */
9410 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
9411 /* Must be called before global_trace.buffer is allocated */
9412 trace_printk_init_buffers();
9414 /* To save memory, keep the ring buffer size to its minimum */
9415 if (ring_buffer_expanded)
9416 ring_buf_size = trace_buf_size;
9420 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
9421 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9423 raw_spin_lock_init(&global_trace.start_lock);
9426 * The prepare callbacks allocates some memory for the ring buffer. We
9427 * don't free the buffer if the if the CPU goes down. If we were to free
9428 * the buffer, then the user would lose any trace that was in the
9429 * buffer. The memory will be removed once the "instance" is removed.
9431 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9432 "trace/RB:preapre", trace_rb_cpu_prepare,
9435 goto out_free_cpumask;
9436 /* Used for event triggers */
9438 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9440 goto out_rm_hp_state;
9442 if (trace_create_savedcmd() < 0)
9443 goto out_free_temp_buffer;
9445 /* TODO: make the number of buffers hot pluggable with CPUS */
9446 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
9447 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
9448 goto out_free_savedcmd;
9451 if (global_trace.buffer_disabled)
9454 if (trace_boot_clock) {
9455 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9457 pr_warn("Trace clock %s not defined, going back to default\n",
9462 * register_tracer() might reference current_trace, so it
9463 * needs to be set before we register anything. This is
9464 * just a bootstrap of current_trace anyway.
9466 global_trace.current_trace = &nop_trace;
9468 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9470 ftrace_init_global_array_ops(&global_trace);
9472 init_trace_flags_index(&global_trace);
9474 register_tracer(&nop_trace);
9476 /* Function tracing may start here (via kernel command line) */
9477 init_function_trace();
9479 /* All seems OK, enable tracing */
9480 tracing_disabled = 0;
9482 atomic_notifier_chain_register(&panic_notifier_list,
9483 &trace_panic_notifier);
9485 register_die_notifier(&trace_die_notifier);
9487 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9489 INIT_LIST_HEAD(&global_trace.systems);
9490 INIT_LIST_HEAD(&global_trace.events);
9491 INIT_LIST_HEAD(&global_trace.hist_vars);
9492 INIT_LIST_HEAD(&global_trace.err_log);
9493 list_add(&global_trace.list, &ftrace_trace_arrays);
9495 apply_trace_boot_options();
9497 register_snapshot_cmd();
9502 free_saved_cmdlines_buffer(savedcmd);
9503 out_free_temp_buffer:
9504 ring_buffer_free(temp_buffer);
9506 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9508 free_cpumask_var(global_trace.tracing_cpumask);
9509 out_free_buffer_mask:
9510 free_cpumask_var(tracing_buffer_mask);
9515 void __init early_trace_init(void)
9517 if (tracepoint_printk) {
9518 tracepoint_print_iter =
9519 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
9520 if (MEM_FAIL(!tracepoint_print_iter,
9521 "Failed to allocate trace iterator\n"))
9522 tracepoint_printk = 0;
9524 static_key_enable(&tracepoint_printk_key.key);
9526 tracer_alloc_buffers();
9529 void __init trace_init(void)
9534 __init static int clear_boot_tracer(void)
9537 * The default tracer at boot buffer is an init section.
9538 * This function is called in lateinit. If we did not
9539 * find the boot tracer, then clear it out, to prevent
9540 * later registration from accessing the buffer that is
9541 * about to be freed.
9543 if (!default_bootup_tracer)
9546 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
9547 default_bootup_tracer);
9548 default_bootup_tracer = NULL;
9553 fs_initcall(tracer_init_tracefs);
9554 late_initcall_sync(clear_boot_tracer);
9556 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9557 __init static int tracing_set_default_clock(void)
9559 /* sched_clock_stable() is determined in late_initcall */
9560 if (!trace_boot_clock && !sched_clock_stable()) {
9561 if (security_locked_down(LOCKDOWN_TRACEFS)) {
9562 pr_warn("Can not set tracing clock due to lockdown\n");
9567 "Unstable clock detected, switching default tracing clock to \"global\"\n"
9568 "If you want to keep using the local clock, then add:\n"
9569 " \"trace_clock=local\"\n"
9570 "on the kernel command line\n");
9571 tracing_set_clock(&global_trace, "global");
9576 late_initcall_sync(tracing_set_default_clock);