2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled = 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
99 cpumask_var_t __read_mostly tracing_buffer_mask;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning;
122 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
128 static bool allocate_snapshot;
130 static int __init set_cmdline_ftrace(char *str)
132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133 default_bootup_tracer = bootup_tracer_buf;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = true;
138 __setup("ftrace=", set_cmdline_ftrace);
140 static int __init set_ftrace_dump_on_oops(char *str)
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
156 static int __init stop_trace_on_warning(char *str)
158 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
159 __disable_trace_on_warning = 1;
162 __setup("traceoff_on_warning", stop_trace_on_warning);
164 static int __init boot_alloc_snapshot(char *str)
166 allocate_snapshot = true;
167 /* We also need the main ring buffer expanded */
168 ring_buffer_expanded = true;
171 __setup("alloc_snapshot", boot_alloc_snapshot);
174 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
175 static char *trace_boot_options __initdata;
177 static int __init set_trace_boot_options(char *str)
179 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
180 trace_boot_options = trace_boot_options_buf;
183 __setup("trace_options=", set_trace_boot_options);
185 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
186 static char *trace_boot_clock __initdata;
188 static int __init set_trace_boot_clock(char *str)
190 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
191 trace_boot_clock = trace_boot_clock_buf;
194 __setup("trace_clock=", set_trace_boot_clock);
197 unsigned long long ns2usecs(cycle_t nsec)
205 * The global_trace is the descriptor that holds the tracing
206 * buffers for the live tracing. For each CPU, it contains
207 * a link list of pages that will store trace entries. The
208 * page descriptor of the pages in the memory is used to hold
209 * the link list by linking the lru item in the page descriptor
210 * to each of the pages in the buffer per CPU.
212 * For each active CPU there is a data field that holds the
213 * pages for the buffer for that CPU. Each CPU has the same number
214 * of pages allocated for its buffer.
216 static struct trace_array global_trace;
218 LIST_HEAD(ftrace_trace_arrays);
220 int trace_array_get(struct trace_array *this_tr)
222 struct trace_array *tr;
225 mutex_lock(&trace_types_lock);
226 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
233 mutex_unlock(&trace_types_lock);
238 static void __trace_array_put(struct trace_array *this_tr)
240 WARN_ON(!this_tr->ref);
244 void trace_array_put(struct trace_array *this_tr)
246 mutex_lock(&trace_types_lock);
247 __trace_array_put(this_tr);
248 mutex_unlock(&trace_types_lock);
251 int filter_check_discard(struct ftrace_event_file *file, void *rec,
252 struct ring_buffer *buffer,
253 struct ring_buffer_event *event)
255 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
256 !filter_match_preds(file->filter, rec)) {
257 ring_buffer_discard_commit(buffer, event);
263 EXPORT_SYMBOL_GPL(filter_check_discard);
265 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
266 struct ring_buffer *buffer,
267 struct ring_buffer_event *event)
269 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
270 !filter_match_preds(call->filter, rec)) {
271 ring_buffer_discard_commit(buffer, event);
277 EXPORT_SYMBOL_GPL(call_filter_check_discard);
279 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
283 /* Early boot up does not have a buffer yet */
285 return trace_clock_local();
287 ts = ring_buffer_time_stamp(buf->buffer, cpu);
288 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
293 cycle_t ftrace_now(int cpu)
295 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
299 * tracing_is_enabled - Show if global_trace has been disabled
301 * Shows if the global trace has been enabled or not. It uses the
302 * mirror flag "buffer_disabled" to be used in fast paths such as for
303 * the irqsoff tracer. But it may be inaccurate due to races. If you
304 * need to know the accurate state, use tracing_is_on() which is a little
305 * slower, but accurate.
307 int tracing_is_enabled(void)
310 * For quick access (irqsoff uses this in fast path), just
311 * return the mirror variable of the state of the ring buffer.
312 * It's a little racy, but we don't really care.
315 return !global_trace.buffer_disabled;
319 * trace_buf_size is the size in bytes that is allocated
320 * for a buffer. Note, the number of bytes is always rounded
323 * This number is purposely set to a low number of 16384.
324 * If the dump on oops happens, it will be much appreciated
325 * to not have to wait for all that output. Anyway this can be
326 * boot time and run time configurable.
328 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
330 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
332 /* trace_types holds a link list of available tracers. */
333 static struct tracer *trace_types __read_mostly;
336 * trace_types_lock is used to protect the trace_types list.
338 DEFINE_MUTEX(trace_types_lock);
341 * serialize the access of the ring buffer
343 * ring buffer serializes readers, but it is low level protection.
344 * The validity of the events (which returns by ring_buffer_peek() ..etc)
345 * are not protected by ring buffer.
347 * The content of events may become garbage if we allow other process consumes
348 * these events concurrently:
349 * A) the page of the consumed events may become a normal page
350 * (not reader page) in ring buffer, and this page will be rewrited
351 * by events producer.
352 * B) The page of the consumed events may become a page for splice_read,
353 * and this page will be returned to system.
355 * These primitives allow multi process access to different cpu ring buffer
358 * These primitives don't distinguish read-only and read-consume access.
359 * Multi read-only access are also serialized.
363 static DECLARE_RWSEM(all_cpu_access_lock);
364 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
366 static inline void trace_access_lock(int cpu)
368 if (cpu == RING_BUFFER_ALL_CPUS) {
369 /* gain it for accessing the whole ring buffer. */
370 down_write(&all_cpu_access_lock);
372 /* gain it for accessing a cpu ring buffer. */
374 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
375 down_read(&all_cpu_access_lock);
377 /* Secondly block other access to this @cpu ring buffer. */
378 mutex_lock(&per_cpu(cpu_access_lock, cpu));
382 static inline void trace_access_unlock(int cpu)
384 if (cpu == RING_BUFFER_ALL_CPUS) {
385 up_write(&all_cpu_access_lock);
387 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
388 up_read(&all_cpu_access_lock);
392 static inline void trace_access_lock_init(void)
396 for_each_possible_cpu(cpu)
397 mutex_init(&per_cpu(cpu_access_lock, cpu));
402 static DEFINE_MUTEX(access_lock);
404 static inline void trace_access_lock(int cpu)
407 mutex_lock(&access_lock);
410 static inline void trace_access_unlock(int cpu)
413 mutex_unlock(&access_lock);
416 static inline void trace_access_lock_init(void)
422 /* trace_flags holds trace_options default values */
423 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
424 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
425 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
426 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
428 static void tracer_tracing_on(struct trace_array *tr)
430 if (tr->trace_buffer.buffer)
431 ring_buffer_record_on(tr->trace_buffer.buffer);
433 * This flag is looked at when buffers haven't been allocated
434 * yet, or by some tracers (like irqsoff), that just want to
435 * know if the ring buffer has been disabled, but it can handle
436 * races of where it gets disabled but we still do a record.
437 * As the check is in the fast path of the tracers, it is more
438 * important to be fast than accurate.
440 tr->buffer_disabled = 0;
441 /* Make the flag seen by readers */
446 * tracing_on - enable tracing buffers
448 * This function enables tracing buffers that may have been
449 * disabled with tracing_off.
451 void tracing_on(void)
453 tracer_tracing_on(&global_trace);
455 EXPORT_SYMBOL_GPL(tracing_on);
458 * __trace_puts - write a constant string into the trace buffer.
459 * @ip: The address of the caller
460 * @str: The constant string to write
461 * @size: The size of the string.
463 int __trace_puts(unsigned long ip, const char *str, int size)
465 struct ring_buffer_event *event;
466 struct ring_buffer *buffer;
467 struct print_entry *entry;
468 unsigned long irq_flags;
472 if (!(trace_flags & TRACE_ITER_PRINTK))
475 pc = preempt_count();
477 if (unlikely(tracing_selftest_running || tracing_disabled))
480 alloc = sizeof(*entry) + size + 2; /* possible \n added */
482 local_save_flags(irq_flags);
483 buffer = global_trace.trace_buffer.buffer;
484 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
489 entry = ring_buffer_event_data(event);
492 memcpy(&entry->buf, str, size);
494 /* Add a newline if necessary */
495 if (entry->buf[size - 1] != '\n') {
496 entry->buf[size] = '\n';
497 entry->buf[size + 1] = '\0';
499 entry->buf[size] = '\0';
501 __buffer_unlock_commit(buffer, event);
502 ftrace_trace_stack(buffer, irq_flags, 4, pc);
506 EXPORT_SYMBOL_GPL(__trace_puts);
509 * __trace_bputs - write the pointer to a constant string into trace buffer
510 * @ip: The address of the caller
511 * @str: The constant string to write to the buffer to
513 int __trace_bputs(unsigned long ip, const char *str)
515 struct ring_buffer_event *event;
516 struct ring_buffer *buffer;
517 struct bputs_entry *entry;
518 unsigned long irq_flags;
519 int size = sizeof(struct bputs_entry);
522 if (!(trace_flags & TRACE_ITER_PRINTK))
525 pc = preempt_count();
527 if (unlikely(tracing_selftest_running || tracing_disabled))
530 local_save_flags(irq_flags);
531 buffer = global_trace.trace_buffer.buffer;
532 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
537 entry = ring_buffer_event_data(event);
541 __buffer_unlock_commit(buffer, event);
542 ftrace_trace_stack(buffer, irq_flags, 4, pc);
546 EXPORT_SYMBOL_GPL(__trace_bputs);
548 #ifdef CONFIG_TRACER_SNAPSHOT
550 * trace_snapshot - take a snapshot of the current buffer.
552 * This causes a swap between the snapshot buffer and the current live
553 * tracing buffer. You can use this to take snapshots of the live
554 * trace when some condition is triggered, but continue to trace.
556 * Note, make sure to allocate the snapshot with either
557 * a tracing_snapshot_alloc(), or by doing it manually
558 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
560 * If the snapshot buffer is not allocated, it will stop tracing.
561 * Basically making a permanent snapshot.
563 void tracing_snapshot(void)
565 struct trace_array *tr = &global_trace;
566 struct tracer *tracer = tr->current_trace;
570 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
571 internal_trace_puts("*** snapshot is being ignored ***\n");
575 if (!tr->allocated_snapshot) {
576 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
577 internal_trace_puts("*** stopping trace here! ***\n");
582 /* Note, snapshot can not be used when the tracer uses it */
583 if (tracer->use_max_tr) {
584 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
585 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
589 local_irq_save(flags);
590 update_max_tr(tr, current, smp_processor_id());
591 local_irq_restore(flags);
593 EXPORT_SYMBOL_GPL(tracing_snapshot);
595 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
596 struct trace_buffer *size_buf, int cpu_id);
597 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
599 static int alloc_snapshot(struct trace_array *tr)
603 if (!tr->allocated_snapshot) {
605 /* allocate spare buffer */
606 ret = resize_buffer_duplicate_size(&tr->max_buffer,
607 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
611 tr->allocated_snapshot = true;
617 static void free_snapshot(struct trace_array *tr)
620 * We don't free the ring buffer. instead, resize it because
621 * The max_tr ring buffer has some state (e.g. ring->clock) and
622 * we want preserve it.
624 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
625 set_buffer_entries(&tr->max_buffer, 1);
626 tracing_reset_online_cpus(&tr->max_buffer);
627 tr->allocated_snapshot = false;
631 * tracing_alloc_snapshot - allocate snapshot buffer.
633 * This only allocates the snapshot buffer if it isn't already
634 * allocated - it doesn't also take a snapshot.
636 * This is meant to be used in cases where the snapshot buffer needs
637 * to be set up for events that can't sleep but need to be able to
638 * trigger a snapshot.
640 int tracing_alloc_snapshot(void)
642 struct trace_array *tr = &global_trace;
645 ret = alloc_snapshot(tr);
650 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
653 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
655 * This is similar to trace_snapshot(), but it will allocate the
656 * snapshot buffer if it isn't already allocated. Use this only
657 * where it is safe to sleep, as the allocation may sleep.
659 * This causes a swap between the snapshot buffer and the current live
660 * tracing buffer. You can use this to take snapshots of the live
661 * trace when some condition is triggered, but continue to trace.
663 void tracing_snapshot_alloc(void)
667 ret = tracing_alloc_snapshot();
673 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
675 void tracing_snapshot(void)
677 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
679 EXPORT_SYMBOL_GPL(tracing_snapshot);
680 int tracing_alloc_snapshot(void)
682 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
685 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
686 void tracing_snapshot_alloc(void)
691 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
692 #endif /* CONFIG_TRACER_SNAPSHOT */
694 static void tracer_tracing_off(struct trace_array *tr)
696 if (tr->trace_buffer.buffer)
697 ring_buffer_record_off(tr->trace_buffer.buffer);
699 * This flag is looked at when buffers haven't been allocated
700 * yet, or by some tracers (like irqsoff), that just want to
701 * know if the ring buffer has been disabled, but it can handle
702 * races of where it gets disabled but we still do a record.
703 * As the check is in the fast path of the tracers, it is more
704 * important to be fast than accurate.
706 tr->buffer_disabled = 1;
707 /* Make the flag seen by readers */
712 * tracing_off - turn off tracing buffers
714 * This function stops the tracing buffers from recording data.
715 * It does not disable any overhead the tracers themselves may
716 * be causing. This function simply causes all recording to
717 * the ring buffers to fail.
719 void tracing_off(void)
721 tracer_tracing_off(&global_trace);
723 EXPORT_SYMBOL_GPL(tracing_off);
725 void disable_trace_on_warning(void)
727 if (__disable_trace_on_warning)
732 * tracer_tracing_is_on - show real state of ring buffer enabled
733 * @tr : the trace array to know if ring buffer is enabled
735 * Shows real state of the ring buffer if it is enabled or not.
737 static int tracer_tracing_is_on(struct trace_array *tr)
739 if (tr->trace_buffer.buffer)
740 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
741 return !tr->buffer_disabled;
745 * tracing_is_on - show state of ring buffers enabled
747 int tracing_is_on(void)
749 return tracer_tracing_is_on(&global_trace);
751 EXPORT_SYMBOL_GPL(tracing_is_on);
753 static int __init set_buf_size(char *str)
755 unsigned long buf_size;
759 buf_size = memparse(str, &str);
760 /* nr_entries can not be zero */
763 trace_buf_size = buf_size;
766 __setup("trace_buf_size=", set_buf_size);
768 static int __init set_tracing_thresh(char *str)
770 unsigned long threshold;
775 ret = kstrtoul(str, 0, &threshold);
778 tracing_thresh = threshold * 1000;
781 __setup("tracing_thresh=", set_tracing_thresh);
783 unsigned long nsecs_to_usecs(unsigned long nsecs)
788 /* These must match the bit postions in trace_iterator_flags */
789 static const char *trace_options[] = {
822 int in_ns; /* is this clock in nanoseconds? */
824 { trace_clock_local, "local", 1 },
825 { trace_clock_global, "global", 1 },
826 { trace_clock_counter, "counter", 0 },
827 { trace_clock_jiffies, "uptime", 0 },
828 { trace_clock, "perf", 1 },
829 { ktime_get_mono_fast_ns, "mono", 1 },
834 * trace_parser_get_init - gets the buffer for trace parser
836 int trace_parser_get_init(struct trace_parser *parser, int size)
838 memset(parser, 0, sizeof(*parser));
840 parser->buffer = kmalloc(size, GFP_KERNEL);
849 * trace_parser_put - frees the buffer for trace parser
851 void trace_parser_put(struct trace_parser *parser)
853 kfree(parser->buffer);
857 * trace_get_user - reads the user input string separated by space
858 * (matched by isspace(ch))
860 * For each string found the 'struct trace_parser' is updated,
861 * and the function returns.
863 * Returns number of bytes read.
865 * See kernel/trace/trace.h for 'struct trace_parser' details.
867 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
868 size_t cnt, loff_t *ppos)
875 trace_parser_clear(parser);
877 ret = get_user(ch, ubuf++);
885 * The parser is not finished with the last write,
886 * continue reading the user input without skipping spaces.
889 /* skip white space */
890 while (cnt && isspace(ch)) {
891 ret = get_user(ch, ubuf++);
898 /* only spaces were written */
908 /* read the non-space input */
909 while (cnt && !isspace(ch)) {
910 if (parser->idx < parser->size - 1)
911 parser->buffer[parser->idx++] = ch;
916 ret = get_user(ch, ubuf++);
923 /* We either got finished input or we have to wait for another call. */
925 parser->buffer[parser->idx] = 0;
926 parser->cont = false;
927 } else if (parser->idx < parser->size - 1) {
929 parser->buffer[parser->idx++] = ch;
942 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
946 if (s->len <= s->readpos)
949 len = s->len - s->readpos;
952 memcpy(buf, s->buffer + s->readpos, cnt);
958 unsigned long __read_mostly tracing_thresh;
960 #ifdef CONFIG_TRACER_MAX_TRACE
962 * Copy the new maximum trace into the separate maximum-trace
963 * structure. (this way the maximum trace is permanently saved,
964 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
967 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
969 struct trace_buffer *trace_buf = &tr->trace_buffer;
970 struct trace_buffer *max_buf = &tr->max_buffer;
971 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
972 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
975 max_buf->time_start = data->preempt_timestamp;
977 max_data->saved_latency = tr->max_latency;
978 max_data->critical_start = data->critical_start;
979 max_data->critical_end = data->critical_end;
981 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
982 max_data->pid = tsk->pid;
984 * If tsk == current, then use current_uid(), as that does not use
985 * RCU. The irq tracer can be called out of RCU scope.
988 max_data->uid = current_uid();
990 max_data->uid = task_uid(tsk);
992 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
993 max_data->policy = tsk->policy;
994 max_data->rt_priority = tsk->rt_priority;
996 /* record this tasks comm */
997 tracing_record_cmdline(tsk);
1001 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1003 * @tsk: the task with the latency
1004 * @cpu: The cpu that initiated the trace.
1006 * Flip the buffers between the @tr and the max_tr and record information
1007 * about which task was the cause of this latency.
1010 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1012 struct ring_buffer *buf;
1017 WARN_ON_ONCE(!irqs_disabled());
1019 if (!tr->allocated_snapshot) {
1020 /* Only the nop tracer should hit this when disabling */
1021 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1025 arch_spin_lock(&tr->max_lock);
1027 buf = tr->trace_buffer.buffer;
1028 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1029 tr->max_buffer.buffer = buf;
1031 __update_max_tr(tr, tsk, cpu);
1032 arch_spin_unlock(&tr->max_lock);
1036 * update_max_tr_single - only copy one trace over, and reset the rest
1038 * @tsk - task with the latency
1039 * @cpu - the cpu of the buffer to copy.
1041 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1044 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1051 WARN_ON_ONCE(!irqs_disabled());
1052 if (!tr->allocated_snapshot) {
1053 /* Only the nop tracer should hit this when disabling */
1054 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1058 arch_spin_lock(&tr->max_lock);
1060 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1062 if (ret == -EBUSY) {
1064 * We failed to swap the buffer due to a commit taking
1065 * place on this CPU. We fail to record, but we reset
1066 * the max trace buffer (no one writes directly to it)
1067 * and flag that it failed.
1069 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1070 "Failed to swap buffers due to commit in progress\n");
1073 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1075 __update_max_tr(tr, tsk, cpu);
1076 arch_spin_unlock(&tr->max_lock);
1078 #endif /* CONFIG_TRACER_MAX_TRACE */
1080 static int wait_on_pipe(struct trace_iterator *iter)
1082 /* Iterators are static, they should be filled or empty */
1083 if (trace_buffer_iter(iter, iter->cpu_file))
1086 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1089 #ifdef CONFIG_FTRACE_STARTUP_TEST
1090 static int run_tracer_selftest(struct tracer *type)
1092 struct trace_array *tr = &global_trace;
1093 struct tracer *saved_tracer = tr->current_trace;
1096 if (!type->selftest || tracing_selftest_disabled)
1100 * Run a selftest on this tracer.
1101 * Here we reset the trace buffer, and set the current
1102 * tracer to be this tracer. The tracer can then run some
1103 * internal tracing to verify that everything is in order.
1104 * If we fail, we do not register this tracer.
1106 tracing_reset_online_cpus(&tr->trace_buffer);
1108 tr->current_trace = type;
1110 #ifdef CONFIG_TRACER_MAX_TRACE
1111 if (type->use_max_tr) {
1112 /* If we expanded the buffers, make sure the max is expanded too */
1113 if (ring_buffer_expanded)
1114 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1115 RING_BUFFER_ALL_CPUS);
1116 tr->allocated_snapshot = true;
1120 /* the test is responsible for initializing and enabling */
1121 pr_info("Testing tracer %s: ", type->name);
1122 ret = type->selftest(type, tr);
1123 /* the test is responsible for resetting too */
1124 tr->current_trace = saved_tracer;
1126 printk(KERN_CONT "FAILED!\n");
1127 /* Add the warning after printing 'FAILED' */
1131 /* Only reset on passing, to avoid touching corrupted buffers */
1132 tracing_reset_online_cpus(&tr->trace_buffer);
1134 #ifdef CONFIG_TRACER_MAX_TRACE
1135 if (type->use_max_tr) {
1136 tr->allocated_snapshot = false;
1138 /* Shrink the max buffer again */
1139 if (ring_buffer_expanded)
1140 ring_buffer_resize(tr->max_buffer.buffer, 1,
1141 RING_BUFFER_ALL_CPUS);
1145 printk(KERN_CONT "PASSED\n");
1149 static inline int run_tracer_selftest(struct tracer *type)
1153 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1156 * register_tracer - register a tracer with the ftrace system.
1157 * @type - the plugin for the tracer
1159 * Register a new plugin tracer.
1161 int register_tracer(struct tracer *type)
1167 pr_info("Tracer must have a name\n");
1171 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1172 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1176 mutex_lock(&trace_types_lock);
1178 tracing_selftest_running = true;
1180 for (t = trace_types; t; t = t->next) {
1181 if (strcmp(type->name, t->name) == 0) {
1183 pr_info("Tracer %s already registered\n",
1190 if (!type->set_flag)
1191 type->set_flag = &dummy_set_flag;
1193 type->flags = &dummy_tracer_flags;
1195 if (!type->flags->opts)
1196 type->flags->opts = dummy_tracer_opt;
1198 ret = run_tracer_selftest(type);
1202 type->next = trace_types;
1206 tracing_selftest_running = false;
1207 mutex_unlock(&trace_types_lock);
1209 if (ret || !default_bootup_tracer)
1212 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1215 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1216 /* Do we want this tracer to start on bootup? */
1217 tracing_set_tracer(&global_trace, type->name);
1218 default_bootup_tracer = NULL;
1219 /* disable other selftests, since this will break it. */
1220 tracing_selftest_disabled = true;
1221 #ifdef CONFIG_FTRACE_STARTUP_TEST
1222 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1230 void tracing_reset(struct trace_buffer *buf, int cpu)
1232 struct ring_buffer *buffer = buf->buffer;
1237 ring_buffer_record_disable(buffer);
1239 /* Make sure all commits have finished */
1240 synchronize_sched();
1241 ring_buffer_reset_cpu(buffer, cpu);
1243 ring_buffer_record_enable(buffer);
1246 void tracing_reset_online_cpus(struct trace_buffer *buf)
1248 struct ring_buffer *buffer = buf->buffer;
1254 ring_buffer_record_disable(buffer);
1256 /* Make sure all commits have finished */
1257 synchronize_sched();
1259 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1261 for_each_online_cpu(cpu)
1262 ring_buffer_reset_cpu(buffer, cpu);
1264 ring_buffer_record_enable(buffer);
1267 /* Must have trace_types_lock held */
1268 void tracing_reset_all_online_cpus(void)
1270 struct trace_array *tr;
1272 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1273 tracing_reset_online_cpus(&tr->trace_buffer);
1274 #ifdef CONFIG_TRACER_MAX_TRACE
1275 tracing_reset_online_cpus(&tr->max_buffer);
1280 #define SAVED_CMDLINES_DEFAULT 128
1281 #define NO_CMDLINE_MAP UINT_MAX
1282 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1283 struct saved_cmdlines_buffer {
1284 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1285 unsigned *map_cmdline_to_pid;
1286 unsigned cmdline_num;
1288 char *saved_cmdlines;
1290 static struct saved_cmdlines_buffer *savedcmd;
1292 /* temporary disable recording */
1293 static atomic_t trace_record_cmdline_disabled __read_mostly;
1295 static inline char *get_saved_cmdlines(int idx)
1297 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1300 static inline void set_cmdline(int idx, const char *cmdline)
1302 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1305 static int allocate_cmdlines_buffer(unsigned int val,
1306 struct saved_cmdlines_buffer *s)
1308 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1310 if (!s->map_cmdline_to_pid)
1313 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1314 if (!s->saved_cmdlines) {
1315 kfree(s->map_cmdline_to_pid);
1320 s->cmdline_num = val;
1321 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1322 sizeof(s->map_pid_to_cmdline));
1323 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1324 val * sizeof(*s->map_cmdline_to_pid));
1329 static int trace_create_savedcmd(void)
1333 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1337 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1347 int is_tracing_stopped(void)
1349 return global_trace.stop_count;
1353 * tracing_start - quick start of the tracer
1355 * If tracing is enabled but was stopped by tracing_stop,
1356 * this will start the tracer back up.
1358 void tracing_start(void)
1360 struct ring_buffer *buffer;
1361 unsigned long flags;
1363 if (tracing_disabled)
1366 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1367 if (--global_trace.stop_count) {
1368 if (global_trace.stop_count < 0) {
1369 /* Someone screwed up their debugging */
1371 global_trace.stop_count = 0;
1376 /* Prevent the buffers from switching */
1377 arch_spin_lock(&global_trace.max_lock);
1379 buffer = global_trace.trace_buffer.buffer;
1381 ring_buffer_record_enable(buffer);
1383 #ifdef CONFIG_TRACER_MAX_TRACE
1384 buffer = global_trace.max_buffer.buffer;
1386 ring_buffer_record_enable(buffer);
1389 arch_spin_unlock(&global_trace.max_lock);
1392 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1395 static void tracing_start_tr(struct trace_array *tr)
1397 struct ring_buffer *buffer;
1398 unsigned long flags;
1400 if (tracing_disabled)
1403 /* If global, we need to also start the max tracer */
1404 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1405 return tracing_start();
1407 raw_spin_lock_irqsave(&tr->start_lock, flags);
1409 if (--tr->stop_count) {
1410 if (tr->stop_count < 0) {
1411 /* Someone screwed up their debugging */
1418 buffer = tr->trace_buffer.buffer;
1420 ring_buffer_record_enable(buffer);
1423 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1427 * tracing_stop - quick stop of the tracer
1429 * Light weight way to stop tracing. Use in conjunction with
1432 void tracing_stop(void)
1434 struct ring_buffer *buffer;
1435 unsigned long flags;
1437 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1438 if (global_trace.stop_count++)
1441 /* Prevent the buffers from switching */
1442 arch_spin_lock(&global_trace.max_lock);
1444 buffer = global_trace.trace_buffer.buffer;
1446 ring_buffer_record_disable(buffer);
1448 #ifdef CONFIG_TRACER_MAX_TRACE
1449 buffer = global_trace.max_buffer.buffer;
1451 ring_buffer_record_disable(buffer);
1454 arch_spin_unlock(&global_trace.max_lock);
1457 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1460 static void tracing_stop_tr(struct trace_array *tr)
1462 struct ring_buffer *buffer;
1463 unsigned long flags;
1465 /* If global, we need to also stop the max tracer */
1466 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1467 return tracing_stop();
1469 raw_spin_lock_irqsave(&tr->start_lock, flags);
1470 if (tr->stop_count++)
1473 buffer = tr->trace_buffer.buffer;
1475 ring_buffer_record_disable(buffer);
1478 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1481 void trace_stop_cmdline_recording(void);
1483 static int trace_save_cmdline(struct task_struct *tsk)
1487 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1491 * It's not the end of the world if we don't get
1492 * the lock, but we also don't want to spin
1493 * nor do we want to disable interrupts,
1494 * so if we miss here, then better luck next time.
1496 if (!arch_spin_trylock(&trace_cmdline_lock))
1499 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1500 if (idx == NO_CMDLINE_MAP) {
1501 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1504 * Check whether the cmdline buffer at idx has a pid
1505 * mapped. We are going to overwrite that entry so we
1506 * need to clear the map_pid_to_cmdline. Otherwise we
1507 * would read the new comm for the old pid.
1509 pid = savedcmd->map_cmdline_to_pid[idx];
1510 if (pid != NO_CMDLINE_MAP)
1511 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1513 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1514 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1516 savedcmd->cmdline_idx = idx;
1519 set_cmdline(idx, tsk->comm);
1521 arch_spin_unlock(&trace_cmdline_lock);
1526 static void __trace_find_cmdline(int pid, char comm[])
1531 strcpy(comm, "<idle>");
1535 if (WARN_ON_ONCE(pid < 0)) {
1536 strcpy(comm, "<XXX>");
1540 if (pid > PID_MAX_DEFAULT) {
1541 strcpy(comm, "<...>");
1545 map = savedcmd->map_pid_to_cmdline[pid];
1546 if (map != NO_CMDLINE_MAP)
1547 strcpy(comm, get_saved_cmdlines(map));
1549 strcpy(comm, "<...>");
1552 void trace_find_cmdline(int pid, char comm[])
1555 arch_spin_lock(&trace_cmdline_lock);
1557 __trace_find_cmdline(pid, comm);
1559 arch_spin_unlock(&trace_cmdline_lock);
1563 void tracing_record_cmdline(struct task_struct *tsk)
1565 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1568 if (!__this_cpu_read(trace_cmdline_save))
1571 if (trace_save_cmdline(tsk))
1572 __this_cpu_write(trace_cmdline_save, false);
1576 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1579 struct task_struct *tsk = current;
1581 entry->preempt_count = pc & 0xff;
1582 entry->pid = (tsk) ? tsk->pid : 0;
1584 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1585 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1587 TRACE_FLAG_IRQS_NOSUPPORT |
1589 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1590 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1591 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1592 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1594 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1596 struct ring_buffer_event *
1597 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1600 unsigned long flags, int pc)
1602 struct ring_buffer_event *event;
1604 event = ring_buffer_lock_reserve(buffer, len);
1605 if (event != NULL) {
1606 struct trace_entry *ent = ring_buffer_event_data(event);
1608 tracing_generic_entry_update(ent, flags, pc);
1616 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1618 __this_cpu_write(trace_cmdline_save, true);
1619 ring_buffer_unlock_commit(buffer, event);
1623 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1624 struct ring_buffer_event *event,
1625 unsigned long flags, int pc)
1627 __buffer_unlock_commit(buffer, event);
1629 ftrace_trace_stack(buffer, flags, 6, pc);
1630 ftrace_trace_userstack(buffer, flags, pc);
1633 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1634 struct ring_buffer_event *event,
1635 unsigned long flags, int pc)
1637 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1639 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1641 static struct ring_buffer *temp_buffer;
1643 struct ring_buffer_event *
1644 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1645 struct ftrace_event_file *ftrace_file,
1646 int type, unsigned long len,
1647 unsigned long flags, int pc)
1649 struct ring_buffer_event *entry;
1651 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1652 entry = trace_buffer_lock_reserve(*current_rb,
1653 type, len, flags, pc);
1655 * If tracing is off, but we have triggers enabled
1656 * we still need to look at the event data. Use the temp_buffer
1657 * to store the trace event for the tigger to use. It's recusive
1658 * safe and will not be recorded anywhere.
1660 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1661 *current_rb = temp_buffer;
1662 entry = trace_buffer_lock_reserve(*current_rb,
1663 type, len, flags, pc);
1667 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1669 struct ring_buffer_event *
1670 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1671 int type, unsigned long len,
1672 unsigned long flags, int pc)
1674 *current_rb = global_trace.trace_buffer.buffer;
1675 return trace_buffer_lock_reserve(*current_rb,
1676 type, len, flags, pc);
1678 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1680 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1681 struct ring_buffer_event *event,
1682 unsigned long flags, int pc)
1684 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1686 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1688 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1689 struct ring_buffer_event *event,
1690 unsigned long flags, int pc,
1691 struct pt_regs *regs)
1693 __buffer_unlock_commit(buffer, event);
1695 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1696 ftrace_trace_userstack(buffer, flags, pc);
1698 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1700 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1701 struct ring_buffer_event *event)
1703 ring_buffer_discard_commit(buffer, event);
1705 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1708 trace_function(struct trace_array *tr,
1709 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1712 struct ftrace_event_call *call = &event_function;
1713 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1714 struct ring_buffer_event *event;
1715 struct ftrace_entry *entry;
1717 /* If we are reading the ring buffer, don't trace */
1718 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1721 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1725 entry = ring_buffer_event_data(event);
1727 entry->parent_ip = parent_ip;
1729 if (!call_filter_check_discard(call, entry, buffer, event))
1730 __buffer_unlock_commit(buffer, event);
1733 #ifdef CONFIG_STACKTRACE
1735 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1736 struct ftrace_stack {
1737 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1740 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1741 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1743 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1744 unsigned long flags,
1745 int skip, int pc, struct pt_regs *regs)
1747 struct ftrace_event_call *call = &event_kernel_stack;
1748 struct ring_buffer_event *event;
1749 struct stack_entry *entry;
1750 struct stack_trace trace;
1752 int size = FTRACE_STACK_ENTRIES;
1754 trace.nr_entries = 0;
1758 * Since events can happen in NMIs there's no safe way to
1759 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1760 * or NMI comes in, it will just have to use the default
1761 * FTRACE_STACK_SIZE.
1763 preempt_disable_notrace();
1765 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1767 * We don't need any atomic variables, just a barrier.
1768 * If an interrupt comes in, we don't care, because it would
1769 * have exited and put the counter back to what we want.
1770 * We just need a barrier to keep gcc from moving things
1774 if (use_stack == 1) {
1775 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1776 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1779 save_stack_trace_regs(regs, &trace);
1781 save_stack_trace(&trace);
1783 if (trace.nr_entries > size)
1784 size = trace.nr_entries;
1786 /* From now on, use_stack is a boolean */
1789 size *= sizeof(unsigned long);
1791 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1792 sizeof(*entry) + size, flags, pc);
1795 entry = ring_buffer_event_data(event);
1797 memset(&entry->caller, 0, size);
1800 memcpy(&entry->caller, trace.entries,
1801 trace.nr_entries * sizeof(unsigned long));
1803 trace.max_entries = FTRACE_STACK_ENTRIES;
1804 trace.entries = entry->caller;
1806 save_stack_trace_regs(regs, &trace);
1808 save_stack_trace(&trace);
1811 entry->size = trace.nr_entries;
1813 if (!call_filter_check_discard(call, entry, buffer, event))
1814 __buffer_unlock_commit(buffer, event);
1817 /* Again, don't let gcc optimize things here */
1819 __this_cpu_dec(ftrace_stack_reserve);
1820 preempt_enable_notrace();
1824 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1825 int skip, int pc, struct pt_regs *regs)
1827 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1830 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1833 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1836 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1839 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1842 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1845 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1849 * trace_dump_stack - record a stack back trace in the trace buffer
1850 * @skip: Number of functions to skip (helper handlers)
1852 void trace_dump_stack(int skip)
1854 unsigned long flags;
1856 if (tracing_disabled || tracing_selftest_running)
1859 local_save_flags(flags);
1862 * Skip 3 more, seems to get us at the caller of
1866 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1867 flags, skip, preempt_count(), NULL);
1870 static DEFINE_PER_CPU(int, user_stack_count);
1873 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1875 struct ftrace_event_call *call = &event_user_stack;
1876 struct ring_buffer_event *event;
1877 struct userstack_entry *entry;
1878 struct stack_trace trace;
1880 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1884 * NMIs can not handle page faults, even with fix ups.
1885 * The save user stack can (and often does) fault.
1887 if (unlikely(in_nmi()))
1891 * prevent recursion, since the user stack tracing may
1892 * trigger other kernel events.
1895 if (__this_cpu_read(user_stack_count))
1898 __this_cpu_inc(user_stack_count);
1900 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1901 sizeof(*entry), flags, pc);
1903 goto out_drop_count;
1904 entry = ring_buffer_event_data(event);
1906 entry->tgid = current->tgid;
1907 memset(&entry->caller, 0, sizeof(entry->caller));
1909 trace.nr_entries = 0;
1910 trace.max_entries = FTRACE_STACK_ENTRIES;
1912 trace.entries = entry->caller;
1914 save_stack_trace_user(&trace);
1915 if (!call_filter_check_discard(call, entry, buffer, event))
1916 __buffer_unlock_commit(buffer, event);
1919 __this_cpu_dec(user_stack_count);
1925 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1927 ftrace_trace_userstack(tr, flags, preempt_count());
1931 #endif /* CONFIG_STACKTRACE */
1933 /* created for use with alloc_percpu */
1934 struct trace_buffer_struct {
1935 char buffer[TRACE_BUF_SIZE];
1938 static struct trace_buffer_struct *trace_percpu_buffer;
1939 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1940 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1941 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1944 * The buffer used is dependent on the context. There is a per cpu
1945 * buffer for normal context, softirq contex, hard irq context and
1946 * for NMI context. Thise allows for lockless recording.
1948 * Note, if the buffers failed to be allocated, then this returns NULL
1950 static char *get_trace_buf(void)
1952 struct trace_buffer_struct *percpu_buffer;
1955 * If we have allocated per cpu buffers, then we do not
1956 * need to do any locking.
1959 percpu_buffer = trace_percpu_nmi_buffer;
1961 percpu_buffer = trace_percpu_irq_buffer;
1962 else if (in_softirq())
1963 percpu_buffer = trace_percpu_sirq_buffer;
1965 percpu_buffer = trace_percpu_buffer;
1970 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1973 static int alloc_percpu_trace_buffer(void)
1975 struct trace_buffer_struct *buffers;
1976 struct trace_buffer_struct *sirq_buffers;
1977 struct trace_buffer_struct *irq_buffers;
1978 struct trace_buffer_struct *nmi_buffers;
1980 buffers = alloc_percpu(struct trace_buffer_struct);
1984 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1988 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1992 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1996 trace_percpu_buffer = buffers;
1997 trace_percpu_sirq_buffer = sirq_buffers;
1998 trace_percpu_irq_buffer = irq_buffers;
1999 trace_percpu_nmi_buffer = nmi_buffers;
2004 free_percpu(irq_buffers);
2006 free_percpu(sirq_buffers);
2008 free_percpu(buffers);
2010 WARN(1, "Could not allocate percpu trace_printk buffer");
2014 static int buffers_allocated;
2016 void trace_printk_init_buffers(void)
2018 if (buffers_allocated)
2021 if (alloc_percpu_trace_buffer())
2024 /* trace_printk() is for debug use only. Don't use it in production. */
2026 pr_warning("\n**********************************************************\n");
2027 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2028 pr_warning("** **\n");
2029 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2030 pr_warning("** **\n");
2031 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2032 pr_warning("** unsafe for produciton use. **\n");
2033 pr_warning("** **\n");
2034 pr_warning("** If you see this message and you are not debugging **\n");
2035 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2036 pr_warning("** **\n");
2037 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2038 pr_warning("**********************************************************\n");
2040 /* Expand the buffers to set size */
2041 tracing_update_buffers();
2043 buffers_allocated = 1;
2046 * trace_printk_init_buffers() can be called by modules.
2047 * If that happens, then we need to start cmdline recording
2048 * directly here. If the global_trace.buffer is already
2049 * allocated here, then this was called by module code.
2051 if (global_trace.trace_buffer.buffer)
2052 tracing_start_cmdline_record();
2055 void trace_printk_start_comm(void)
2057 /* Start tracing comms if trace printk is set */
2058 if (!buffers_allocated)
2060 tracing_start_cmdline_record();
2063 static void trace_printk_start_stop_comm(int enabled)
2065 if (!buffers_allocated)
2069 tracing_start_cmdline_record();
2071 tracing_stop_cmdline_record();
2075 * trace_vbprintk - write binary msg to tracing buffer
2078 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2080 struct ftrace_event_call *call = &event_bprint;
2081 struct ring_buffer_event *event;
2082 struct ring_buffer *buffer;
2083 struct trace_array *tr = &global_trace;
2084 struct bprint_entry *entry;
2085 unsigned long flags;
2087 int len = 0, size, pc;
2089 if (unlikely(tracing_selftest_running || tracing_disabled))
2092 /* Don't pollute graph traces with trace_vprintk internals */
2093 pause_graph_tracing();
2095 pc = preempt_count();
2096 preempt_disable_notrace();
2098 tbuffer = get_trace_buf();
2104 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2106 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2109 local_save_flags(flags);
2110 size = sizeof(*entry) + sizeof(u32) * len;
2111 buffer = tr->trace_buffer.buffer;
2112 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2116 entry = ring_buffer_event_data(event);
2120 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2121 if (!call_filter_check_discard(call, entry, buffer, event)) {
2122 __buffer_unlock_commit(buffer, event);
2123 ftrace_trace_stack(buffer, flags, 6, pc);
2127 preempt_enable_notrace();
2128 unpause_graph_tracing();
2132 EXPORT_SYMBOL_GPL(trace_vbprintk);
2135 __trace_array_vprintk(struct ring_buffer *buffer,
2136 unsigned long ip, const char *fmt, va_list args)
2138 struct ftrace_event_call *call = &event_print;
2139 struct ring_buffer_event *event;
2140 int len = 0, size, pc;
2141 struct print_entry *entry;
2142 unsigned long flags;
2145 if (tracing_disabled || tracing_selftest_running)
2148 /* Don't pollute graph traces with trace_vprintk internals */
2149 pause_graph_tracing();
2151 pc = preempt_count();
2152 preempt_disable_notrace();
2155 tbuffer = get_trace_buf();
2161 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2162 if (len > TRACE_BUF_SIZE)
2165 local_save_flags(flags);
2166 size = sizeof(*entry) + len + 1;
2167 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2171 entry = ring_buffer_event_data(event);
2174 memcpy(&entry->buf, tbuffer, len);
2175 entry->buf[len] = '\0';
2176 if (!call_filter_check_discard(call, entry, buffer, event)) {
2177 __buffer_unlock_commit(buffer, event);
2178 ftrace_trace_stack(buffer, flags, 6, pc);
2181 preempt_enable_notrace();
2182 unpause_graph_tracing();
2187 int trace_array_vprintk(struct trace_array *tr,
2188 unsigned long ip, const char *fmt, va_list args)
2190 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2193 int trace_array_printk(struct trace_array *tr,
2194 unsigned long ip, const char *fmt, ...)
2199 if (!(trace_flags & TRACE_ITER_PRINTK))
2203 ret = trace_array_vprintk(tr, ip, fmt, ap);
2208 int trace_array_printk_buf(struct ring_buffer *buffer,
2209 unsigned long ip, const char *fmt, ...)
2214 if (!(trace_flags & TRACE_ITER_PRINTK))
2218 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2223 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2225 return trace_array_vprintk(&global_trace, ip, fmt, args);
2227 EXPORT_SYMBOL_GPL(trace_vprintk);
2229 static void trace_iterator_increment(struct trace_iterator *iter)
2231 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2235 ring_buffer_read(buf_iter, NULL);
2238 static struct trace_entry *
2239 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2240 unsigned long *lost_events)
2242 struct ring_buffer_event *event;
2243 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2246 event = ring_buffer_iter_peek(buf_iter, ts);
2248 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2252 iter->ent_size = ring_buffer_event_length(event);
2253 return ring_buffer_event_data(event);
2259 static struct trace_entry *
2260 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2261 unsigned long *missing_events, u64 *ent_ts)
2263 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2264 struct trace_entry *ent, *next = NULL;
2265 unsigned long lost_events = 0, next_lost = 0;
2266 int cpu_file = iter->cpu_file;
2267 u64 next_ts = 0, ts;
2273 * If we are in a per_cpu trace file, don't bother by iterating over
2274 * all cpu and peek directly.
2276 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2277 if (ring_buffer_empty_cpu(buffer, cpu_file))
2279 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2281 *ent_cpu = cpu_file;
2286 for_each_tracing_cpu(cpu) {
2288 if (ring_buffer_empty_cpu(buffer, cpu))
2291 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2294 * Pick the entry with the smallest timestamp:
2296 if (ent && (!next || ts < next_ts)) {
2300 next_lost = lost_events;
2301 next_size = iter->ent_size;
2305 iter->ent_size = next_size;
2308 *ent_cpu = next_cpu;
2314 *missing_events = next_lost;
2319 /* Find the next real entry, without updating the iterator itself */
2320 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2321 int *ent_cpu, u64 *ent_ts)
2323 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2326 /* Find the next real entry, and increment the iterator to the next entry */
2327 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2329 iter->ent = __find_next_entry(iter, &iter->cpu,
2330 &iter->lost_events, &iter->ts);
2333 trace_iterator_increment(iter);
2335 return iter->ent ? iter : NULL;
2338 static void trace_consume(struct trace_iterator *iter)
2340 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2341 &iter->lost_events);
2344 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2346 struct trace_iterator *iter = m->private;
2350 WARN_ON_ONCE(iter->leftover);
2354 /* can't go backwards */
2359 ent = trace_find_next_entry_inc(iter);
2363 while (ent && iter->idx < i)
2364 ent = trace_find_next_entry_inc(iter);
2371 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2373 struct ring_buffer_event *event;
2374 struct ring_buffer_iter *buf_iter;
2375 unsigned long entries = 0;
2378 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2380 buf_iter = trace_buffer_iter(iter, cpu);
2384 ring_buffer_iter_reset(buf_iter);
2387 * We could have the case with the max latency tracers
2388 * that a reset never took place on a cpu. This is evident
2389 * by the timestamp being before the start of the buffer.
2391 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2392 if (ts >= iter->trace_buffer->time_start)
2395 ring_buffer_read(buf_iter, NULL);
2398 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2402 * The current tracer is copied to avoid a global locking
2405 static void *s_start(struct seq_file *m, loff_t *pos)
2407 struct trace_iterator *iter = m->private;
2408 struct trace_array *tr = iter->tr;
2409 int cpu_file = iter->cpu_file;
2415 * copy the tracer to avoid using a global lock all around.
2416 * iter->trace is a copy of current_trace, the pointer to the
2417 * name may be used instead of a strcmp(), as iter->trace->name
2418 * will point to the same string as current_trace->name.
2420 mutex_lock(&trace_types_lock);
2421 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2422 *iter->trace = *tr->current_trace;
2423 mutex_unlock(&trace_types_lock);
2425 #ifdef CONFIG_TRACER_MAX_TRACE
2426 if (iter->snapshot && iter->trace->use_max_tr)
2427 return ERR_PTR(-EBUSY);
2430 if (!iter->snapshot)
2431 atomic_inc(&trace_record_cmdline_disabled);
2433 if (*pos != iter->pos) {
2438 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2439 for_each_tracing_cpu(cpu)
2440 tracing_iter_reset(iter, cpu);
2442 tracing_iter_reset(iter, cpu_file);
2445 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2450 * If we overflowed the seq_file before, then we want
2451 * to just reuse the trace_seq buffer again.
2457 p = s_next(m, p, &l);
2461 trace_event_read_lock();
2462 trace_access_lock(cpu_file);
2466 static void s_stop(struct seq_file *m, void *p)
2468 struct trace_iterator *iter = m->private;
2470 #ifdef CONFIG_TRACER_MAX_TRACE
2471 if (iter->snapshot && iter->trace->use_max_tr)
2475 if (!iter->snapshot)
2476 atomic_dec(&trace_record_cmdline_disabled);
2478 trace_access_unlock(iter->cpu_file);
2479 trace_event_read_unlock();
2483 get_total_entries(struct trace_buffer *buf,
2484 unsigned long *total, unsigned long *entries)
2486 unsigned long count;
2492 for_each_tracing_cpu(cpu) {
2493 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2495 * If this buffer has skipped entries, then we hold all
2496 * entries for the trace and we need to ignore the
2497 * ones before the time stamp.
2499 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2500 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2501 /* total is the same as the entries */
2505 ring_buffer_overrun_cpu(buf->buffer, cpu);
2510 static void print_lat_help_header(struct seq_file *m)
2512 seq_puts(m, "# _------=> CPU# \n"
2513 "# / _-----=> irqs-off \n"
2514 "# | / _----=> need-resched \n"
2515 "# || / _---=> hardirq/softirq \n"
2516 "# ||| / _--=> preempt-depth \n"
2518 "# cmd pid ||||| time | caller \n"
2519 "# \\ / ||||| \\ | / \n");
2522 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2524 unsigned long total;
2525 unsigned long entries;
2527 get_total_entries(buf, &total, &entries);
2528 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2529 entries, total, num_online_cpus());
2533 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2535 print_event_info(buf, m);
2536 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2540 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2542 print_event_info(buf, m);
2543 seq_puts(m, "# _-----=> irqs-off\n"
2544 "# / _----=> need-resched\n"
2545 "# | / _---=> hardirq/softirq\n"
2546 "# || / _--=> preempt-depth\n"
2548 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2549 "# | | | |||| | |\n");
2553 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2555 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2556 struct trace_buffer *buf = iter->trace_buffer;
2557 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2558 struct tracer *type = iter->trace;
2559 unsigned long entries;
2560 unsigned long total;
2561 const char *name = "preemption";
2565 get_total_entries(buf, &total, &entries);
2567 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2569 seq_puts(m, "# -----------------------------------"
2570 "---------------------------------\n");
2571 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2572 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2573 nsecs_to_usecs(data->saved_latency),
2577 #if defined(CONFIG_PREEMPT_NONE)
2579 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2581 #elif defined(CONFIG_PREEMPT)
2586 /* These are reserved for later use */
2589 seq_printf(m, " #P:%d)\n", num_online_cpus());
2593 seq_puts(m, "# -----------------\n");
2594 seq_printf(m, "# | task: %.16s-%d "
2595 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2596 data->comm, data->pid,
2597 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2598 data->policy, data->rt_priority);
2599 seq_puts(m, "# -----------------\n");
2601 if (data->critical_start) {
2602 seq_puts(m, "# => started at: ");
2603 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2604 trace_print_seq(m, &iter->seq);
2605 seq_puts(m, "\n# => ended at: ");
2606 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2607 trace_print_seq(m, &iter->seq);
2608 seq_puts(m, "\n#\n");
2614 static void test_cpu_buff_start(struct trace_iterator *iter)
2616 struct trace_seq *s = &iter->seq;
2618 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2621 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2624 if (cpumask_test_cpu(iter->cpu, iter->started))
2627 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2630 cpumask_set_cpu(iter->cpu, iter->started);
2632 /* Don't print started cpu buffer for the first entry of the trace */
2634 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2638 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2640 struct trace_seq *s = &iter->seq;
2641 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2642 struct trace_entry *entry;
2643 struct trace_event *event;
2647 test_cpu_buff_start(iter);
2649 event = ftrace_find_event(entry->type);
2651 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2652 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2653 trace_print_lat_context(iter);
2655 trace_print_context(iter);
2658 if (trace_seq_has_overflowed(s))
2659 return TRACE_TYPE_PARTIAL_LINE;
2662 return event->funcs->trace(iter, sym_flags, event);
2664 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2666 return trace_handle_return(s);
2669 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2671 struct trace_seq *s = &iter->seq;
2672 struct trace_entry *entry;
2673 struct trace_event *event;
2677 if (trace_flags & TRACE_ITER_CONTEXT_INFO)
2678 trace_seq_printf(s, "%d %d %llu ",
2679 entry->pid, iter->cpu, iter->ts);
2681 if (trace_seq_has_overflowed(s))
2682 return TRACE_TYPE_PARTIAL_LINE;
2684 event = ftrace_find_event(entry->type);
2686 return event->funcs->raw(iter, 0, event);
2688 trace_seq_printf(s, "%d ?\n", entry->type);
2690 return trace_handle_return(s);
2693 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2695 struct trace_seq *s = &iter->seq;
2696 unsigned char newline = '\n';
2697 struct trace_entry *entry;
2698 struct trace_event *event;
2702 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2703 SEQ_PUT_HEX_FIELD(s, entry->pid);
2704 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2705 SEQ_PUT_HEX_FIELD(s, iter->ts);
2706 if (trace_seq_has_overflowed(s))
2707 return TRACE_TYPE_PARTIAL_LINE;
2710 event = ftrace_find_event(entry->type);
2712 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2713 if (ret != TRACE_TYPE_HANDLED)
2717 SEQ_PUT_FIELD(s, newline);
2719 return trace_handle_return(s);
2722 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2724 struct trace_seq *s = &iter->seq;
2725 struct trace_entry *entry;
2726 struct trace_event *event;
2730 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2731 SEQ_PUT_FIELD(s, entry->pid);
2732 SEQ_PUT_FIELD(s, iter->cpu);
2733 SEQ_PUT_FIELD(s, iter->ts);
2734 if (trace_seq_has_overflowed(s))
2735 return TRACE_TYPE_PARTIAL_LINE;
2738 event = ftrace_find_event(entry->type);
2739 return event ? event->funcs->binary(iter, 0, event) :
2743 int trace_empty(struct trace_iterator *iter)
2745 struct ring_buffer_iter *buf_iter;
2748 /* If we are looking at one CPU buffer, only check that one */
2749 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2750 cpu = iter->cpu_file;
2751 buf_iter = trace_buffer_iter(iter, cpu);
2753 if (!ring_buffer_iter_empty(buf_iter))
2756 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2762 for_each_tracing_cpu(cpu) {
2763 buf_iter = trace_buffer_iter(iter, cpu);
2765 if (!ring_buffer_iter_empty(buf_iter))
2768 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2776 /* Called with trace_event_read_lock() held. */
2777 enum print_line_t print_trace_line(struct trace_iterator *iter)
2779 enum print_line_t ret;
2781 if (iter->lost_events) {
2782 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2783 iter->cpu, iter->lost_events);
2784 if (trace_seq_has_overflowed(&iter->seq))
2785 return TRACE_TYPE_PARTIAL_LINE;
2788 if (iter->trace && iter->trace->print_line) {
2789 ret = iter->trace->print_line(iter);
2790 if (ret != TRACE_TYPE_UNHANDLED)
2794 if (iter->ent->type == TRACE_BPUTS &&
2795 trace_flags & TRACE_ITER_PRINTK &&
2796 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2797 return trace_print_bputs_msg_only(iter);
2799 if (iter->ent->type == TRACE_BPRINT &&
2800 trace_flags & TRACE_ITER_PRINTK &&
2801 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2802 return trace_print_bprintk_msg_only(iter);
2804 if (iter->ent->type == TRACE_PRINT &&
2805 trace_flags & TRACE_ITER_PRINTK &&
2806 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2807 return trace_print_printk_msg_only(iter);
2809 if (trace_flags & TRACE_ITER_BIN)
2810 return print_bin_fmt(iter);
2812 if (trace_flags & TRACE_ITER_HEX)
2813 return print_hex_fmt(iter);
2815 if (trace_flags & TRACE_ITER_RAW)
2816 return print_raw_fmt(iter);
2818 return print_trace_fmt(iter);
2821 void trace_latency_header(struct seq_file *m)
2823 struct trace_iterator *iter = m->private;
2825 /* print nothing if the buffers are empty */
2826 if (trace_empty(iter))
2829 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2830 print_trace_header(m, iter);
2832 if (!(trace_flags & TRACE_ITER_VERBOSE))
2833 print_lat_help_header(m);
2836 void trace_default_header(struct seq_file *m)
2838 struct trace_iterator *iter = m->private;
2840 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2843 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2844 /* print nothing if the buffers are empty */
2845 if (trace_empty(iter))
2847 print_trace_header(m, iter);
2848 if (!(trace_flags & TRACE_ITER_VERBOSE))
2849 print_lat_help_header(m);
2851 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2852 if (trace_flags & TRACE_ITER_IRQ_INFO)
2853 print_func_help_header_irq(iter->trace_buffer, m);
2855 print_func_help_header(iter->trace_buffer, m);
2860 static void test_ftrace_alive(struct seq_file *m)
2862 if (!ftrace_is_dead())
2864 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2865 "# MAY BE MISSING FUNCTION EVENTS\n");
2868 #ifdef CONFIG_TRACER_MAX_TRACE
2869 static void show_snapshot_main_help(struct seq_file *m)
2871 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2872 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2873 "# Takes a snapshot of the main buffer.\n"
2874 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2875 "# (Doesn't have to be '2' works with any number that\n"
2876 "# is not a '0' or '1')\n");
2879 static void show_snapshot_percpu_help(struct seq_file *m)
2881 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2882 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2883 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2884 "# Takes a snapshot of the main buffer for this cpu.\n");
2886 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2887 "# Must use main snapshot file to allocate.\n");
2889 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2890 "# (Doesn't have to be '2' works with any number that\n"
2891 "# is not a '0' or '1')\n");
2894 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2896 if (iter->tr->allocated_snapshot)
2897 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2899 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2901 seq_puts(m, "# Snapshot commands:\n");
2902 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2903 show_snapshot_main_help(m);
2905 show_snapshot_percpu_help(m);
2908 /* Should never be called */
2909 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2912 static int s_show(struct seq_file *m, void *v)
2914 struct trace_iterator *iter = v;
2917 if (iter->ent == NULL) {
2919 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2921 test_ftrace_alive(m);
2923 if (iter->snapshot && trace_empty(iter))
2924 print_snapshot_help(m, iter);
2925 else if (iter->trace && iter->trace->print_header)
2926 iter->trace->print_header(m);
2928 trace_default_header(m);
2930 } else if (iter->leftover) {
2932 * If we filled the seq_file buffer earlier, we
2933 * want to just show it now.
2935 ret = trace_print_seq(m, &iter->seq);
2937 /* ret should this time be zero, but you never know */
2938 iter->leftover = ret;
2941 print_trace_line(iter);
2942 ret = trace_print_seq(m, &iter->seq);
2944 * If we overflow the seq_file buffer, then it will
2945 * ask us for this data again at start up.
2947 * ret is 0 if seq_file write succeeded.
2950 iter->leftover = ret;
2957 * Should be used after trace_array_get(), trace_types_lock
2958 * ensures that i_cdev was already initialized.
2960 static inline int tracing_get_cpu(struct inode *inode)
2962 if (inode->i_cdev) /* See trace_create_cpu_file() */
2963 return (long)inode->i_cdev - 1;
2964 return RING_BUFFER_ALL_CPUS;
2967 static const struct seq_operations tracer_seq_ops = {
2974 static struct trace_iterator *
2975 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2977 struct trace_array *tr = inode->i_private;
2978 struct trace_iterator *iter;
2981 if (tracing_disabled)
2982 return ERR_PTR(-ENODEV);
2984 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2986 return ERR_PTR(-ENOMEM);
2988 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2990 if (!iter->buffer_iter)
2994 * We make a copy of the current tracer to avoid concurrent
2995 * changes on it while we are reading.
2997 mutex_lock(&trace_types_lock);
2998 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3002 *iter->trace = *tr->current_trace;
3004 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3009 #ifdef CONFIG_TRACER_MAX_TRACE
3010 /* Currently only the top directory has a snapshot */
3011 if (tr->current_trace->print_max || snapshot)
3012 iter->trace_buffer = &tr->max_buffer;
3015 iter->trace_buffer = &tr->trace_buffer;
3016 iter->snapshot = snapshot;
3018 iter->cpu_file = tracing_get_cpu(inode);
3019 mutex_init(&iter->mutex);
3021 /* Notify the tracer early; before we stop tracing. */
3022 if (iter->trace && iter->trace->open)
3023 iter->trace->open(iter);
3025 /* Annotate start of buffers if we had overruns */
3026 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3027 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3029 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3030 if (trace_clocks[tr->clock_id].in_ns)
3031 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3033 /* stop the trace while dumping if we are not opening "snapshot" */
3034 if (!iter->snapshot)
3035 tracing_stop_tr(tr);
3037 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3038 for_each_tracing_cpu(cpu) {
3039 iter->buffer_iter[cpu] =
3040 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3042 ring_buffer_read_prepare_sync();
3043 for_each_tracing_cpu(cpu) {
3044 ring_buffer_read_start(iter->buffer_iter[cpu]);
3045 tracing_iter_reset(iter, cpu);
3048 cpu = iter->cpu_file;
3049 iter->buffer_iter[cpu] =
3050 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3051 ring_buffer_read_prepare_sync();
3052 ring_buffer_read_start(iter->buffer_iter[cpu]);
3053 tracing_iter_reset(iter, cpu);
3056 mutex_unlock(&trace_types_lock);
3061 mutex_unlock(&trace_types_lock);
3063 kfree(iter->buffer_iter);
3065 seq_release_private(inode, file);
3066 return ERR_PTR(-ENOMEM);
3069 int tracing_open_generic(struct inode *inode, struct file *filp)
3071 if (tracing_disabled)
3074 filp->private_data = inode->i_private;
3078 bool tracing_is_disabled(void)
3080 return (tracing_disabled) ? true: false;
3084 * Open and update trace_array ref count.
3085 * Must have the current trace_array passed to it.
3087 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3089 struct trace_array *tr = inode->i_private;
3091 if (tracing_disabled)
3094 if (trace_array_get(tr) < 0)
3097 filp->private_data = inode->i_private;
3102 static int tracing_release(struct inode *inode, struct file *file)
3104 struct trace_array *tr = inode->i_private;
3105 struct seq_file *m = file->private_data;
3106 struct trace_iterator *iter;
3109 if (!(file->f_mode & FMODE_READ)) {
3110 trace_array_put(tr);
3114 /* Writes do not use seq_file */
3116 mutex_lock(&trace_types_lock);
3118 for_each_tracing_cpu(cpu) {
3119 if (iter->buffer_iter[cpu])
3120 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3123 if (iter->trace && iter->trace->close)
3124 iter->trace->close(iter);
3126 if (!iter->snapshot)
3127 /* reenable tracing if it was previously enabled */
3128 tracing_start_tr(tr);
3130 __trace_array_put(tr);
3132 mutex_unlock(&trace_types_lock);
3134 mutex_destroy(&iter->mutex);
3135 free_cpumask_var(iter->started);
3137 kfree(iter->buffer_iter);
3138 seq_release_private(inode, file);
3143 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3145 struct trace_array *tr = inode->i_private;
3147 trace_array_put(tr);
3151 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3153 struct trace_array *tr = inode->i_private;
3155 trace_array_put(tr);
3157 return single_release(inode, file);
3160 static int tracing_open(struct inode *inode, struct file *file)
3162 struct trace_array *tr = inode->i_private;
3163 struct trace_iterator *iter;
3166 if (trace_array_get(tr) < 0)
3169 /* If this file was open for write, then erase contents */
3170 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3171 int cpu = tracing_get_cpu(inode);
3173 if (cpu == RING_BUFFER_ALL_CPUS)
3174 tracing_reset_online_cpus(&tr->trace_buffer);
3176 tracing_reset(&tr->trace_buffer, cpu);
3179 if (file->f_mode & FMODE_READ) {
3180 iter = __tracing_open(inode, file, false);
3182 ret = PTR_ERR(iter);
3183 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3184 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3188 trace_array_put(tr);
3194 * Some tracers are not suitable for instance buffers.
3195 * A tracer is always available for the global array (toplevel)
3196 * or if it explicitly states that it is.
3199 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3201 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3204 /* Find the next tracer that this trace array may use */
3205 static struct tracer *
3206 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3208 while (t && !trace_ok_for_array(t, tr))
3215 t_next(struct seq_file *m, void *v, loff_t *pos)
3217 struct trace_array *tr = m->private;
3218 struct tracer *t = v;
3223 t = get_tracer_for_array(tr, t->next);
3228 static void *t_start(struct seq_file *m, loff_t *pos)
3230 struct trace_array *tr = m->private;
3234 mutex_lock(&trace_types_lock);
3236 t = get_tracer_for_array(tr, trace_types);
3237 for (; t && l < *pos; t = t_next(m, t, &l))
3243 static void t_stop(struct seq_file *m, void *p)
3245 mutex_unlock(&trace_types_lock);
3248 static int t_show(struct seq_file *m, void *v)
3250 struct tracer *t = v;
3255 seq_puts(m, t->name);
3264 static const struct seq_operations show_traces_seq_ops = {
3271 static int show_traces_open(struct inode *inode, struct file *file)
3273 struct trace_array *tr = inode->i_private;
3277 if (tracing_disabled)
3280 ret = seq_open(file, &show_traces_seq_ops);
3284 m = file->private_data;
3291 tracing_write_stub(struct file *filp, const char __user *ubuf,
3292 size_t count, loff_t *ppos)
3297 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3301 if (file->f_mode & FMODE_READ)
3302 ret = seq_lseek(file, offset, whence);
3304 file->f_pos = ret = 0;
3309 static const struct file_operations tracing_fops = {
3310 .open = tracing_open,
3312 .write = tracing_write_stub,
3313 .llseek = tracing_lseek,
3314 .release = tracing_release,
3317 static const struct file_operations show_traces_fops = {
3318 .open = show_traces_open,
3320 .release = seq_release,
3321 .llseek = seq_lseek,
3325 * The tracer itself will not take this lock, but still we want
3326 * to provide a consistent cpumask to user-space:
3328 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3331 * Temporary storage for the character representation of the
3332 * CPU bitmask (and one more byte for the newline):
3334 static char mask_str[NR_CPUS + 1];
3337 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3338 size_t count, loff_t *ppos)
3340 struct trace_array *tr = file_inode(filp)->i_private;
3343 mutex_lock(&tracing_cpumask_update_lock);
3345 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3346 if (count - len < 2) {
3350 len += sprintf(mask_str + len, "\n");
3351 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3354 mutex_unlock(&tracing_cpumask_update_lock);
3360 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3361 size_t count, loff_t *ppos)
3363 struct trace_array *tr = file_inode(filp)->i_private;
3364 cpumask_var_t tracing_cpumask_new;
3367 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3370 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3374 mutex_lock(&tracing_cpumask_update_lock);
3376 local_irq_disable();
3377 arch_spin_lock(&tr->max_lock);
3378 for_each_tracing_cpu(cpu) {
3380 * Increase/decrease the disabled counter if we are
3381 * about to flip a bit in the cpumask:
3383 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3384 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3385 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3386 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3388 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3389 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3390 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3391 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3394 arch_spin_unlock(&tr->max_lock);
3397 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3399 mutex_unlock(&tracing_cpumask_update_lock);
3400 free_cpumask_var(tracing_cpumask_new);
3405 free_cpumask_var(tracing_cpumask_new);
3410 static const struct file_operations tracing_cpumask_fops = {
3411 .open = tracing_open_generic_tr,
3412 .read = tracing_cpumask_read,
3413 .write = tracing_cpumask_write,
3414 .release = tracing_release_generic_tr,
3415 .llseek = generic_file_llseek,
3418 static int tracing_trace_options_show(struct seq_file *m, void *v)
3420 struct tracer_opt *trace_opts;
3421 struct trace_array *tr = m->private;
3425 mutex_lock(&trace_types_lock);
3426 tracer_flags = tr->current_trace->flags->val;
3427 trace_opts = tr->current_trace->flags->opts;
3429 for (i = 0; trace_options[i]; i++) {
3430 if (trace_flags & (1 << i))
3431 seq_printf(m, "%s\n", trace_options[i]);
3433 seq_printf(m, "no%s\n", trace_options[i]);
3436 for (i = 0; trace_opts[i].name; i++) {
3437 if (tracer_flags & trace_opts[i].bit)
3438 seq_printf(m, "%s\n", trace_opts[i].name);
3440 seq_printf(m, "no%s\n", trace_opts[i].name);
3442 mutex_unlock(&trace_types_lock);
3447 static int __set_tracer_option(struct trace_array *tr,
3448 struct tracer_flags *tracer_flags,
3449 struct tracer_opt *opts, int neg)
3451 struct tracer *trace = tr->current_trace;
3454 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3459 tracer_flags->val &= ~opts->bit;
3461 tracer_flags->val |= opts->bit;
3465 /* Try to assign a tracer specific option */
3466 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3468 struct tracer *trace = tr->current_trace;
3469 struct tracer_flags *tracer_flags = trace->flags;
3470 struct tracer_opt *opts = NULL;
3473 for (i = 0; tracer_flags->opts[i].name; i++) {
3474 opts = &tracer_flags->opts[i];
3476 if (strcmp(cmp, opts->name) == 0)
3477 return __set_tracer_option(tr, trace->flags, opts, neg);
3483 /* Some tracers require overwrite to stay enabled */
3484 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3486 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3492 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3494 /* do nothing if flag is already set */
3495 if (!!(trace_flags & mask) == !!enabled)
3498 /* Give the tracer a chance to approve the change */
3499 if (tr->current_trace->flag_changed)
3500 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3504 trace_flags |= mask;
3506 trace_flags &= ~mask;
3508 if (mask == TRACE_ITER_RECORD_CMD)
3509 trace_event_enable_cmd_record(enabled);
3511 if (mask == TRACE_ITER_OVERWRITE) {
3512 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3513 #ifdef CONFIG_TRACER_MAX_TRACE
3514 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3518 if (mask == TRACE_ITER_PRINTK)
3519 trace_printk_start_stop_comm(enabled);
3524 static int trace_set_options(struct trace_array *tr, char *option)
3531 cmp = strstrip(option);
3533 if (strncmp(cmp, "no", 2) == 0) {
3538 mutex_lock(&trace_types_lock);
3540 for (i = 0; trace_options[i]; i++) {
3541 if (strcmp(cmp, trace_options[i]) == 0) {
3542 ret = set_tracer_flag(tr, 1 << i, !neg);
3547 /* If no option could be set, test the specific tracer options */
3548 if (!trace_options[i])
3549 ret = set_tracer_option(tr, cmp, neg);
3551 mutex_unlock(&trace_types_lock);
3557 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3558 size_t cnt, loff_t *ppos)
3560 struct seq_file *m = filp->private_data;
3561 struct trace_array *tr = m->private;
3565 if (cnt >= sizeof(buf))
3568 if (copy_from_user(&buf, ubuf, cnt))
3573 ret = trace_set_options(tr, buf);
3582 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3584 struct trace_array *tr = inode->i_private;
3587 if (tracing_disabled)
3590 if (trace_array_get(tr) < 0)
3593 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3595 trace_array_put(tr);
3600 static const struct file_operations tracing_iter_fops = {
3601 .open = tracing_trace_options_open,
3603 .llseek = seq_lseek,
3604 .release = tracing_single_release_tr,
3605 .write = tracing_trace_options_write,
3608 static const char readme_msg[] =
3609 "tracing mini-HOWTO:\n\n"
3610 "# echo 0 > tracing_on : quick way to disable tracing\n"
3611 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3612 " Important files:\n"
3613 " trace\t\t\t- The static contents of the buffer\n"
3614 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3615 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3616 " current_tracer\t- function and latency tracers\n"
3617 " available_tracers\t- list of configured tracers for current_tracer\n"
3618 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3619 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3620 " trace_clock\t\t-change the clock used to order events\n"
3621 " local: Per cpu clock but may not be synced across CPUs\n"
3622 " global: Synced across CPUs but slows tracing down.\n"
3623 " counter: Not a clock, but just an increment\n"
3624 " uptime: Jiffy counter from time of boot\n"
3625 " perf: Same clock that perf events use\n"
3626 #ifdef CONFIG_X86_64
3627 " x86-tsc: TSC cycle counter\n"
3629 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3630 " tracing_cpumask\t- Limit which CPUs to trace\n"
3631 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3632 "\t\t\t Remove sub-buffer with rmdir\n"
3633 " trace_options\t\t- Set format or modify how tracing happens\n"
3634 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3635 "\t\t\t option name\n"
3636 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3637 #ifdef CONFIG_DYNAMIC_FTRACE
3638 "\n available_filter_functions - list of functions that can be filtered on\n"
3639 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3640 "\t\t\t functions\n"
3641 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3642 "\t modules: Can select a group via module\n"
3643 "\t Format: :mod:<module-name>\n"
3644 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3645 "\t triggers: a command to perform when function is hit\n"
3646 "\t Format: <function>:<trigger>[:count]\n"
3647 "\t trigger: traceon, traceoff\n"
3648 "\t\t enable_event:<system>:<event>\n"
3649 "\t\t disable_event:<system>:<event>\n"
3650 #ifdef CONFIG_STACKTRACE
3653 #ifdef CONFIG_TRACER_SNAPSHOT
3658 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3659 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3660 "\t The first one will disable tracing every time do_fault is hit\n"
3661 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3662 "\t The first time do trap is hit and it disables tracing, the\n"
3663 "\t counter will decrement to 2. If tracing is already disabled,\n"
3664 "\t the counter will not decrement. It only decrements when the\n"
3665 "\t trigger did work\n"
3666 "\t To remove trigger without count:\n"
3667 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3668 "\t To remove trigger with a count:\n"
3669 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3670 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3671 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3672 "\t modules: Can select a group via module command :mod:\n"
3673 "\t Does not accept triggers\n"
3674 #endif /* CONFIG_DYNAMIC_FTRACE */
3675 #ifdef CONFIG_FUNCTION_TRACER
3676 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3679 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3680 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3681 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3682 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3684 #ifdef CONFIG_TRACER_SNAPSHOT
3685 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3686 "\t\t\t snapshot buffer. Read the contents for more\n"
3687 "\t\t\t information\n"
3689 #ifdef CONFIG_STACK_TRACER
3690 " stack_trace\t\t- Shows the max stack trace when active\n"
3691 " stack_max_size\t- Shows current max stack size that was traced\n"
3692 "\t\t\t Write into this file to reset the max size (trigger a\n"
3693 "\t\t\t new trace)\n"
3694 #ifdef CONFIG_DYNAMIC_FTRACE
3695 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3698 #endif /* CONFIG_STACK_TRACER */
3699 " events/\t\t- Directory containing all trace event subsystems:\n"
3700 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3701 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3702 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3704 " filter\t\t- If set, only events passing filter are traced\n"
3705 " events/<system>/<event>/\t- Directory containing control files for\n"
3707 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3708 " filter\t\t- If set, only events passing filter are traced\n"
3709 " trigger\t\t- If set, a command to perform when event is hit\n"
3710 "\t Format: <trigger>[:count][if <filter>]\n"
3711 "\t trigger: traceon, traceoff\n"
3712 "\t enable_event:<system>:<event>\n"
3713 "\t disable_event:<system>:<event>\n"
3714 #ifdef CONFIG_STACKTRACE
3717 #ifdef CONFIG_TRACER_SNAPSHOT
3720 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3721 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3722 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3723 "\t events/block/block_unplug/trigger\n"
3724 "\t The first disables tracing every time block_unplug is hit.\n"
3725 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3726 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3727 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3728 "\t Like function triggers, the counter is only decremented if it\n"
3729 "\t enabled or disabled tracing.\n"
3730 "\t To remove a trigger without a count:\n"
3731 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3732 "\t To remove a trigger with a count:\n"
3733 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3734 "\t Filters can be ignored when removing a trigger.\n"
3738 tracing_readme_read(struct file *filp, char __user *ubuf,
3739 size_t cnt, loff_t *ppos)
3741 return simple_read_from_buffer(ubuf, cnt, ppos,
3742 readme_msg, strlen(readme_msg));
3745 static const struct file_operations tracing_readme_fops = {
3746 .open = tracing_open_generic,
3747 .read = tracing_readme_read,
3748 .llseek = generic_file_llseek,
3751 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3753 unsigned int *ptr = v;
3755 if (*pos || m->count)
3760 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3762 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3771 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3777 arch_spin_lock(&trace_cmdline_lock);
3779 v = &savedcmd->map_cmdline_to_pid[0];
3781 v = saved_cmdlines_next(m, v, &l);
3789 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3791 arch_spin_unlock(&trace_cmdline_lock);
3795 static int saved_cmdlines_show(struct seq_file *m, void *v)
3797 char buf[TASK_COMM_LEN];
3798 unsigned int *pid = v;
3800 __trace_find_cmdline(*pid, buf);
3801 seq_printf(m, "%d %s\n", *pid, buf);
3805 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3806 .start = saved_cmdlines_start,
3807 .next = saved_cmdlines_next,
3808 .stop = saved_cmdlines_stop,
3809 .show = saved_cmdlines_show,
3812 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3814 if (tracing_disabled)
3817 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3820 static const struct file_operations tracing_saved_cmdlines_fops = {
3821 .open = tracing_saved_cmdlines_open,
3823 .llseek = seq_lseek,
3824 .release = seq_release,
3828 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3829 size_t cnt, loff_t *ppos)
3834 arch_spin_lock(&trace_cmdline_lock);
3835 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3836 arch_spin_unlock(&trace_cmdline_lock);
3838 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3841 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3843 kfree(s->saved_cmdlines);
3844 kfree(s->map_cmdline_to_pid);
3848 static int tracing_resize_saved_cmdlines(unsigned int val)
3850 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3852 s = kmalloc(sizeof(*s), GFP_KERNEL);
3856 if (allocate_cmdlines_buffer(val, s) < 0) {
3861 arch_spin_lock(&trace_cmdline_lock);
3862 savedcmd_temp = savedcmd;
3864 arch_spin_unlock(&trace_cmdline_lock);
3865 free_saved_cmdlines_buffer(savedcmd_temp);
3871 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3872 size_t cnt, loff_t *ppos)
3877 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3881 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3882 if (!val || val > PID_MAX_DEFAULT)
3885 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3894 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3895 .open = tracing_open_generic,
3896 .read = tracing_saved_cmdlines_size_read,
3897 .write = tracing_saved_cmdlines_size_write,
3901 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3902 size_t cnt, loff_t *ppos)
3904 struct trace_array *tr = filp->private_data;
3905 char buf[MAX_TRACER_SIZE+2];
3908 mutex_lock(&trace_types_lock);
3909 r = sprintf(buf, "%s\n", tr->current_trace->name);
3910 mutex_unlock(&trace_types_lock);
3912 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3915 int tracer_init(struct tracer *t, struct trace_array *tr)
3917 tracing_reset_online_cpus(&tr->trace_buffer);
3921 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3925 for_each_tracing_cpu(cpu)
3926 per_cpu_ptr(buf->data, cpu)->entries = val;
3929 #ifdef CONFIG_TRACER_MAX_TRACE
3930 /* resize @tr's buffer to the size of @size_tr's entries */
3931 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3932 struct trace_buffer *size_buf, int cpu_id)
3936 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3937 for_each_tracing_cpu(cpu) {
3938 ret = ring_buffer_resize(trace_buf->buffer,
3939 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3942 per_cpu_ptr(trace_buf->data, cpu)->entries =
3943 per_cpu_ptr(size_buf->data, cpu)->entries;
3946 ret = ring_buffer_resize(trace_buf->buffer,
3947 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3949 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3950 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3955 #endif /* CONFIG_TRACER_MAX_TRACE */
3957 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3958 unsigned long size, int cpu)
3963 * If kernel or user changes the size of the ring buffer
3964 * we use the size that was given, and we can forget about
3965 * expanding it later.
3967 ring_buffer_expanded = true;
3969 /* May be called before buffers are initialized */
3970 if (!tr->trace_buffer.buffer)
3973 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3977 #ifdef CONFIG_TRACER_MAX_TRACE
3978 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3979 !tr->current_trace->use_max_tr)
3982 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3984 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3985 &tr->trace_buffer, cpu);
3988 * AARGH! We are left with different
3989 * size max buffer!!!!
3990 * The max buffer is our "snapshot" buffer.
3991 * When a tracer needs a snapshot (one of the
3992 * latency tracers), it swaps the max buffer
3993 * with the saved snap shot. We succeeded to
3994 * update the size of the main buffer, but failed to
3995 * update the size of the max buffer. But when we tried
3996 * to reset the main buffer to the original size, we
3997 * failed there too. This is very unlikely to
3998 * happen, but if it does, warn and kill all
4002 tracing_disabled = 1;
4007 if (cpu == RING_BUFFER_ALL_CPUS)
4008 set_buffer_entries(&tr->max_buffer, size);
4010 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4013 #endif /* CONFIG_TRACER_MAX_TRACE */
4015 if (cpu == RING_BUFFER_ALL_CPUS)
4016 set_buffer_entries(&tr->trace_buffer, size);
4018 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4023 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4024 unsigned long size, int cpu_id)
4028 mutex_lock(&trace_types_lock);
4030 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4031 /* make sure, this cpu is enabled in the mask */
4032 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4038 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4043 mutex_unlock(&trace_types_lock);
4050 * tracing_update_buffers - used by tracing facility to expand ring buffers
4052 * To save on memory when the tracing is never used on a system with it
4053 * configured in. The ring buffers are set to a minimum size. But once
4054 * a user starts to use the tracing facility, then they need to grow
4055 * to their default size.
4057 * This function is to be called when a tracer is about to be used.
4059 int tracing_update_buffers(void)
4063 mutex_lock(&trace_types_lock);
4064 if (!ring_buffer_expanded)
4065 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4066 RING_BUFFER_ALL_CPUS);
4067 mutex_unlock(&trace_types_lock);
4072 struct trace_option_dentry;
4074 static struct trace_option_dentry *
4075 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4078 destroy_trace_option_files(struct trace_option_dentry *topts);
4081 * Used to clear out the tracer before deletion of an instance.
4082 * Must have trace_types_lock held.
4084 static void tracing_set_nop(struct trace_array *tr)
4086 if (tr->current_trace == &nop_trace)
4089 tr->current_trace->enabled--;
4091 if (tr->current_trace->reset)
4092 tr->current_trace->reset(tr);
4094 tr->current_trace = &nop_trace;
4097 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4099 static struct trace_option_dentry *topts;
4101 #ifdef CONFIG_TRACER_MAX_TRACE
4106 mutex_lock(&trace_types_lock);
4108 if (!ring_buffer_expanded) {
4109 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4110 RING_BUFFER_ALL_CPUS);
4116 for (t = trace_types; t; t = t->next) {
4117 if (strcmp(t->name, buf) == 0)
4124 if (t == tr->current_trace)
4127 /* Some tracers are only allowed for the top level buffer */
4128 if (!trace_ok_for_array(t, tr)) {
4133 trace_branch_disable();
4135 tr->current_trace->enabled--;
4137 if (tr->current_trace->reset)
4138 tr->current_trace->reset(tr);
4140 /* Current trace needs to be nop_trace before synchronize_sched */
4141 tr->current_trace = &nop_trace;
4143 #ifdef CONFIG_TRACER_MAX_TRACE
4144 had_max_tr = tr->allocated_snapshot;
4146 if (had_max_tr && !t->use_max_tr) {
4148 * We need to make sure that the update_max_tr sees that
4149 * current_trace changed to nop_trace to keep it from
4150 * swapping the buffers after we resize it.
4151 * The update_max_tr is called from interrupts disabled
4152 * so a synchronized_sched() is sufficient.
4154 synchronize_sched();
4158 /* Currently, only the top instance has options */
4159 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4160 destroy_trace_option_files(topts);
4161 topts = create_trace_option_files(tr, t);
4164 #ifdef CONFIG_TRACER_MAX_TRACE
4165 if (t->use_max_tr && !had_max_tr) {
4166 ret = alloc_snapshot(tr);
4173 ret = tracer_init(t, tr);
4178 tr->current_trace = t;
4179 tr->current_trace->enabled++;
4180 trace_branch_enable(tr);
4182 mutex_unlock(&trace_types_lock);
4188 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4189 size_t cnt, loff_t *ppos)
4191 struct trace_array *tr = filp->private_data;
4192 char buf[MAX_TRACER_SIZE+1];
4199 if (cnt > MAX_TRACER_SIZE)
4200 cnt = MAX_TRACER_SIZE;
4202 if (copy_from_user(&buf, ubuf, cnt))
4207 /* strip ending whitespace. */
4208 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4211 err = tracing_set_tracer(tr, buf);
4221 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4222 size_t cnt, loff_t *ppos)
4227 r = snprintf(buf, sizeof(buf), "%ld\n",
4228 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4229 if (r > sizeof(buf))
4231 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4235 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4236 size_t cnt, loff_t *ppos)
4241 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4251 tracing_thresh_read(struct file *filp, char __user *ubuf,
4252 size_t cnt, loff_t *ppos)
4254 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4258 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4259 size_t cnt, loff_t *ppos)
4261 struct trace_array *tr = filp->private_data;
4264 mutex_lock(&trace_types_lock);
4265 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4269 if (tr->current_trace->update_thresh) {
4270 ret = tr->current_trace->update_thresh(tr);
4277 mutex_unlock(&trace_types_lock);
4283 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4284 size_t cnt, loff_t *ppos)
4286 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4290 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4291 size_t cnt, loff_t *ppos)
4293 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4296 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4298 struct trace_array *tr = inode->i_private;
4299 struct trace_iterator *iter;
4302 if (tracing_disabled)
4305 if (trace_array_get(tr) < 0)
4308 mutex_lock(&trace_types_lock);
4310 /* create a buffer to store the information to pass to userspace */
4311 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4314 __trace_array_put(tr);
4319 * We make a copy of the current tracer to avoid concurrent
4320 * changes on it while we are reading.
4322 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4327 *iter->trace = *tr->current_trace;
4329 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4334 /* trace pipe does not show start of buffer */
4335 cpumask_setall(iter->started);
4337 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4338 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4340 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4341 if (trace_clocks[tr->clock_id].in_ns)
4342 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4345 iter->trace_buffer = &tr->trace_buffer;
4346 iter->cpu_file = tracing_get_cpu(inode);
4347 mutex_init(&iter->mutex);
4348 filp->private_data = iter;
4350 if (iter->trace->pipe_open)
4351 iter->trace->pipe_open(iter);
4353 nonseekable_open(inode, filp);
4355 mutex_unlock(&trace_types_lock);
4361 __trace_array_put(tr);
4362 mutex_unlock(&trace_types_lock);
4366 static int tracing_release_pipe(struct inode *inode, struct file *file)
4368 struct trace_iterator *iter = file->private_data;
4369 struct trace_array *tr = inode->i_private;
4371 mutex_lock(&trace_types_lock);
4373 if (iter->trace->pipe_close)
4374 iter->trace->pipe_close(iter);
4376 mutex_unlock(&trace_types_lock);
4378 free_cpumask_var(iter->started);
4379 mutex_destroy(&iter->mutex);
4383 trace_array_put(tr);
4389 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4391 /* Iterators are static, they should be filled or empty */
4392 if (trace_buffer_iter(iter, iter->cpu_file))
4393 return POLLIN | POLLRDNORM;
4395 if (trace_flags & TRACE_ITER_BLOCK)
4397 * Always select as readable when in blocking mode
4399 return POLLIN | POLLRDNORM;
4401 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4406 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4408 struct trace_iterator *iter = filp->private_data;
4410 return trace_poll(iter, filp, poll_table);
4413 /* Must be called with trace_types_lock mutex held. */
4414 static int tracing_wait_pipe(struct file *filp)
4416 struct trace_iterator *iter = filp->private_data;
4419 while (trace_empty(iter)) {
4421 if ((filp->f_flags & O_NONBLOCK)) {
4426 * We block until we read something and tracing is disabled.
4427 * We still block if tracing is disabled, but we have never
4428 * read anything. This allows a user to cat this file, and
4429 * then enable tracing. But after we have read something,
4430 * we give an EOF when tracing is again disabled.
4432 * iter->pos will be 0 if we haven't read anything.
4434 if (!tracing_is_on() && iter->pos)
4437 mutex_unlock(&iter->mutex);
4439 ret = wait_on_pipe(iter);
4441 mutex_lock(&iter->mutex);
4446 if (signal_pending(current))
4457 tracing_read_pipe(struct file *filp, char __user *ubuf,
4458 size_t cnt, loff_t *ppos)
4460 struct trace_iterator *iter = filp->private_data;
4461 struct trace_array *tr = iter->tr;
4464 /* return any leftover data */
4465 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4469 trace_seq_init(&iter->seq);
4471 /* copy the tracer to avoid using a global lock all around */
4472 mutex_lock(&trace_types_lock);
4473 if (unlikely(iter->trace->name != tr->current_trace->name))
4474 *iter->trace = *tr->current_trace;
4475 mutex_unlock(&trace_types_lock);
4478 * Avoid more than one consumer on a single file descriptor
4479 * This is just a matter of traces coherency, the ring buffer itself
4482 mutex_lock(&iter->mutex);
4483 if (iter->trace->read) {
4484 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4490 sret = tracing_wait_pipe(filp);
4494 /* stop when tracing is finished */
4495 if (trace_empty(iter)) {
4500 if (cnt >= PAGE_SIZE)
4501 cnt = PAGE_SIZE - 1;
4503 /* reset all but tr, trace, and overruns */
4504 memset(&iter->seq, 0,
4505 sizeof(struct trace_iterator) -
4506 offsetof(struct trace_iterator, seq));
4507 cpumask_clear(iter->started);
4510 trace_event_read_lock();
4511 trace_access_lock(iter->cpu_file);
4512 while (trace_find_next_entry_inc(iter) != NULL) {
4513 enum print_line_t ret;
4514 int len = iter->seq.len;
4516 ret = print_trace_line(iter);
4517 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4518 /* don't print partial lines */
4519 iter->seq.len = len;
4522 if (ret != TRACE_TYPE_NO_CONSUME)
4523 trace_consume(iter);
4525 if (iter->seq.len >= cnt)
4529 * Setting the full flag means we reached the trace_seq buffer
4530 * size and we should leave by partial output condition above.
4531 * One of the trace_seq_* functions is not used properly.
4533 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4536 trace_access_unlock(iter->cpu_file);
4537 trace_event_read_unlock();
4539 /* Now copy what we have to the user */
4540 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4541 if (iter->seq.readpos >= iter->seq.len)
4542 trace_seq_init(&iter->seq);
4545 * If there was nothing to send to user, in spite of consuming trace
4546 * entries, go back to wait for more entries.
4552 mutex_unlock(&iter->mutex);
4557 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4560 __free_page(spd->pages[idx]);
4563 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4565 .confirm = generic_pipe_buf_confirm,
4566 .release = generic_pipe_buf_release,
4567 .steal = generic_pipe_buf_steal,
4568 .get = generic_pipe_buf_get,
4572 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4577 /* Seq buffer is page-sized, exactly what we need. */
4579 count = iter->seq.len;
4580 ret = print_trace_line(iter);
4581 count = iter->seq.len - count;
4584 iter->seq.len -= count;
4587 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4588 iter->seq.len -= count;
4592 if (ret != TRACE_TYPE_NO_CONSUME)
4593 trace_consume(iter);
4595 if (!trace_find_next_entry_inc(iter)) {
4605 static ssize_t tracing_splice_read_pipe(struct file *filp,
4607 struct pipe_inode_info *pipe,
4611 struct page *pages_def[PIPE_DEF_BUFFERS];
4612 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4613 struct trace_iterator *iter = filp->private_data;
4614 struct splice_pipe_desc spd = {
4616 .partial = partial_def,
4617 .nr_pages = 0, /* This gets updated below. */
4618 .nr_pages_max = PIPE_DEF_BUFFERS,
4620 .ops = &tracing_pipe_buf_ops,
4621 .spd_release = tracing_spd_release_pipe,
4623 struct trace_array *tr = iter->tr;
4628 if (splice_grow_spd(pipe, &spd))
4631 /* copy the tracer to avoid using a global lock all around */
4632 mutex_lock(&trace_types_lock);
4633 if (unlikely(iter->trace->name != tr->current_trace->name))
4634 *iter->trace = *tr->current_trace;
4635 mutex_unlock(&trace_types_lock);
4637 mutex_lock(&iter->mutex);
4639 if (iter->trace->splice_read) {
4640 ret = iter->trace->splice_read(iter, filp,
4641 ppos, pipe, len, flags);
4646 ret = tracing_wait_pipe(filp);
4650 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4655 trace_event_read_lock();
4656 trace_access_lock(iter->cpu_file);
4658 /* Fill as many pages as possible. */
4659 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4660 spd.pages[i] = alloc_page(GFP_KERNEL);
4664 rem = tracing_fill_pipe_page(rem, iter);
4666 /* Copy the data into the page, so we can start over. */
4667 ret = trace_seq_to_buffer(&iter->seq,
4668 page_address(spd.pages[i]),
4671 __free_page(spd.pages[i]);
4674 spd.partial[i].offset = 0;
4675 spd.partial[i].len = iter->seq.len;
4677 trace_seq_init(&iter->seq);
4680 trace_access_unlock(iter->cpu_file);
4681 trace_event_read_unlock();
4682 mutex_unlock(&iter->mutex);
4686 ret = splice_to_pipe(pipe, &spd);
4688 splice_shrink_spd(&spd);
4692 mutex_unlock(&iter->mutex);
4697 tracing_entries_read(struct file *filp, char __user *ubuf,
4698 size_t cnt, loff_t *ppos)
4700 struct inode *inode = file_inode(filp);
4701 struct trace_array *tr = inode->i_private;
4702 int cpu = tracing_get_cpu(inode);
4707 mutex_lock(&trace_types_lock);
4709 if (cpu == RING_BUFFER_ALL_CPUS) {
4710 int cpu, buf_size_same;
4715 /* check if all cpu sizes are same */
4716 for_each_tracing_cpu(cpu) {
4717 /* fill in the size from first enabled cpu */
4719 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4720 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4726 if (buf_size_same) {
4727 if (!ring_buffer_expanded)
4728 r = sprintf(buf, "%lu (expanded: %lu)\n",
4730 trace_buf_size >> 10);
4732 r = sprintf(buf, "%lu\n", size >> 10);
4734 r = sprintf(buf, "X\n");
4736 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4738 mutex_unlock(&trace_types_lock);
4740 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4745 tracing_entries_write(struct file *filp, const char __user *ubuf,
4746 size_t cnt, loff_t *ppos)
4748 struct inode *inode = file_inode(filp);
4749 struct trace_array *tr = inode->i_private;
4753 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4757 /* must have at least 1 entry */
4761 /* value is in KB */
4763 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4773 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4774 size_t cnt, loff_t *ppos)
4776 struct trace_array *tr = filp->private_data;
4779 unsigned long size = 0, expanded_size = 0;
4781 mutex_lock(&trace_types_lock);
4782 for_each_tracing_cpu(cpu) {
4783 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4784 if (!ring_buffer_expanded)
4785 expanded_size += trace_buf_size >> 10;
4787 if (ring_buffer_expanded)
4788 r = sprintf(buf, "%lu\n", size);
4790 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4791 mutex_unlock(&trace_types_lock);
4793 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4797 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4798 size_t cnt, loff_t *ppos)
4801 * There is no need to read what the user has written, this function
4802 * is just to make sure that there is no error when "echo" is used
4811 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4813 struct trace_array *tr = inode->i_private;
4815 /* disable tracing ? */
4816 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4817 tracer_tracing_off(tr);
4818 /* resize the ring buffer to 0 */
4819 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4821 trace_array_put(tr);
4827 tracing_mark_write(struct file *filp, const char __user *ubuf,
4828 size_t cnt, loff_t *fpos)
4830 unsigned long addr = (unsigned long)ubuf;
4831 struct trace_array *tr = filp->private_data;
4832 struct ring_buffer_event *event;
4833 struct ring_buffer *buffer;
4834 struct print_entry *entry;
4835 unsigned long irq_flags;
4836 struct page *pages[2];
4846 if (tracing_disabled)
4849 if (!(trace_flags & TRACE_ITER_MARKERS))
4852 if (cnt > TRACE_BUF_SIZE)
4853 cnt = TRACE_BUF_SIZE;
4856 * Userspace is injecting traces into the kernel trace buffer.
4857 * We want to be as non intrusive as possible.
4858 * To do so, we do not want to allocate any special buffers
4859 * or take any locks, but instead write the userspace data
4860 * straight into the ring buffer.
4862 * First we need to pin the userspace buffer into memory,
4863 * which, most likely it is, because it just referenced it.
4864 * But there's no guarantee that it is. By using get_user_pages_fast()
4865 * and kmap_atomic/kunmap_atomic() we can get access to the
4866 * pages directly. We then write the data directly into the
4869 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4871 /* check if we cross pages */
4872 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4875 offset = addr & (PAGE_SIZE - 1);
4878 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4879 if (ret < nr_pages) {
4881 put_page(pages[ret]);
4886 for (i = 0; i < nr_pages; i++)
4887 map_page[i] = kmap_atomic(pages[i]);
4889 local_save_flags(irq_flags);
4890 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4891 buffer = tr->trace_buffer.buffer;
4892 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4893 irq_flags, preempt_count());
4895 /* Ring buffer disabled, return as if not open for write */
4900 entry = ring_buffer_event_data(event);
4901 entry->ip = _THIS_IP_;
4903 if (nr_pages == 2) {
4904 len = PAGE_SIZE - offset;
4905 memcpy(&entry->buf, map_page[0] + offset, len);
4906 memcpy(&entry->buf[len], map_page[1], cnt - len);
4908 memcpy(&entry->buf, map_page[0] + offset, cnt);
4910 if (entry->buf[cnt - 1] != '\n') {
4911 entry->buf[cnt] = '\n';
4912 entry->buf[cnt + 1] = '\0';
4914 entry->buf[cnt] = '\0';
4916 __buffer_unlock_commit(buffer, event);
4923 for (i = 0; i < nr_pages; i++){
4924 kunmap_atomic(map_page[i]);
4931 static int tracing_clock_show(struct seq_file *m, void *v)
4933 struct trace_array *tr = m->private;
4936 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4938 "%s%s%s%s", i ? " " : "",
4939 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4940 i == tr->clock_id ? "]" : "");
4946 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4950 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4951 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4954 if (i == ARRAY_SIZE(trace_clocks))
4957 mutex_lock(&trace_types_lock);
4961 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4964 * New clock may not be consistent with the previous clock.
4965 * Reset the buffer so that it doesn't have incomparable timestamps.
4967 tracing_reset_online_cpus(&tr->trace_buffer);
4969 #ifdef CONFIG_TRACER_MAX_TRACE
4970 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4971 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4972 tracing_reset_online_cpus(&tr->max_buffer);
4975 mutex_unlock(&trace_types_lock);
4980 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4981 size_t cnt, loff_t *fpos)
4983 struct seq_file *m = filp->private_data;
4984 struct trace_array *tr = m->private;
4986 const char *clockstr;
4989 if (cnt >= sizeof(buf))
4992 if (copy_from_user(&buf, ubuf, cnt))
4997 clockstr = strstrip(buf);
4999 ret = tracing_set_clock(tr, clockstr);
5008 static int tracing_clock_open(struct inode *inode, struct file *file)
5010 struct trace_array *tr = inode->i_private;
5013 if (tracing_disabled)
5016 if (trace_array_get(tr))
5019 ret = single_open(file, tracing_clock_show, inode->i_private);
5021 trace_array_put(tr);
5026 struct ftrace_buffer_info {
5027 struct trace_iterator iter;
5032 #ifdef CONFIG_TRACER_SNAPSHOT
5033 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5035 struct trace_array *tr = inode->i_private;
5036 struct trace_iterator *iter;
5040 if (trace_array_get(tr) < 0)
5043 if (file->f_mode & FMODE_READ) {
5044 iter = __tracing_open(inode, file, true);
5046 ret = PTR_ERR(iter);
5048 /* Writes still need the seq_file to hold the private data */
5050 m = kzalloc(sizeof(*m), GFP_KERNEL);
5053 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5061 iter->trace_buffer = &tr->max_buffer;
5062 iter->cpu_file = tracing_get_cpu(inode);
5064 file->private_data = m;
5068 trace_array_put(tr);
5074 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5077 struct seq_file *m = filp->private_data;
5078 struct trace_iterator *iter = m->private;
5079 struct trace_array *tr = iter->tr;
5083 ret = tracing_update_buffers();
5087 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5091 mutex_lock(&trace_types_lock);
5093 if (tr->current_trace->use_max_tr) {
5100 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5104 if (tr->allocated_snapshot)
5108 /* Only allow per-cpu swap if the ring buffer supports it */
5109 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5110 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5115 if (!tr->allocated_snapshot) {
5116 ret = alloc_snapshot(tr);
5120 local_irq_disable();
5121 /* Now, we're going to swap */
5122 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5123 update_max_tr(tr, current, smp_processor_id());
5125 update_max_tr_single(tr, current, iter->cpu_file);
5129 if (tr->allocated_snapshot) {
5130 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5131 tracing_reset_online_cpus(&tr->max_buffer);
5133 tracing_reset(&tr->max_buffer, iter->cpu_file);
5143 mutex_unlock(&trace_types_lock);
5147 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5149 struct seq_file *m = file->private_data;
5152 ret = tracing_release(inode, file);
5154 if (file->f_mode & FMODE_READ)
5157 /* If write only, the seq_file is just a stub */
5165 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5166 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5167 size_t count, loff_t *ppos);
5168 static int tracing_buffers_release(struct inode *inode, struct file *file);
5169 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5170 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5172 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5174 struct ftrace_buffer_info *info;
5177 ret = tracing_buffers_open(inode, filp);
5181 info = filp->private_data;
5183 if (info->iter.trace->use_max_tr) {
5184 tracing_buffers_release(inode, filp);
5188 info->iter.snapshot = true;
5189 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5194 #endif /* CONFIG_TRACER_SNAPSHOT */
5197 static const struct file_operations tracing_thresh_fops = {
5198 .open = tracing_open_generic,
5199 .read = tracing_thresh_read,
5200 .write = tracing_thresh_write,
5201 .llseek = generic_file_llseek,
5204 static const struct file_operations tracing_max_lat_fops = {
5205 .open = tracing_open_generic,
5206 .read = tracing_max_lat_read,
5207 .write = tracing_max_lat_write,
5208 .llseek = generic_file_llseek,
5211 static const struct file_operations set_tracer_fops = {
5212 .open = tracing_open_generic,
5213 .read = tracing_set_trace_read,
5214 .write = tracing_set_trace_write,
5215 .llseek = generic_file_llseek,
5218 static const struct file_operations tracing_pipe_fops = {
5219 .open = tracing_open_pipe,
5220 .poll = tracing_poll_pipe,
5221 .read = tracing_read_pipe,
5222 .splice_read = tracing_splice_read_pipe,
5223 .release = tracing_release_pipe,
5224 .llseek = no_llseek,
5227 static const struct file_operations tracing_entries_fops = {
5228 .open = tracing_open_generic_tr,
5229 .read = tracing_entries_read,
5230 .write = tracing_entries_write,
5231 .llseek = generic_file_llseek,
5232 .release = tracing_release_generic_tr,
5235 static const struct file_operations tracing_total_entries_fops = {
5236 .open = tracing_open_generic_tr,
5237 .read = tracing_total_entries_read,
5238 .llseek = generic_file_llseek,
5239 .release = tracing_release_generic_tr,
5242 static const struct file_operations tracing_free_buffer_fops = {
5243 .open = tracing_open_generic_tr,
5244 .write = tracing_free_buffer_write,
5245 .release = tracing_free_buffer_release,
5248 static const struct file_operations tracing_mark_fops = {
5249 .open = tracing_open_generic_tr,
5250 .write = tracing_mark_write,
5251 .llseek = generic_file_llseek,
5252 .release = tracing_release_generic_tr,
5255 static const struct file_operations trace_clock_fops = {
5256 .open = tracing_clock_open,
5258 .llseek = seq_lseek,
5259 .release = tracing_single_release_tr,
5260 .write = tracing_clock_write,
5263 #ifdef CONFIG_TRACER_SNAPSHOT
5264 static const struct file_operations snapshot_fops = {
5265 .open = tracing_snapshot_open,
5267 .write = tracing_snapshot_write,
5268 .llseek = tracing_lseek,
5269 .release = tracing_snapshot_release,
5272 static const struct file_operations snapshot_raw_fops = {
5273 .open = snapshot_raw_open,
5274 .read = tracing_buffers_read,
5275 .release = tracing_buffers_release,
5276 .splice_read = tracing_buffers_splice_read,
5277 .llseek = no_llseek,
5280 #endif /* CONFIG_TRACER_SNAPSHOT */
5282 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5284 struct trace_array *tr = inode->i_private;
5285 struct ftrace_buffer_info *info;
5288 if (tracing_disabled)
5291 if (trace_array_get(tr) < 0)
5294 info = kzalloc(sizeof(*info), GFP_KERNEL);
5296 trace_array_put(tr);
5300 mutex_lock(&trace_types_lock);
5303 info->iter.cpu_file = tracing_get_cpu(inode);
5304 info->iter.trace = tr->current_trace;
5305 info->iter.trace_buffer = &tr->trace_buffer;
5307 /* Force reading ring buffer for first read */
5308 info->read = (unsigned int)-1;
5310 filp->private_data = info;
5312 mutex_unlock(&trace_types_lock);
5314 ret = nonseekable_open(inode, filp);
5316 trace_array_put(tr);
5322 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5324 struct ftrace_buffer_info *info = filp->private_data;
5325 struct trace_iterator *iter = &info->iter;
5327 return trace_poll(iter, filp, poll_table);
5331 tracing_buffers_read(struct file *filp, char __user *ubuf,
5332 size_t count, loff_t *ppos)
5334 struct ftrace_buffer_info *info = filp->private_data;
5335 struct trace_iterator *iter = &info->iter;
5342 mutex_lock(&trace_types_lock);
5344 #ifdef CONFIG_TRACER_MAX_TRACE
5345 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5352 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5358 /* Do we have previous read data to read? */
5359 if (info->read < PAGE_SIZE)
5363 trace_access_lock(iter->cpu_file);
5364 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5368 trace_access_unlock(iter->cpu_file);
5371 if (trace_empty(iter)) {
5372 if ((filp->f_flags & O_NONBLOCK)) {
5376 mutex_unlock(&trace_types_lock);
5377 ret = wait_on_pipe(iter);
5378 mutex_lock(&trace_types_lock);
5383 if (signal_pending(current)) {
5395 size = PAGE_SIZE - info->read;
5399 ret = copy_to_user(ubuf, info->spare + info->read, size);
5410 mutex_unlock(&trace_types_lock);
5415 static int tracing_buffers_release(struct inode *inode, struct file *file)
5417 struct ftrace_buffer_info *info = file->private_data;
5418 struct trace_iterator *iter = &info->iter;
5420 mutex_lock(&trace_types_lock);
5422 __trace_array_put(iter->tr);
5425 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5428 mutex_unlock(&trace_types_lock);
5434 struct ring_buffer *buffer;
5439 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5440 struct pipe_buffer *buf)
5442 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5447 ring_buffer_free_read_page(ref->buffer, ref->page);
5452 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5453 struct pipe_buffer *buf)
5455 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5460 /* Pipe buffer operations for a buffer. */
5461 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5463 .confirm = generic_pipe_buf_confirm,
5464 .release = buffer_pipe_buf_release,
5465 .steal = generic_pipe_buf_steal,
5466 .get = buffer_pipe_buf_get,
5470 * Callback from splice_to_pipe(), if we need to release some pages
5471 * at the end of the spd in case we error'ed out in filling the pipe.
5473 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5475 struct buffer_ref *ref =
5476 (struct buffer_ref *)spd->partial[i].private;
5481 ring_buffer_free_read_page(ref->buffer, ref->page);
5483 spd->partial[i].private = 0;
5487 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5488 struct pipe_inode_info *pipe, size_t len,
5491 struct ftrace_buffer_info *info = file->private_data;
5492 struct trace_iterator *iter = &info->iter;
5493 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5494 struct page *pages_def[PIPE_DEF_BUFFERS];
5495 struct splice_pipe_desc spd = {
5497 .partial = partial_def,
5498 .nr_pages_max = PIPE_DEF_BUFFERS,
5500 .ops = &buffer_pipe_buf_ops,
5501 .spd_release = buffer_spd_release,
5503 struct buffer_ref *ref;
5504 int entries, size, i;
5507 mutex_lock(&trace_types_lock);
5509 #ifdef CONFIG_TRACER_MAX_TRACE
5510 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5516 if (splice_grow_spd(pipe, &spd)) {
5521 if (*ppos & (PAGE_SIZE - 1)) {
5526 if (len & (PAGE_SIZE - 1)) {
5527 if (len < PAGE_SIZE) {
5535 trace_access_lock(iter->cpu_file);
5536 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5538 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5542 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5547 ref->buffer = iter->trace_buffer->buffer;
5548 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5554 r = ring_buffer_read_page(ref->buffer, &ref->page,
5555 len, iter->cpu_file, 1);
5557 ring_buffer_free_read_page(ref->buffer, ref->page);
5563 * zero out any left over data, this is going to
5566 size = ring_buffer_page_len(ref->page);
5567 if (size < PAGE_SIZE)
5568 memset(ref->page + size, 0, PAGE_SIZE - size);
5570 page = virt_to_page(ref->page);
5572 spd.pages[i] = page;
5573 spd.partial[i].len = PAGE_SIZE;
5574 spd.partial[i].offset = 0;
5575 spd.partial[i].private = (unsigned long)ref;
5579 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5582 trace_access_unlock(iter->cpu_file);
5585 /* did we read anything? */
5586 if (!spd.nr_pages) {
5587 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5591 mutex_unlock(&trace_types_lock);
5592 ret = wait_on_pipe(iter);
5593 mutex_lock(&trace_types_lock);
5596 if (signal_pending(current)) {
5603 ret = splice_to_pipe(pipe, &spd);
5604 splice_shrink_spd(&spd);
5606 mutex_unlock(&trace_types_lock);
5611 static const struct file_operations tracing_buffers_fops = {
5612 .open = tracing_buffers_open,
5613 .read = tracing_buffers_read,
5614 .poll = tracing_buffers_poll,
5615 .release = tracing_buffers_release,
5616 .splice_read = tracing_buffers_splice_read,
5617 .llseek = no_llseek,
5621 tracing_stats_read(struct file *filp, char __user *ubuf,
5622 size_t count, loff_t *ppos)
5624 struct inode *inode = file_inode(filp);
5625 struct trace_array *tr = inode->i_private;
5626 struct trace_buffer *trace_buf = &tr->trace_buffer;
5627 int cpu = tracing_get_cpu(inode);
5628 struct trace_seq *s;
5630 unsigned long long t;
5631 unsigned long usec_rem;
5633 s = kmalloc(sizeof(*s), GFP_KERNEL);
5639 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5640 trace_seq_printf(s, "entries: %ld\n", cnt);
5642 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5643 trace_seq_printf(s, "overrun: %ld\n", cnt);
5645 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5646 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5648 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5649 trace_seq_printf(s, "bytes: %ld\n", cnt);
5651 if (trace_clocks[tr->clock_id].in_ns) {
5652 /* local or global for trace_clock */
5653 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5654 usec_rem = do_div(t, USEC_PER_SEC);
5655 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5658 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5659 usec_rem = do_div(t, USEC_PER_SEC);
5660 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5662 /* counter or tsc mode for trace_clock */
5663 trace_seq_printf(s, "oldest event ts: %llu\n",
5664 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5666 trace_seq_printf(s, "now ts: %llu\n",
5667 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5670 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5671 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5673 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5674 trace_seq_printf(s, "read events: %ld\n", cnt);
5676 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5683 static const struct file_operations tracing_stats_fops = {
5684 .open = tracing_open_generic_tr,
5685 .read = tracing_stats_read,
5686 .llseek = generic_file_llseek,
5687 .release = tracing_release_generic_tr,
5690 #ifdef CONFIG_DYNAMIC_FTRACE
5692 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5698 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5699 size_t cnt, loff_t *ppos)
5701 static char ftrace_dyn_info_buffer[1024];
5702 static DEFINE_MUTEX(dyn_info_mutex);
5703 unsigned long *p = filp->private_data;
5704 char *buf = ftrace_dyn_info_buffer;
5705 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5708 mutex_lock(&dyn_info_mutex);
5709 r = sprintf(buf, "%ld ", *p);
5711 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5714 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5716 mutex_unlock(&dyn_info_mutex);
5721 static const struct file_operations tracing_dyn_info_fops = {
5722 .open = tracing_open_generic,
5723 .read = tracing_read_dyn_info,
5724 .llseek = generic_file_llseek,
5726 #endif /* CONFIG_DYNAMIC_FTRACE */
5728 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5730 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5736 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5738 unsigned long *count = (long *)data;
5750 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5751 struct ftrace_probe_ops *ops, void *data)
5753 long count = (long)data;
5755 seq_printf(m, "%ps:", (void *)ip);
5757 seq_puts(m, "snapshot");
5760 seq_puts(m, ":unlimited\n");
5762 seq_printf(m, ":count=%ld\n", count);
5767 static struct ftrace_probe_ops snapshot_probe_ops = {
5768 .func = ftrace_snapshot,
5769 .print = ftrace_snapshot_print,
5772 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5773 .func = ftrace_count_snapshot,
5774 .print = ftrace_snapshot_print,
5778 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5779 char *glob, char *cmd, char *param, int enable)
5781 struct ftrace_probe_ops *ops;
5782 void *count = (void *)-1;
5786 /* hash funcs only work with set_ftrace_filter */
5790 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5792 if (glob[0] == '!') {
5793 unregister_ftrace_function_probe_func(glob+1, ops);
5800 number = strsep(¶m, ":");
5802 if (!strlen(number))
5806 * We use the callback data field (which is a pointer)
5809 ret = kstrtoul(number, 0, (unsigned long *)&count);
5814 ret = register_ftrace_function_probe(glob, ops, count);
5817 alloc_snapshot(&global_trace);
5819 return ret < 0 ? ret : 0;
5822 static struct ftrace_func_command ftrace_snapshot_cmd = {
5824 .func = ftrace_trace_snapshot_callback,
5827 static __init int register_snapshot_cmd(void)
5829 return register_ftrace_command(&ftrace_snapshot_cmd);
5832 static inline __init int register_snapshot_cmd(void) { return 0; }
5833 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5835 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5840 if (!debugfs_initialized())
5843 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5844 tr->dir = debugfs_create_dir("tracing", NULL);
5847 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5852 struct dentry *tracing_init_dentry(void)
5854 return tracing_init_dentry_tr(&global_trace);
5857 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5859 struct dentry *d_tracer;
5862 return tr->percpu_dir;
5864 d_tracer = tracing_init_dentry_tr(tr);
5868 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5870 WARN_ONCE(!tr->percpu_dir,
5871 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5873 return tr->percpu_dir;
5876 static struct dentry *
5877 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5878 void *data, long cpu, const struct file_operations *fops)
5880 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5882 if (ret) /* See tracing_get_cpu() */
5883 ret->d_inode->i_cdev = (void *)(cpu + 1);
5888 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5890 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5891 struct dentry *d_cpu;
5892 char cpu_dir[30]; /* 30 characters should be more than enough */
5897 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5898 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5900 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5904 /* per cpu trace_pipe */
5905 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5906 tr, cpu, &tracing_pipe_fops);
5909 trace_create_cpu_file("trace", 0644, d_cpu,
5910 tr, cpu, &tracing_fops);
5912 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5913 tr, cpu, &tracing_buffers_fops);
5915 trace_create_cpu_file("stats", 0444, d_cpu,
5916 tr, cpu, &tracing_stats_fops);
5918 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5919 tr, cpu, &tracing_entries_fops);
5921 #ifdef CONFIG_TRACER_SNAPSHOT
5922 trace_create_cpu_file("snapshot", 0644, d_cpu,
5923 tr, cpu, &snapshot_fops);
5925 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5926 tr, cpu, &snapshot_raw_fops);
5930 #ifdef CONFIG_FTRACE_SELFTEST
5931 /* Let selftest have access to static functions in this file */
5932 #include "trace_selftest.c"
5935 struct trace_option_dentry {
5936 struct tracer_opt *opt;
5937 struct tracer_flags *flags;
5938 struct trace_array *tr;
5939 struct dentry *entry;
5943 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5946 struct trace_option_dentry *topt = filp->private_data;
5949 if (topt->flags->val & topt->opt->bit)
5954 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5958 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5961 struct trace_option_dentry *topt = filp->private_data;
5965 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5969 if (val != 0 && val != 1)
5972 if (!!(topt->flags->val & topt->opt->bit) != val) {
5973 mutex_lock(&trace_types_lock);
5974 ret = __set_tracer_option(topt->tr, topt->flags,
5976 mutex_unlock(&trace_types_lock);
5987 static const struct file_operations trace_options_fops = {
5988 .open = tracing_open_generic,
5989 .read = trace_options_read,
5990 .write = trace_options_write,
5991 .llseek = generic_file_llseek,
5995 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5998 long index = (long)filp->private_data;
6001 if (trace_flags & (1 << index))
6006 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6010 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6013 struct trace_array *tr = &global_trace;
6014 long index = (long)filp->private_data;
6018 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6022 if (val != 0 && val != 1)
6025 mutex_lock(&trace_types_lock);
6026 ret = set_tracer_flag(tr, 1 << index, val);
6027 mutex_unlock(&trace_types_lock);
6037 static const struct file_operations trace_options_core_fops = {
6038 .open = tracing_open_generic,
6039 .read = trace_options_core_read,
6040 .write = trace_options_core_write,
6041 .llseek = generic_file_llseek,
6044 struct dentry *trace_create_file(const char *name,
6046 struct dentry *parent,
6048 const struct file_operations *fops)
6052 ret = debugfs_create_file(name, mode, parent, data, fops);
6054 pr_warning("Could not create debugfs '%s' entry\n", name);
6060 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6062 struct dentry *d_tracer;
6067 d_tracer = tracing_init_dentry_tr(tr);
6071 tr->options = debugfs_create_dir("options", d_tracer);
6073 pr_warning("Could not create debugfs directory 'options'\n");
6081 create_trace_option_file(struct trace_array *tr,
6082 struct trace_option_dentry *topt,
6083 struct tracer_flags *flags,
6084 struct tracer_opt *opt)
6086 struct dentry *t_options;
6088 t_options = trace_options_init_dentry(tr);
6092 topt->flags = flags;
6096 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6097 &trace_options_fops);
6101 static struct trace_option_dentry *
6102 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6104 struct trace_option_dentry *topts;
6105 struct tracer_flags *flags;
6106 struct tracer_opt *opts;
6112 flags = tracer->flags;
6114 if (!flags || !flags->opts)
6119 for (cnt = 0; opts[cnt].name; cnt++)
6122 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6126 for (cnt = 0; opts[cnt].name; cnt++)
6127 create_trace_option_file(tr, &topts[cnt], flags,
6134 destroy_trace_option_files(struct trace_option_dentry *topts)
6141 for (cnt = 0; topts[cnt].opt; cnt++)
6142 debugfs_remove(topts[cnt].entry);
6147 static struct dentry *
6148 create_trace_option_core_file(struct trace_array *tr,
6149 const char *option, long index)
6151 struct dentry *t_options;
6153 t_options = trace_options_init_dentry(tr);
6157 return trace_create_file(option, 0644, t_options, (void *)index,
6158 &trace_options_core_fops);
6161 static __init void create_trace_options_dir(struct trace_array *tr)
6163 struct dentry *t_options;
6166 t_options = trace_options_init_dentry(tr);
6170 for (i = 0; trace_options[i]; i++)
6171 create_trace_option_core_file(tr, trace_options[i], i);
6175 rb_simple_read(struct file *filp, char __user *ubuf,
6176 size_t cnt, loff_t *ppos)
6178 struct trace_array *tr = filp->private_data;
6182 r = tracer_tracing_is_on(tr);
6183 r = sprintf(buf, "%d\n", r);
6185 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6189 rb_simple_write(struct file *filp, const char __user *ubuf,
6190 size_t cnt, loff_t *ppos)
6192 struct trace_array *tr = filp->private_data;
6193 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6197 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6202 mutex_lock(&trace_types_lock);
6204 tracer_tracing_on(tr);
6205 if (tr->current_trace->start)
6206 tr->current_trace->start(tr);
6208 tracer_tracing_off(tr);
6209 if (tr->current_trace->stop)
6210 tr->current_trace->stop(tr);
6212 mutex_unlock(&trace_types_lock);
6220 static const struct file_operations rb_simple_fops = {
6221 .open = tracing_open_generic_tr,
6222 .read = rb_simple_read,
6223 .write = rb_simple_write,
6224 .release = tracing_release_generic_tr,
6225 .llseek = default_llseek,
6228 struct dentry *trace_instance_dir;
6231 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6234 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6236 enum ring_buffer_flags rb_flags;
6238 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6242 buf->buffer = ring_buffer_alloc(size, rb_flags);
6246 buf->data = alloc_percpu(struct trace_array_cpu);
6248 ring_buffer_free(buf->buffer);
6252 /* Allocate the first page for all buffers */
6253 set_buffer_entries(&tr->trace_buffer,
6254 ring_buffer_size(tr->trace_buffer.buffer, 0));
6259 static int allocate_trace_buffers(struct trace_array *tr, int size)
6263 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6267 #ifdef CONFIG_TRACER_MAX_TRACE
6268 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6269 allocate_snapshot ? size : 1);
6271 ring_buffer_free(tr->trace_buffer.buffer);
6272 free_percpu(tr->trace_buffer.data);
6275 tr->allocated_snapshot = allocate_snapshot;
6278 * Only the top level trace array gets its snapshot allocated
6279 * from the kernel command line.
6281 allocate_snapshot = false;
6286 static void free_trace_buffer(struct trace_buffer *buf)
6289 ring_buffer_free(buf->buffer);
6291 free_percpu(buf->data);
6296 static void free_trace_buffers(struct trace_array *tr)
6301 free_trace_buffer(&tr->trace_buffer);
6303 #ifdef CONFIG_TRACER_MAX_TRACE
6304 free_trace_buffer(&tr->max_buffer);
6308 static int new_instance_create(const char *name)
6310 struct trace_array *tr;
6313 mutex_lock(&trace_types_lock);
6316 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6317 if (tr->name && strcmp(tr->name, name) == 0)
6322 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6326 tr->name = kstrdup(name, GFP_KERNEL);
6330 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6333 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6335 raw_spin_lock_init(&tr->start_lock);
6337 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6339 tr->current_trace = &nop_trace;
6341 INIT_LIST_HEAD(&tr->systems);
6342 INIT_LIST_HEAD(&tr->events);
6344 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6347 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6351 ret = event_trace_add_tracer(tr->dir, tr);
6353 debugfs_remove_recursive(tr->dir);
6357 init_tracer_debugfs(tr, tr->dir);
6359 list_add(&tr->list, &ftrace_trace_arrays);
6361 mutex_unlock(&trace_types_lock);
6366 free_trace_buffers(tr);
6367 free_cpumask_var(tr->tracing_cpumask);
6372 mutex_unlock(&trace_types_lock);
6378 static int instance_delete(const char *name)
6380 struct trace_array *tr;
6384 mutex_lock(&trace_types_lock);
6387 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6388 if (tr->name && strcmp(tr->name, name) == 0) {
6400 list_del(&tr->list);
6402 tracing_set_nop(tr);
6403 event_trace_del_tracer(tr);
6404 ftrace_destroy_function_files(tr);
6405 debugfs_remove_recursive(tr->dir);
6406 free_trace_buffers(tr);
6414 mutex_unlock(&trace_types_lock);
6419 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6421 struct dentry *parent;
6424 /* Paranoid: Make sure the parent is the "instances" directory */
6425 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6426 if (WARN_ON_ONCE(parent != trace_instance_dir))
6430 * The inode mutex is locked, but debugfs_create_dir() will also
6431 * take the mutex. As the instances directory can not be destroyed
6432 * or changed in any other way, it is safe to unlock it, and
6433 * let the dentry try. If two users try to make the same dir at
6434 * the same time, then the new_instance_create() will determine the
6437 mutex_unlock(&inode->i_mutex);
6439 ret = new_instance_create(dentry->d_iname);
6441 mutex_lock(&inode->i_mutex);
6446 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6448 struct dentry *parent;
6451 /* Paranoid: Make sure the parent is the "instances" directory */
6452 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6453 if (WARN_ON_ONCE(parent != trace_instance_dir))
6456 /* The caller did a dget() on dentry */
6457 mutex_unlock(&dentry->d_inode->i_mutex);
6460 * The inode mutex is locked, but debugfs_create_dir() will also
6461 * take the mutex. As the instances directory can not be destroyed
6462 * or changed in any other way, it is safe to unlock it, and
6463 * let the dentry try. If two users try to make the same dir at
6464 * the same time, then the instance_delete() will determine the
6467 mutex_unlock(&inode->i_mutex);
6469 ret = instance_delete(dentry->d_iname);
6471 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6472 mutex_lock(&dentry->d_inode->i_mutex);
6477 static const struct inode_operations instance_dir_inode_operations = {
6478 .lookup = simple_lookup,
6479 .mkdir = instance_mkdir,
6480 .rmdir = instance_rmdir,
6483 static __init void create_trace_instances(struct dentry *d_tracer)
6485 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6486 if (WARN_ON(!trace_instance_dir))
6489 /* Hijack the dir inode operations, to allow mkdir */
6490 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6494 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6498 trace_create_file("available_tracers", 0444, d_tracer,
6499 tr, &show_traces_fops);
6501 trace_create_file("current_tracer", 0644, d_tracer,
6502 tr, &set_tracer_fops);
6504 trace_create_file("tracing_cpumask", 0644, d_tracer,
6505 tr, &tracing_cpumask_fops);
6507 trace_create_file("trace_options", 0644, d_tracer,
6508 tr, &tracing_iter_fops);
6510 trace_create_file("trace", 0644, d_tracer,
6513 trace_create_file("trace_pipe", 0444, d_tracer,
6514 tr, &tracing_pipe_fops);
6516 trace_create_file("buffer_size_kb", 0644, d_tracer,
6517 tr, &tracing_entries_fops);
6519 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6520 tr, &tracing_total_entries_fops);
6522 trace_create_file("free_buffer", 0200, d_tracer,
6523 tr, &tracing_free_buffer_fops);
6525 trace_create_file("trace_marker", 0220, d_tracer,
6526 tr, &tracing_mark_fops);
6528 trace_create_file("trace_clock", 0644, d_tracer, tr,
6531 trace_create_file("tracing_on", 0644, d_tracer,
6532 tr, &rb_simple_fops);
6534 #ifdef CONFIG_TRACER_MAX_TRACE
6535 trace_create_file("tracing_max_latency", 0644, d_tracer,
6536 &tr->max_latency, &tracing_max_lat_fops);
6539 if (ftrace_create_function_files(tr, d_tracer))
6540 WARN(1, "Could not allocate function filter files");
6542 #ifdef CONFIG_TRACER_SNAPSHOT
6543 trace_create_file("snapshot", 0644, d_tracer,
6544 tr, &snapshot_fops);
6547 for_each_tracing_cpu(cpu)
6548 tracing_init_debugfs_percpu(tr, cpu);
6552 static __init int tracer_init_debugfs(void)
6554 struct dentry *d_tracer;
6556 trace_access_lock_init();
6558 d_tracer = tracing_init_dentry();
6562 init_tracer_debugfs(&global_trace, d_tracer);
6564 trace_create_file("tracing_thresh", 0644, d_tracer,
6565 &global_trace, &tracing_thresh_fops);
6567 trace_create_file("README", 0444, d_tracer,
6568 NULL, &tracing_readme_fops);
6570 trace_create_file("saved_cmdlines", 0444, d_tracer,
6571 NULL, &tracing_saved_cmdlines_fops);
6573 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6574 NULL, &tracing_saved_cmdlines_size_fops);
6576 #ifdef CONFIG_DYNAMIC_FTRACE
6577 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6578 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6581 create_trace_instances(d_tracer);
6583 create_trace_options_dir(&global_trace);
6588 static int trace_panic_handler(struct notifier_block *this,
6589 unsigned long event, void *unused)
6591 if (ftrace_dump_on_oops)
6592 ftrace_dump(ftrace_dump_on_oops);
6596 static struct notifier_block trace_panic_notifier = {
6597 .notifier_call = trace_panic_handler,
6599 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6602 static int trace_die_handler(struct notifier_block *self,
6608 if (ftrace_dump_on_oops)
6609 ftrace_dump(ftrace_dump_on_oops);
6617 static struct notifier_block trace_die_notifier = {
6618 .notifier_call = trace_die_handler,
6623 * printk is set to max of 1024, we really don't need it that big.
6624 * Nothing should be printing 1000 characters anyway.
6626 #define TRACE_MAX_PRINT 1000
6629 * Define here KERN_TRACE so that we have one place to modify
6630 * it if we decide to change what log level the ftrace dump
6633 #define KERN_TRACE KERN_EMERG
6636 trace_printk_seq(struct trace_seq *s)
6638 /* Probably should print a warning here. */
6639 if (s->len >= TRACE_MAX_PRINT)
6640 s->len = TRACE_MAX_PRINT;
6642 /* should be zero ended, but we are paranoid. */
6643 s->buffer[s->len] = 0;
6645 printk(KERN_TRACE "%s", s->buffer);
6650 void trace_init_global_iter(struct trace_iterator *iter)
6652 iter->tr = &global_trace;
6653 iter->trace = iter->tr->current_trace;
6654 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6655 iter->trace_buffer = &global_trace.trace_buffer;
6657 if (iter->trace && iter->trace->open)
6658 iter->trace->open(iter);
6660 /* Annotate start of buffers if we had overruns */
6661 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6662 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6664 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6665 if (trace_clocks[iter->tr->clock_id].in_ns)
6666 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6669 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6671 /* use static because iter can be a bit big for the stack */
6672 static struct trace_iterator iter;
6673 static atomic_t dump_running;
6674 unsigned int old_userobj;
6675 unsigned long flags;
6678 /* Only allow one dump user at a time. */
6679 if (atomic_inc_return(&dump_running) != 1) {
6680 atomic_dec(&dump_running);
6685 * Always turn off tracing when we dump.
6686 * We don't need to show trace output of what happens
6687 * between multiple crashes.
6689 * If the user does a sysrq-z, then they can re-enable
6690 * tracing with echo 1 > tracing_on.
6694 local_irq_save(flags);
6696 /* Simulate the iterator */
6697 trace_init_global_iter(&iter);
6699 for_each_tracing_cpu(cpu) {
6700 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6703 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6705 /* don't look at user memory in panic mode */
6706 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6708 switch (oops_dump_mode) {
6710 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6713 iter.cpu_file = raw_smp_processor_id();
6718 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6719 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6722 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6724 /* Did function tracer already get disabled? */
6725 if (ftrace_is_dead()) {
6726 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6727 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6731 * We need to stop all tracing on all CPUS to read the
6732 * the next buffer. This is a bit expensive, but is
6733 * not done often. We fill all what we can read,
6734 * and then release the locks again.
6737 while (!trace_empty(&iter)) {
6740 printk(KERN_TRACE "---------------------------------\n");
6744 /* reset all but tr, trace, and overruns */
6745 memset(&iter.seq, 0,
6746 sizeof(struct trace_iterator) -
6747 offsetof(struct trace_iterator, seq));
6748 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6751 if (trace_find_next_entry_inc(&iter) != NULL) {
6754 ret = print_trace_line(&iter);
6755 if (ret != TRACE_TYPE_NO_CONSUME)
6756 trace_consume(&iter);
6758 touch_nmi_watchdog();
6760 trace_printk_seq(&iter.seq);
6764 printk(KERN_TRACE " (ftrace buffer empty)\n");
6766 printk(KERN_TRACE "---------------------------------\n");
6769 trace_flags |= old_userobj;
6771 for_each_tracing_cpu(cpu) {
6772 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6774 atomic_dec(&dump_running);
6775 local_irq_restore(flags);
6777 EXPORT_SYMBOL_GPL(ftrace_dump);
6779 __init static int tracer_alloc_buffers(void)
6785 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6788 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6789 goto out_free_buffer_mask;
6791 /* Only allocate trace_printk buffers if a trace_printk exists */
6792 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6793 /* Must be called before global_trace.buffer is allocated */
6794 trace_printk_init_buffers();
6796 /* To save memory, keep the ring buffer size to its minimum */
6797 if (ring_buffer_expanded)
6798 ring_buf_size = trace_buf_size;
6802 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6803 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6805 raw_spin_lock_init(&global_trace.start_lock);
6807 /* Used for event triggers */
6808 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6810 goto out_free_cpumask;
6812 if (trace_create_savedcmd() < 0)
6813 goto out_free_temp_buffer;
6815 /* TODO: make the number of buffers hot pluggable with CPUS */
6816 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6817 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6819 goto out_free_savedcmd;
6822 if (global_trace.buffer_disabled)
6825 if (trace_boot_clock) {
6826 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6828 pr_warning("Trace clock %s not defined, going back to default\n",
6833 * register_tracer() might reference current_trace, so it
6834 * needs to be set before we register anything. This is
6835 * just a bootstrap of current_trace anyway.
6837 global_trace.current_trace = &nop_trace;
6839 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6841 ftrace_init_global_array_ops(&global_trace);
6843 register_tracer(&nop_trace);
6845 /* All seems OK, enable tracing */
6846 tracing_disabled = 0;
6848 atomic_notifier_chain_register(&panic_notifier_list,
6849 &trace_panic_notifier);
6851 register_die_notifier(&trace_die_notifier);
6853 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6855 INIT_LIST_HEAD(&global_trace.systems);
6856 INIT_LIST_HEAD(&global_trace.events);
6857 list_add(&global_trace.list, &ftrace_trace_arrays);
6859 while (trace_boot_options) {
6862 option = strsep(&trace_boot_options, ",");
6863 trace_set_options(&global_trace, option);
6866 register_snapshot_cmd();
6871 free_saved_cmdlines_buffer(savedcmd);
6872 out_free_temp_buffer:
6873 ring_buffer_free(temp_buffer);
6875 free_cpumask_var(global_trace.tracing_cpumask);
6876 out_free_buffer_mask:
6877 free_cpumask_var(tracing_buffer_mask);
6882 __init static int clear_boot_tracer(void)
6885 * The default tracer at boot buffer is an init section.
6886 * This function is called in lateinit. If we did not
6887 * find the boot tracer, then clear it out, to prevent
6888 * later registration from accessing the buffer that is
6889 * about to be freed.
6891 if (!default_bootup_tracer)
6894 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6895 default_bootup_tracer);
6896 default_bootup_tracer = NULL;
6901 early_initcall(tracer_alloc_buffers);
6902 fs_initcall(tracer_init_debugfs);
6903 late_initcall(clear_boot_tracer);