1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer based function tracer
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/seq_file.h>
21 #include <linux/notifier.h>
22 #include <linux/irqflags.h>
23 #include <linux/debugfs.h>
24 #include <linux/tracefs.h>
25 #include <linux/pagemap.h>
26 #include <linux/hardirq.h>
27 #include <linux/linkage.h>
28 #include <linux/uaccess.h>
29 #include <linux/vmalloc.h>
30 #include <linux/ftrace.h>
31 #include <linux/module.h>
32 #include <linux/percpu.h>
33 #include <linux/splice.h>
34 #include <linux/kdebug.h>
35 #include <linux/string.h>
36 #include <linux/mount.h>
37 #include <linux/rwsem.h>
38 #include <linux/slab.h>
39 #include <linux/ctype.h>
40 #include <linux/init.h>
41 #include <linux/poll.h>
42 #include <linux/nmi.h>
44 #include <linux/trace.h>
45 #include <linux/sched/clock.h>
46 #include <linux/sched/rt.h>
49 #include "trace_output.h"
52 * On boot up, the ring buffer is set to the minimum size, so that
53 * we do not waste memory on systems that are not using tracing.
55 bool ring_buffer_expanded;
58 * We need to change this state when a selftest is running.
59 * A selftest will lurk into the ring-buffer to count the
60 * entries inserted during the selftest although some concurrent
61 * insertions into the ring-buffer such as trace_printk could occurred
62 * at the same time, giving false positive or negative results.
64 static bool __read_mostly tracing_selftest_running;
67 * If a tracer is running, we do not want to run SELFTEST.
69 bool __read_mostly tracing_selftest_disabled;
71 /* Pipe tracepoints to printk */
72 struct trace_iterator *tracepoint_print_iter;
73 int tracepoint_printk;
74 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
76 /* For tracers that don't implement custom flags */
77 static struct tracer_opt dummy_tracer_opt[] = {
82 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
88 * To prevent the comm cache from being overwritten when no
89 * tracing is active, only save the comm when a trace event
92 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
95 * Kill all tracing for good (never come back).
96 * It is initialized to 1 but will turn to zero if the initialization
97 * of the tracer is successful. But that is the only place that sets
100 static int tracing_disabled = 1;
102 cpumask_var_t __read_mostly tracing_buffer_mask;
105 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
107 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
108 * is set, then ftrace_dump is called. This will output the contents
109 * of the ftrace buffers to the console. This is very useful for
110 * capturing traces that lead to crashes and outputing it to a
113 * It is default off, but you can enable it with either specifying
114 * "ftrace_dump_on_oops" in the kernel command line, or setting
115 * /proc/sys/kernel/ftrace_dump_on_oops
116 * Set 1 if you want to dump buffers of all CPUs
117 * Set 2 if you want to dump the buffer of the CPU that triggered oops
120 enum ftrace_dump_mode ftrace_dump_on_oops;
122 /* When set, tracing will stop when a WARN*() is hit */
123 int __disable_trace_on_warning;
125 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
126 /* Map of enums to their values, for "eval_map" file */
127 struct trace_eval_map_head {
129 unsigned long length;
132 union trace_eval_map_item;
134 struct trace_eval_map_tail {
136 * "end" is first and points to NULL as it must be different
137 * than "mod" or "eval_string"
139 union trace_eval_map_item *next;
140 const char *end; /* points to NULL */
143 static DEFINE_MUTEX(trace_eval_mutex);
146 * The trace_eval_maps are saved in an array with two extra elements,
147 * one at the beginning, and one at the end. The beginning item contains
148 * the count of the saved maps (head.length), and the module they
149 * belong to if not built in (head.mod). The ending item contains a
150 * pointer to the next array of saved eval_map items.
152 union trace_eval_map_item {
153 struct trace_eval_map map;
154 struct trace_eval_map_head head;
155 struct trace_eval_map_tail tail;
158 static union trace_eval_map_item *trace_eval_maps;
159 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
161 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
163 #define MAX_TRACER_SIZE 100
164 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
165 static char *default_bootup_tracer;
167 static bool allocate_snapshot;
169 static int __init set_cmdline_ftrace(char *str)
171 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
172 default_bootup_tracer = bootup_tracer_buf;
173 /* We are using ftrace early, expand it */
174 ring_buffer_expanded = true;
177 __setup("ftrace=", set_cmdline_ftrace);
179 static int __init set_ftrace_dump_on_oops(char *str)
181 if (*str++ != '=' || !*str) {
182 ftrace_dump_on_oops = DUMP_ALL;
186 if (!strcmp("orig_cpu", str)) {
187 ftrace_dump_on_oops = DUMP_ORIG;
193 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
195 static int __init stop_trace_on_warning(char *str)
197 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
198 __disable_trace_on_warning = 1;
201 __setup("traceoff_on_warning", stop_trace_on_warning);
203 static int __init boot_alloc_snapshot(char *str)
205 allocate_snapshot = true;
206 /* We also need the main ring buffer expanded */
207 ring_buffer_expanded = true;
210 __setup("alloc_snapshot", boot_alloc_snapshot);
213 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
215 static int __init set_trace_boot_options(char *str)
217 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
220 __setup("trace_options=", set_trace_boot_options);
222 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
223 static char *trace_boot_clock __initdata;
225 static int __init set_trace_boot_clock(char *str)
227 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
228 trace_boot_clock = trace_boot_clock_buf;
231 __setup("trace_clock=", set_trace_boot_clock);
233 static int __init set_tracepoint_printk(char *str)
235 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
236 tracepoint_printk = 1;
239 __setup("tp_printk", set_tracepoint_printk);
241 unsigned long long ns2usecs(u64 nsec)
248 /* trace_flags holds trace_options default values */
249 #define TRACE_DEFAULT_FLAGS \
250 (FUNCTION_DEFAULT_FLAGS | \
251 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
252 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
253 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
254 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
256 /* trace_options that are only supported by global_trace */
257 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
258 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
260 /* trace_flags that are default zero for instances */
261 #define ZEROED_TRACE_FLAGS \
262 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
265 * The global_trace is the descriptor that holds the top-level tracing
266 * buffers for the live tracing.
268 static struct trace_array global_trace = {
269 .trace_flags = TRACE_DEFAULT_FLAGS,
272 LIST_HEAD(ftrace_trace_arrays);
274 int trace_array_get(struct trace_array *this_tr)
276 struct trace_array *tr;
279 mutex_lock(&trace_types_lock);
280 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
287 mutex_unlock(&trace_types_lock);
292 static void __trace_array_put(struct trace_array *this_tr)
294 WARN_ON(!this_tr->ref);
298 void trace_array_put(struct trace_array *this_tr)
300 mutex_lock(&trace_types_lock);
301 __trace_array_put(this_tr);
302 mutex_unlock(&trace_types_lock);
305 int call_filter_check_discard(struct trace_event_call *call, void *rec,
306 struct ring_buffer *buffer,
307 struct ring_buffer_event *event)
309 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
310 !filter_match_preds(call->filter, rec)) {
311 __trace_event_discard_commit(buffer, event);
318 void trace_free_pid_list(struct trace_pid_list *pid_list)
320 vfree(pid_list->pids);
325 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
326 * @filtered_pids: The list of pids to check
327 * @search_pid: The PID to find in @filtered_pids
329 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
332 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
335 * If pid_max changed after filtered_pids was created, we
336 * by default ignore all pids greater than the previous pid_max.
338 if (search_pid >= filtered_pids->pid_max)
341 return test_bit(search_pid, filtered_pids->pids);
345 * trace_ignore_this_task - should a task be ignored for tracing
346 * @filtered_pids: The list of pids to check
347 * @task: The task that should be ignored if not filtered
349 * Checks if @task should be traced or not from @filtered_pids.
350 * Returns true if @task should *NOT* be traced.
351 * Returns false if @task should be traced.
354 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
357 * Return false, because if filtered_pids does not exist,
358 * all pids are good to trace.
363 return !trace_find_filtered_pid(filtered_pids, task->pid);
367 * trace_pid_filter_add_remove_task - Add or remove a task from a pid_list
368 * @pid_list: The list to modify
369 * @self: The current task for fork or NULL for exit
370 * @task: The task to add or remove
372 * If adding a task, if @self is defined, the task is only added if @self
373 * is also included in @pid_list. This happens on fork and tasks should
374 * only be added when the parent is listed. If @self is NULL, then the
375 * @task pid will be removed from the list, which would happen on exit
378 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
379 struct task_struct *self,
380 struct task_struct *task)
385 /* For forks, we only add if the forking task is listed */
387 if (!trace_find_filtered_pid(pid_list, self->pid))
391 /* Sorry, but we don't support pid_max changing after setting */
392 if (task->pid >= pid_list->pid_max)
395 /* "self" is set for forks, and NULL for exits */
397 set_bit(task->pid, pid_list->pids);
399 clear_bit(task->pid, pid_list->pids);
403 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
404 * @pid_list: The pid list to show
405 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
406 * @pos: The position of the file
408 * This is used by the seq_file "next" operation to iterate the pids
409 * listed in a trace_pid_list structure.
411 * Returns the pid+1 as we want to display pid of zero, but NULL would
412 * stop the iteration.
414 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
416 unsigned long pid = (unsigned long)v;
420 /* pid already is +1 of the actual prevous bit */
421 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
423 /* Return pid + 1 to allow zero to be represented */
424 if (pid < pid_list->pid_max)
425 return (void *)(pid + 1);
431 * trace_pid_start - Used for seq_file to start reading pid lists
432 * @pid_list: The pid list to show
433 * @pos: The position of the file
435 * This is used by seq_file "start" operation to start the iteration
438 * Returns the pid+1 as we want to display pid of zero, but NULL would
439 * stop the iteration.
441 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
446 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
447 if (pid >= pid_list->pid_max)
450 /* Return pid + 1 so that zero can be the exit value */
451 for (pid++; pid && l < *pos;
452 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
458 * trace_pid_show - show the current pid in seq_file processing
459 * @m: The seq_file structure to write into
460 * @v: A void pointer of the pid (+1) value to display
462 * Can be directly used by seq_file operations to display the current
465 int trace_pid_show(struct seq_file *m, void *v)
467 unsigned long pid = (unsigned long)v - 1;
469 seq_printf(m, "%lu\n", pid);
473 /* 128 should be much more than enough */
474 #define PID_BUF_SIZE 127
476 int trace_pid_write(struct trace_pid_list *filtered_pids,
477 struct trace_pid_list **new_pid_list,
478 const char __user *ubuf, size_t cnt)
480 struct trace_pid_list *pid_list;
481 struct trace_parser parser;
489 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
493 * Always recreate a new array. The write is an all or nothing
494 * operation. Always create a new array when adding new pids by
495 * the user. If the operation fails, then the current list is
498 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
500 trace_parser_put(&parser);
504 pid_list->pid_max = READ_ONCE(pid_max);
506 /* Only truncating will shrink pid_max */
507 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
508 pid_list->pid_max = filtered_pids->pid_max;
510 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
511 if (!pid_list->pids) {
512 trace_parser_put(&parser);
518 /* copy the current bits to the new max */
519 for_each_set_bit(pid, filtered_pids->pids,
520 filtered_pids->pid_max) {
521 set_bit(pid, pid_list->pids);
530 ret = trace_get_user(&parser, ubuf, cnt, &pos);
531 if (ret < 0 || !trace_parser_loaded(&parser))
539 if (kstrtoul(parser.buffer, 0, &val))
541 if (val >= pid_list->pid_max)
546 set_bit(pid, pid_list->pids);
549 trace_parser_clear(&parser);
552 trace_parser_put(&parser);
555 trace_free_pid_list(pid_list);
560 /* Cleared the list of pids */
561 trace_free_pid_list(pid_list);
566 *new_pid_list = pid_list;
571 static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
575 /* Early boot up does not have a buffer yet */
577 return trace_clock_local();
579 ts = ring_buffer_time_stamp(buf->buffer, cpu);
580 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
585 u64 ftrace_now(int cpu)
587 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
591 * tracing_is_enabled - Show if global_trace has been disabled
593 * Shows if the global trace has been enabled or not. It uses the
594 * mirror flag "buffer_disabled" to be used in fast paths such as for
595 * the irqsoff tracer. But it may be inaccurate due to races. If you
596 * need to know the accurate state, use tracing_is_on() which is a little
597 * slower, but accurate.
599 int tracing_is_enabled(void)
602 * For quick access (irqsoff uses this in fast path), just
603 * return the mirror variable of the state of the ring buffer.
604 * It's a little racy, but we don't really care.
607 return !global_trace.buffer_disabled;
611 * trace_buf_size is the size in bytes that is allocated
612 * for a buffer. Note, the number of bytes is always rounded
615 * This number is purposely set to a low number of 16384.
616 * If the dump on oops happens, it will be much appreciated
617 * to not have to wait for all that output. Anyway this can be
618 * boot time and run time configurable.
620 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
622 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
624 /* trace_types holds a link list of available tracers. */
625 static struct tracer *trace_types __read_mostly;
628 * trace_types_lock is used to protect the trace_types list.
630 DEFINE_MUTEX(trace_types_lock);
633 * serialize the access of the ring buffer
635 * ring buffer serializes readers, but it is low level protection.
636 * The validity of the events (which returns by ring_buffer_peek() ..etc)
637 * are not protected by ring buffer.
639 * The content of events may become garbage if we allow other process consumes
640 * these events concurrently:
641 * A) the page of the consumed events may become a normal page
642 * (not reader page) in ring buffer, and this page will be rewrited
643 * by events producer.
644 * B) The page of the consumed events may become a page for splice_read,
645 * and this page will be returned to system.
647 * These primitives allow multi process access to different cpu ring buffer
650 * These primitives don't distinguish read-only and read-consume access.
651 * Multi read-only access are also serialized.
655 static DECLARE_RWSEM(all_cpu_access_lock);
656 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
658 static inline void trace_access_lock(int cpu)
660 if (cpu == RING_BUFFER_ALL_CPUS) {
661 /* gain it for accessing the whole ring buffer. */
662 down_write(&all_cpu_access_lock);
664 /* gain it for accessing a cpu ring buffer. */
666 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
667 down_read(&all_cpu_access_lock);
669 /* Secondly block other access to this @cpu ring buffer. */
670 mutex_lock(&per_cpu(cpu_access_lock, cpu));
674 static inline void trace_access_unlock(int cpu)
676 if (cpu == RING_BUFFER_ALL_CPUS) {
677 up_write(&all_cpu_access_lock);
679 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
680 up_read(&all_cpu_access_lock);
684 static inline void trace_access_lock_init(void)
688 for_each_possible_cpu(cpu)
689 mutex_init(&per_cpu(cpu_access_lock, cpu));
694 static DEFINE_MUTEX(access_lock);
696 static inline void trace_access_lock(int cpu)
699 mutex_lock(&access_lock);
702 static inline void trace_access_unlock(int cpu)
705 mutex_unlock(&access_lock);
708 static inline void trace_access_lock_init(void)
714 #ifdef CONFIG_STACKTRACE
715 static void __ftrace_trace_stack(struct ring_buffer *buffer,
717 int skip, int pc, struct pt_regs *regs);
718 static inline void ftrace_trace_stack(struct trace_array *tr,
719 struct ring_buffer *buffer,
721 int skip, int pc, struct pt_regs *regs);
724 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
726 int skip, int pc, struct pt_regs *regs)
729 static inline void ftrace_trace_stack(struct trace_array *tr,
730 struct ring_buffer *buffer,
732 int skip, int pc, struct pt_regs *regs)
738 static __always_inline void
739 trace_event_setup(struct ring_buffer_event *event,
740 int type, unsigned long flags, int pc)
742 struct trace_entry *ent = ring_buffer_event_data(event);
744 tracing_generic_entry_update(ent, flags, pc);
748 static __always_inline struct ring_buffer_event *
749 __trace_buffer_lock_reserve(struct ring_buffer *buffer,
752 unsigned long flags, int pc)
754 struct ring_buffer_event *event;
756 event = ring_buffer_lock_reserve(buffer, len);
758 trace_event_setup(event, type, flags, pc);
763 void tracer_tracing_on(struct trace_array *tr)
765 if (tr->trace_buffer.buffer)
766 ring_buffer_record_on(tr->trace_buffer.buffer);
768 * This flag is looked at when buffers haven't been allocated
769 * yet, or by some tracers (like irqsoff), that just want to
770 * know if the ring buffer has been disabled, but it can handle
771 * races of where it gets disabled but we still do a record.
772 * As the check is in the fast path of the tracers, it is more
773 * important to be fast than accurate.
775 tr->buffer_disabled = 0;
776 /* Make the flag seen by readers */
781 * tracing_on - enable tracing buffers
783 * This function enables tracing buffers that may have been
784 * disabled with tracing_off.
786 void tracing_on(void)
788 tracer_tracing_on(&global_trace);
790 EXPORT_SYMBOL_GPL(tracing_on);
793 static __always_inline void
794 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
796 __this_cpu_write(trace_taskinfo_save, true);
798 /* If this is the temp buffer, we need to commit fully */
799 if (this_cpu_read(trace_buffered_event) == event) {
800 /* Length is in event->array[0] */
801 ring_buffer_write(buffer, event->array[0], &event->array[1]);
802 /* Release the temp buffer */
803 this_cpu_dec(trace_buffered_event_cnt);
805 ring_buffer_unlock_commit(buffer, event);
809 * __trace_puts - write a constant string into the trace buffer.
810 * @ip: The address of the caller
811 * @str: The constant string to write
812 * @size: The size of the string.
814 int __trace_puts(unsigned long ip, const char *str, int size)
816 struct ring_buffer_event *event;
817 struct ring_buffer *buffer;
818 struct print_entry *entry;
819 unsigned long irq_flags;
823 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
826 pc = preempt_count();
828 if (unlikely(tracing_selftest_running || tracing_disabled))
831 alloc = sizeof(*entry) + size + 2; /* possible \n added */
833 local_save_flags(irq_flags);
834 buffer = global_trace.trace_buffer.buffer;
835 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
840 entry = ring_buffer_event_data(event);
843 memcpy(&entry->buf, str, size);
845 /* Add a newline if necessary */
846 if (entry->buf[size - 1] != '\n') {
847 entry->buf[size] = '\n';
848 entry->buf[size + 1] = '\0';
850 entry->buf[size] = '\0';
852 __buffer_unlock_commit(buffer, event);
853 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
857 EXPORT_SYMBOL_GPL(__trace_puts);
860 * __trace_bputs - write the pointer to a constant string into trace buffer
861 * @ip: The address of the caller
862 * @str: The constant string to write to the buffer to
864 int __trace_bputs(unsigned long ip, const char *str)
866 struct ring_buffer_event *event;
867 struct ring_buffer *buffer;
868 struct bputs_entry *entry;
869 unsigned long irq_flags;
870 int size = sizeof(struct bputs_entry);
873 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
876 pc = preempt_count();
878 if (unlikely(tracing_selftest_running || tracing_disabled))
881 local_save_flags(irq_flags);
882 buffer = global_trace.trace_buffer.buffer;
883 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
888 entry = ring_buffer_event_data(event);
892 __buffer_unlock_commit(buffer, event);
893 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
897 EXPORT_SYMBOL_GPL(__trace_bputs);
899 #ifdef CONFIG_TRACER_SNAPSHOT
900 void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data)
902 struct tracer *tracer = tr->current_trace;
906 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
907 internal_trace_puts("*** snapshot is being ignored ***\n");
911 if (!tr->allocated_snapshot) {
912 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
913 internal_trace_puts("*** stopping trace here! ***\n");
918 /* Note, snapshot can not be used when the tracer uses it */
919 if (tracer->use_max_tr) {
920 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
921 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
925 local_irq_save(flags);
926 update_max_tr(tr, current, smp_processor_id(), cond_data);
927 local_irq_restore(flags);
930 void tracing_snapshot_instance(struct trace_array *tr)
932 tracing_snapshot_instance_cond(tr, NULL);
936 * tracing_snapshot - take a snapshot of the current buffer.
938 * This causes a swap between the snapshot buffer and the current live
939 * tracing buffer. You can use this to take snapshots of the live
940 * trace when some condition is triggered, but continue to trace.
942 * Note, make sure to allocate the snapshot with either
943 * a tracing_snapshot_alloc(), or by doing it manually
944 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
946 * If the snapshot buffer is not allocated, it will stop tracing.
947 * Basically making a permanent snapshot.
949 void tracing_snapshot(void)
951 struct trace_array *tr = &global_trace;
953 tracing_snapshot_instance(tr);
955 EXPORT_SYMBOL_GPL(tracing_snapshot);
958 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
959 * @tr: The tracing instance to snapshot
960 * @cond_data: The data to be tested conditionally, and possibly saved
962 * This is the same as tracing_snapshot() except that the snapshot is
963 * conditional - the snapshot will only happen if the
964 * cond_snapshot.update() implementation receiving the cond_data
965 * returns true, which means that the trace array's cond_snapshot
966 * update() operation used the cond_data to determine whether the
967 * snapshot should be taken, and if it was, presumably saved it along
970 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
972 tracing_snapshot_instance_cond(tr, cond_data);
974 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
977 * tracing_snapshot_cond_data - get the user data associated with a snapshot
978 * @tr: The tracing instance
980 * When the user enables a conditional snapshot using
981 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
982 * with the snapshot. This accessor is used to retrieve it.
984 * Should not be called from cond_snapshot.update(), since it takes
985 * the tr->max_lock lock, which the code calling
986 * cond_snapshot.update() has already done.
988 * Returns the cond_data associated with the trace array's snapshot.
990 void *tracing_cond_snapshot_data(struct trace_array *tr)
992 void *cond_data = NULL;
994 arch_spin_lock(&tr->max_lock);
996 if (tr->cond_snapshot)
997 cond_data = tr->cond_snapshot->cond_data;
999 arch_spin_unlock(&tr->max_lock);
1003 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1005 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
1006 struct trace_buffer *size_buf, int cpu_id);
1007 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
1009 int tracing_alloc_snapshot_instance(struct trace_array *tr)
1013 if (!tr->allocated_snapshot) {
1015 /* allocate spare buffer */
1016 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1017 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
1021 tr->allocated_snapshot = true;
1027 static void free_snapshot(struct trace_array *tr)
1030 * We don't free the ring buffer. instead, resize it because
1031 * The max_tr ring buffer has some state (e.g. ring->clock) and
1032 * we want preserve it.
1034 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1035 set_buffer_entries(&tr->max_buffer, 1);
1036 tracing_reset_online_cpus(&tr->max_buffer);
1037 tr->allocated_snapshot = false;
1041 * tracing_alloc_snapshot - allocate snapshot buffer.
1043 * This only allocates the snapshot buffer if it isn't already
1044 * allocated - it doesn't also take a snapshot.
1046 * This is meant to be used in cases where the snapshot buffer needs
1047 * to be set up for events that can't sleep but need to be able to
1048 * trigger a snapshot.
1050 int tracing_alloc_snapshot(void)
1052 struct trace_array *tr = &global_trace;
1055 ret = tracing_alloc_snapshot_instance(tr);
1060 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1063 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1065 * This is similar to tracing_snapshot(), but it will allocate the
1066 * snapshot buffer if it isn't already allocated. Use this only
1067 * where it is safe to sleep, as the allocation may sleep.
1069 * This causes a swap between the snapshot buffer and the current live
1070 * tracing buffer. You can use this to take snapshots of the live
1071 * trace when some condition is triggered, but continue to trace.
1073 void tracing_snapshot_alloc(void)
1077 ret = tracing_alloc_snapshot();
1083 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1086 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1087 * @tr: The tracing instance
1088 * @cond_data: User data to associate with the snapshot
1089 * @update: Implementation of the cond_snapshot update function
1091 * Check whether the conditional snapshot for the given instance has
1092 * already been enabled, or if the current tracer is already using a
1093 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1094 * save the cond_data and update function inside.
1096 * Returns 0 if successful, error otherwise.
1098 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1099 cond_update_fn_t update)
1101 struct cond_snapshot *cond_snapshot;
1104 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1108 cond_snapshot->cond_data = cond_data;
1109 cond_snapshot->update = update;
1111 mutex_lock(&trace_types_lock);
1113 ret = tracing_alloc_snapshot_instance(tr);
1117 if (tr->current_trace->use_max_tr) {
1123 * The cond_snapshot can only change to NULL without the
1124 * trace_types_lock. We don't care if we race with it going
1125 * to NULL, but we want to make sure that it's not set to
1126 * something other than NULL when we get here, which we can
1127 * do safely with only holding the trace_types_lock and not
1128 * having to take the max_lock.
1130 if (tr->cond_snapshot) {
1135 arch_spin_lock(&tr->max_lock);
1136 tr->cond_snapshot = cond_snapshot;
1137 arch_spin_unlock(&tr->max_lock);
1139 mutex_unlock(&trace_types_lock);
1144 mutex_unlock(&trace_types_lock);
1145 kfree(cond_snapshot);
1148 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1151 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1152 * @tr: The tracing instance
1154 * Check whether the conditional snapshot for the given instance is
1155 * enabled; if so, free the cond_snapshot associated with it,
1156 * otherwise return -EINVAL.
1158 * Returns 0 if successful, error otherwise.
1160 int tracing_snapshot_cond_disable(struct trace_array *tr)
1164 arch_spin_lock(&tr->max_lock);
1166 if (!tr->cond_snapshot)
1169 kfree(tr->cond_snapshot);
1170 tr->cond_snapshot = NULL;
1173 arch_spin_unlock(&tr->max_lock);
1177 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1179 void tracing_snapshot(void)
1181 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1183 EXPORT_SYMBOL_GPL(tracing_snapshot);
1184 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1186 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1188 EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1189 int tracing_alloc_snapshot(void)
1191 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1194 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1195 void tracing_snapshot_alloc(void)
1200 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1201 void *tracing_cond_snapshot_data(struct trace_array *tr)
1205 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1206 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1210 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1211 int tracing_snapshot_cond_disable(struct trace_array *tr)
1215 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1216 #endif /* CONFIG_TRACER_SNAPSHOT */
1218 void tracer_tracing_off(struct trace_array *tr)
1220 if (tr->trace_buffer.buffer)
1221 ring_buffer_record_off(tr->trace_buffer.buffer);
1223 * This flag is looked at when buffers haven't been allocated
1224 * yet, or by some tracers (like irqsoff), that just want to
1225 * know if the ring buffer has been disabled, but it can handle
1226 * races of where it gets disabled but we still do a record.
1227 * As the check is in the fast path of the tracers, it is more
1228 * important to be fast than accurate.
1230 tr->buffer_disabled = 1;
1231 /* Make the flag seen by readers */
1236 * tracing_off - turn off tracing buffers
1238 * This function stops the tracing buffers from recording data.
1239 * It does not disable any overhead the tracers themselves may
1240 * be causing. This function simply causes all recording to
1241 * the ring buffers to fail.
1243 void tracing_off(void)
1245 tracer_tracing_off(&global_trace);
1247 EXPORT_SYMBOL_GPL(tracing_off);
1249 void disable_trace_on_warning(void)
1251 if (__disable_trace_on_warning)
1256 * tracer_tracing_is_on - show real state of ring buffer enabled
1257 * @tr : the trace array to know if ring buffer is enabled
1259 * Shows real state of the ring buffer if it is enabled or not.
1261 bool tracer_tracing_is_on(struct trace_array *tr)
1263 if (tr->trace_buffer.buffer)
1264 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1265 return !tr->buffer_disabled;
1269 * tracing_is_on - show state of ring buffers enabled
1271 int tracing_is_on(void)
1273 return tracer_tracing_is_on(&global_trace);
1275 EXPORT_SYMBOL_GPL(tracing_is_on);
1277 static int __init set_buf_size(char *str)
1279 unsigned long buf_size;
1283 buf_size = memparse(str, &str);
1284 /* nr_entries can not be zero */
1287 trace_buf_size = buf_size;
1290 __setup("trace_buf_size=", set_buf_size);
1292 static int __init set_tracing_thresh(char *str)
1294 unsigned long threshold;
1299 ret = kstrtoul(str, 0, &threshold);
1302 tracing_thresh = threshold * 1000;
1305 __setup("tracing_thresh=", set_tracing_thresh);
1307 unsigned long nsecs_to_usecs(unsigned long nsecs)
1309 return nsecs / 1000;
1313 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1314 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1315 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1316 * of strings in the order that the evals (enum) were defined.
1321 /* These must match the bit postions in trace_iterator_flags */
1322 static const char *trace_options[] = {
1330 int in_ns; /* is this clock in nanoseconds? */
1331 } trace_clocks[] = {
1332 { trace_clock_local, "local", 1 },
1333 { trace_clock_global, "global", 1 },
1334 { trace_clock_counter, "counter", 0 },
1335 { trace_clock_jiffies, "uptime", 0 },
1336 { trace_clock, "perf", 1 },
1337 { ktime_get_mono_fast_ns, "mono", 1 },
1338 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1339 { ktime_get_boot_fast_ns, "boot", 1 },
1343 bool trace_clock_in_ns(struct trace_array *tr)
1345 if (trace_clocks[tr->clock_id].in_ns)
1352 * trace_parser_get_init - gets the buffer for trace parser
1354 int trace_parser_get_init(struct trace_parser *parser, int size)
1356 memset(parser, 0, sizeof(*parser));
1358 parser->buffer = kmalloc(size, GFP_KERNEL);
1359 if (!parser->buffer)
1362 parser->size = size;
1367 * trace_parser_put - frees the buffer for trace parser
1369 void trace_parser_put(struct trace_parser *parser)
1371 kfree(parser->buffer);
1372 parser->buffer = NULL;
1376 * trace_get_user - reads the user input string separated by space
1377 * (matched by isspace(ch))
1379 * For each string found the 'struct trace_parser' is updated,
1380 * and the function returns.
1382 * Returns number of bytes read.
1384 * See kernel/trace/trace.h for 'struct trace_parser' details.
1386 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1387 size_t cnt, loff_t *ppos)
1394 trace_parser_clear(parser);
1396 ret = get_user(ch, ubuf++);
1404 * The parser is not finished with the last write,
1405 * continue reading the user input without skipping spaces.
1407 if (!parser->cont) {
1408 /* skip white space */
1409 while (cnt && isspace(ch)) {
1410 ret = get_user(ch, ubuf++);
1419 /* only spaces were written */
1420 if (isspace(ch) || !ch) {
1427 /* read the non-space input */
1428 while (cnt && !isspace(ch) && ch) {
1429 if (parser->idx < parser->size - 1)
1430 parser->buffer[parser->idx++] = ch;
1435 ret = get_user(ch, ubuf++);
1442 /* We either got finished input or we have to wait for another call. */
1443 if (isspace(ch) || !ch) {
1444 parser->buffer[parser->idx] = 0;
1445 parser->cont = false;
1446 } else if (parser->idx < parser->size - 1) {
1447 parser->cont = true;
1448 parser->buffer[parser->idx++] = ch;
1449 /* Make sure the parsed string always terminates with '\0'. */
1450 parser->buffer[parser->idx] = 0;
1463 /* TODO add a seq_buf_to_buffer() */
1464 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1468 if (trace_seq_used(s) <= s->seq.readpos)
1471 len = trace_seq_used(s) - s->seq.readpos;
1474 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1476 s->seq.readpos += cnt;
1480 unsigned long __read_mostly tracing_thresh;
1482 #ifdef CONFIG_TRACER_MAX_TRACE
1484 * Copy the new maximum trace into the separate maximum-trace
1485 * structure. (this way the maximum trace is permanently saved,
1486 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1489 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1491 struct trace_buffer *trace_buf = &tr->trace_buffer;
1492 struct trace_buffer *max_buf = &tr->max_buffer;
1493 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1494 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1497 max_buf->time_start = data->preempt_timestamp;
1499 max_data->saved_latency = tr->max_latency;
1500 max_data->critical_start = data->critical_start;
1501 max_data->critical_end = data->critical_end;
1503 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1504 max_data->pid = tsk->pid;
1506 * If tsk == current, then use current_uid(), as that does not use
1507 * RCU. The irq tracer can be called out of RCU scope.
1510 max_data->uid = current_uid();
1512 max_data->uid = task_uid(tsk);
1514 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1515 max_data->policy = tsk->policy;
1516 max_data->rt_priority = tsk->rt_priority;
1518 /* record this tasks comm */
1519 tracing_record_cmdline(tsk);
1523 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1525 * @tsk: the task with the latency
1526 * @cpu: The cpu that initiated the trace.
1527 * @cond_data: User data associated with a conditional snapshot
1529 * Flip the buffers between the @tr and the max_tr and record information
1530 * about which task was the cause of this latency.
1533 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1539 WARN_ON_ONCE(!irqs_disabled());
1541 if (!tr->allocated_snapshot) {
1542 /* Only the nop tracer should hit this when disabling */
1543 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1547 arch_spin_lock(&tr->max_lock);
1549 /* Inherit the recordable setting from trace_buffer */
1550 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1551 ring_buffer_record_on(tr->max_buffer.buffer);
1553 ring_buffer_record_off(tr->max_buffer.buffer);
1555 #ifdef CONFIG_TRACER_SNAPSHOT
1556 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1559 swap(tr->trace_buffer.buffer, tr->max_buffer.buffer);
1561 __update_max_tr(tr, tsk, cpu);
1564 arch_spin_unlock(&tr->max_lock);
1568 * update_max_tr_single - only copy one trace over, and reset the rest
1570 * @tsk - task with the latency
1571 * @cpu - the cpu of the buffer to copy.
1573 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1576 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1583 WARN_ON_ONCE(!irqs_disabled());
1584 if (!tr->allocated_snapshot) {
1585 /* Only the nop tracer should hit this when disabling */
1586 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1590 arch_spin_lock(&tr->max_lock);
1592 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1594 if (ret == -EBUSY) {
1596 * We failed to swap the buffer due to a commit taking
1597 * place on this CPU. We fail to record, but we reset
1598 * the max trace buffer (no one writes directly to it)
1599 * and flag that it failed.
1601 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1602 "Failed to swap buffers due to commit in progress\n");
1605 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1607 __update_max_tr(tr, tsk, cpu);
1608 arch_spin_unlock(&tr->max_lock);
1610 #endif /* CONFIG_TRACER_MAX_TRACE */
1612 static int wait_on_pipe(struct trace_iterator *iter, int full)
1614 /* Iterators are static, they should be filled or empty */
1615 if (trace_buffer_iter(iter, iter->cpu_file))
1618 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1622 #ifdef CONFIG_FTRACE_STARTUP_TEST
1623 static bool selftests_can_run;
1625 struct trace_selftests {
1626 struct list_head list;
1627 struct tracer *type;
1630 static LIST_HEAD(postponed_selftests);
1632 static int save_selftest(struct tracer *type)
1634 struct trace_selftests *selftest;
1636 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1640 selftest->type = type;
1641 list_add(&selftest->list, &postponed_selftests);
1645 static int run_tracer_selftest(struct tracer *type)
1647 struct trace_array *tr = &global_trace;
1648 struct tracer *saved_tracer = tr->current_trace;
1651 if (!type->selftest || tracing_selftest_disabled)
1655 * If a tracer registers early in boot up (before scheduling is
1656 * initialized and such), then do not run its selftests yet.
1657 * Instead, run it a little later in the boot process.
1659 if (!selftests_can_run)
1660 return save_selftest(type);
1663 * Run a selftest on this tracer.
1664 * Here we reset the trace buffer, and set the current
1665 * tracer to be this tracer. The tracer can then run some
1666 * internal tracing to verify that everything is in order.
1667 * If we fail, we do not register this tracer.
1669 tracing_reset_online_cpus(&tr->trace_buffer);
1671 tr->current_trace = type;
1673 #ifdef CONFIG_TRACER_MAX_TRACE
1674 if (type->use_max_tr) {
1675 /* If we expanded the buffers, make sure the max is expanded too */
1676 if (ring_buffer_expanded)
1677 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1678 RING_BUFFER_ALL_CPUS);
1679 tr->allocated_snapshot = true;
1683 /* the test is responsible for initializing and enabling */
1684 pr_info("Testing tracer %s: ", type->name);
1685 ret = type->selftest(type, tr);
1686 /* the test is responsible for resetting too */
1687 tr->current_trace = saved_tracer;
1689 printk(KERN_CONT "FAILED!\n");
1690 /* Add the warning after printing 'FAILED' */
1694 /* Only reset on passing, to avoid touching corrupted buffers */
1695 tracing_reset_online_cpus(&tr->trace_buffer);
1697 #ifdef CONFIG_TRACER_MAX_TRACE
1698 if (type->use_max_tr) {
1699 tr->allocated_snapshot = false;
1701 /* Shrink the max buffer again */
1702 if (ring_buffer_expanded)
1703 ring_buffer_resize(tr->max_buffer.buffer, 1,
1704 RING_BUFFER_ALL_CPUS);
1708 printk(KERN_CONT "PASSED\n");
1712 static __init int init_trace_selftests(void)
1714 struct trace_selftests *p, *n;
1715 struct tracer *t, **last;
1718 selftests_can_run = true;
1720 mutex_lock(&trace_types_lock);
1722 if (list_empty(&postponed_selftests))
1725 pr_info("Running postponed tracer tests:\n");
1727 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1728 ret = run_tracer_selftest(p->type);
1729 /* If the test fails, then warn and remove from available_tracers */
1731 WARN(1, "tracer: %s failed selftest, disabling\n",
1733 last = &trace_types;
1734 for (t = trace_types; t; t = t->next) {
1747 mutex_unlock(&trace_types_lock);
1751 core_initcall(init_trace_selftests);
1753 static inline int run_tracer_selftest(struct tracer *type)
1757 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1759 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1761 static void __init apply_trace_boot_options(void);
1764 * register_tracer - register a tracer with the ftrace system.
1765 * @type - the plugin for the tracer
1767 * Register a new plugin tracer.
1769 int __init register_tracer(struct tracer *type)
1775 pr_info("Tracer must have a name\n");
1779 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1780 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1784 mutex_lock(&trace_types_lock);
1786 tracing_selftest_running = true;
1788 for (t = trace_types; t; t = t->next) {
1789 if (strcmp(type->name, t->name) == 0) {
1791 pr_info("Tracer %s already registered\n",
1798 if (!type->set_flag)
1799 type->set_flag = &dummy_set_flag;
1801 /*allocate a dummy tracer_flags*/
1802 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1807 type->flags->val = 0;
1808 type->flags->opts = dummy_tracer_opt;
1810 if (!type->flags->opts)
1811 type->flags->opts = dummy_tracer_opt;
1813 /* store the tracer for __set_tracer_option */
1814 type->flags->trace = type;
1816 ret = run_tracer_selftest(type);
1820 type->next = trace_types;
1822 add_tracer_options(&global_trace, type);
1825 tracing_selftest_running = false;
1826 mutex_unlock(&trace_types_lock);
1828 if (ret || !default_bootup_tracer)
1831 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1834 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1835 /* Do we want this tracer to start on bootup? */
1836 tracing_set_tracer(&global_trace, type->name);
1837 default_bootup_tracer = NULL;
1839 apply_trace_boot_options();
1841 /* disable other selftests, since this will break it. */
1842 tracing_selftest_disabled = true;
1843 #ifdef CONFIG_FTRACE_STARTUP_TEST
1844 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1852 void tracing_reset(struct trace_buffer *buf, int cpu)
1854 struct ring_buffer *buffer = buf->buffer;
1859 ring_buffer_record_disable(buffer);
1861 /* Make sure all commits have finished */
1863 ring_buffer_reset_cpu(buffer, cpu);
1865 ring_buffer_record_enable(buffer);
1868 void tracing_reset_online_cpus(struct trace_buffer *buf)
1870 struct ring_buffer *buffer = buf->buffer;
1876 ring_buffer_record_disable(buffer);
1878 /* Make sure all commits have finished */
1881 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1883 for_each_online_cpu(cpu)
1884 ring_buffer_reset_cpu(buffer, cpu);
1886 ring_buffer_record_enable(buffer);
1889 /* Must have trace_types_lock held */
1890 void tracing_reset_all_online_cpus(void)
1892 struct trace_array *tr;
1894 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1895 if (!tr->clear_trace)
1897 tr->clear_trace = false;
1898 tracing_reset_online_cpus(&tr->trace_buffer);
1899 #ifdef CONFIG_TRACER_MAX_TRACE
1900 tracing_reset_online_cpus(&tr->max_buffer);
1905 static int *tgid_map;
1907 #define SAVED_CMDLINES_DEFAULT 128
1908 #define NO_CMDLINE_MAP UINT_MAX
1909 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1910 struct saved_cmdlines_buffer {
1911 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1912 unsigned *map_cmdline_to_pid;
1913 unsigned cmdline_num;
1915 char *saved_cmdlines;
1917 static struct saved_cmdlines_buffer *savedcmd;
1919 /* temporary disable recording */
1920 static atomic_t trace_record_taskinfo_disabled __read_mostly;
1922 static inline char *get_saved_cmdlines(int idx)
1924 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1927 static inline void set_cmdline(int idx, const char *cmdline)
1929 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1932 static int allocate_cmdlines_buffer(unsigned int val,
1933 struct saved_cmdlines_buffer *s)
1935 s->map_cmdline_to_pid = kmalloc_array(val,
1936 sizeof(*s->map_cmdline_to_pid),
1938 if (!s->map_cmdline_to_pid)
1941 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
1942 if (!s->saved_cmdlines) {
1943 kfree(s->map_cmdline_to_pid);
1948 s->cmdline_num = val;
1949 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1950 sizeof(s->map_pid_to_cmdline));
1951 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1952 val * sizeof(*s->map_cmdline_to_pid));
1957 static int trace_create_savedcmd(void)
1961 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1965 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1975 int is_tracing_stopped(void)
1977 return global_trace.stop_count;
1981 * tracing_start - quick start of the tracer
1983 * If tracing is enabled but was stopped by tracing_stop,
1984 * this will start the tracer back up.
1986 void tracing_start(void)
1988 struct ring_buffer *buffer;
1989 unsigned long flags;
1991 if (tracing_disabled)
1994 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1995 if (--global_trace.stop_count) {
1996 if (global_trace.stop_count < 0) {
1997 /* Someone screwed up their debugging */
1999 global_trace.stop_count = 0;
2004 /* Prevent the buffers from switching */
2005 arch_spin_lock(&global_trace.max_lock);
2007 buffer = global_trace.trace_buffer.buffer;
2009 ring_buffer_record_enable(buffer);
2011 #ifdef CONFIG_TRACER_MAX_TRACE
2012 buffer = global_trace.max_buffer.buffer;
2014 ring_buffer_record_enable(buffer);
2017 arch_spin_unlock(&global_trace.max_lock);
2020 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2023 static void tracing_start_tr(struct trace_array *tr)
2025 struct ring_buffer *buffer;
2026 unsigned long flags;
2028 if (tracing_disabled)
2031 /* If global, we need to also start the max tracer */
2032 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2033 return tracing_start();
2035 raw_spin_lock_irqsave(&tr->start_lock, flags);
2037 if (--tr->stop_count) {
2038 if (tr->stop_count < 0) {
2039 /* Someone screwed up their debugging */
2046 buffer = tr->trace_buffer.buffer;
2048 ring_buffer_record_enable(buffer);
2051 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2055 * tracing_stop - quick stop of the tracer
2057 * Light weight way to stop tracing. Use in conjunction with
2060 void tracing_stop(void)
2062 struct ring_buffer *buffer;
2063 unsigned long flags;
2065 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2066 if (global_trace.stop_count++)
2069 /* Prevent the buffers from switching */
2070 arch_spin_lock(&global_trace.max_lock);
2072 buffer = global_trace.trace_buffer.buffer;
2074 ring_buffer_record_disable(buffer);
2076 #ifdef CONFIG_TRACER_MAX_TRACE
2077 buffer = global_trace.max_buffer.buffer;
2079 ring_buffer_record_disable(buffer);
2082 arch_spin_unlock(&global_trace.max_lock);
2085 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2088 static void tracing_stop_tr(struct trace_array *tr)
2090 struct ring_buffer *buffer;
2091 unsigned long flags;
2093 /* If global, we need to also stop the max tracer */
2094 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2095 return tracing_stop();
2097 raw_spin_lock_irqsave(&tr->start_lock, flags);
2098 if (tr->stop_count++)
2101 buffer = tr->trace_buffer.buffer;
2103 ring_buffer_record_disable(buffer);
2106 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2109 static int trace_save_cmdline(struct task_struct *tsk)
2113 /* treat recording of idle task as a success */
2117 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
2121 * It's not the end of the world if we don't get
2122 * the lock, but we also don't want to spin
2123 * nor do we want to disable interrupts,
2124 * so if we miss here, then better luck next time.
2126 if (!arch_spin_trylock(&trace_cmdline_lock))
2129 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2130 if (idx == NO_CMDLINE_MAP) {
2131 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
2134 * Check whether the cmdline buffer at idx has a pid
2135 * mapped. We are going to overwrite that entry so we
2136 * need to clear the map_pid_to_cmdline. Otherwise we
2137 * would read the new comm for the old pid.
2139 pid = savedcmd->map_cmdline_to_pid[idx];
2140 if (pid != NO_CMDLINE_MAP)
2141 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
2143 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2144 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
2146 savedcmd->cmdline_idx = idx;
2149 set_cmdline(idx, tsk->comm);
2151 arch_spin_unlock(&trace_cmdline_lock);
2156 static void __trace_find_cmdline(int pid, char comm[])
2161 strcpy(comm, "<idle>");
2165 if (WARN_ON_ONCE(pid < 0)) {
2166 strcpy(comm, "<XXX>");
2170 if (pid > PID_MAX_DEFAULT) {
2171 strcpy(comm, "<...>");
2175 map = savedcmd->map_pid_to_cmdline[pid];
2176 if (map != NO_CMDLINE_MAP)
2177 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
2179 strcpy(comm, "<...>");
2182 void trace_find_cmdline(int pid, char comm[])
2185 arch_spin_lock(&trace_cmdline_lock);
2187 __trace_find_cmdline(pid, comm);
2189 arch_spin_unlock(&trace_cmdline_lock);
2193 int trace_find_tgid(int pid)
2195 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2198 return tgid_map[pid];
2201 static int trace_save_tgid(struct task_struct *tsk)
2203 /* treat recording of idle task as a success */
2207 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
2210 tgid_map[tsk->pid] = tsk->tgid;
2214 static bool tracing_record_taskinfo_skip(int flags)
2216 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2218 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2220 if (!__this_cpu_read(trace_taskinfo_save))
2226 * tracing_record_taskinfo - record the task info of a task
2228 * @task - task to record
2229 * @flags - TRACE_RECORD_CMDLINE for recording comm
2230 * - TRACE_RECORD_TGID for recording tgid
2232 void tracing_record_taskinfo(struct task_struct *task, int flags)
2236 if (tracing_record_taskinfo_skip(flags))
2240 * Record as much task information as possible. If some fail, continue
2241 * to try to record the others.
2243 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2244 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2246 /* If recording any information failed, retry again soon. */
2250 __this_cpu_write(trace_taskinfo_save, false);
2254 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2256 * @prev - previous task during sched_switch
2257 * @next - next task during sched_switch
2258 * @flags - TRACE_RECORD_CMDLINE for recording comm
2259 * TRACE_RECORD_TGID for recording tgid
2261 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2262 struct task_struct *next, int flags)
2266 if (tracing_record_taskinfo_skip(flags))
2270 * Record as much task information as possible. If some fail, continue
2271 * to try to record the others.
2273 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2274 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2275 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2276 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2278 /* If recording any information failed, retry again soon. */
2282 __this_cpu_write(trace_taskinfo_save, false);
2285 /* Helpers to record a specific task information */
2286 void tracing_record_cmdline(struct task_struct *task)
2288 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2291 void tracing_record_tgid(struct task_struct *task)
2293 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2297 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2298 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2299 * simplifies those functions and keeps them in sync.
2301 enum print_line_t trace_handle_return(struct trace_seq *s)
2303 return trace_seq_has_overflowed(s) ?
2304 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2306 EXPORT_SYMBOL_GPL(trace_handle_return);
2309 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
2312 struct task_struct *tsk = current;
2314 entry->preempt_count = pc & 0xff;
2315 entry->pid = (tsk) ? tsk->pid : 0;
2317 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2318 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2320 TRACE_FLAG_IRQS_NOSUPPORT |
2322 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
2323 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2324 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2325 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2326 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2328 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2330 struct ring_buffer_event *
2331 trace_buffer_lock_reserve(struct ring_buffer *buffer,
2334 unsigned long flags, int pc)
2336 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2339 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2340 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2341 static int trace_buffered_event_ref;
2344 * trace_buffered_event_enable - enable buffering events
2346 * When events are being filtered, it is quicker to use a temporary
2347 * buffer to write the event data into if there's a likely chance
2348 * that it will not be committed. The discard of the ring buffer
2349 * is not as fast as committing, and is much slower than copying
2352 * When an event is to be filtered, allocate per cpu buffers to
2353 * write the event data into, and if the event is filtered and discarded
2354 * it is simply dropped, otherwise, the entire data is to be committed
2357 void trace_buffered_event_enable(void)
2359 struct ring_buffer_event *event;
2363 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2365 if (trace_buffered_event_ref++)
2368 for_each_tracing_cpu(cpu) {
2369 page = alloc_pages_node(cpu_to_node(cpu),
2370 GFP_KERNEL | __GFP_NORETRY, 0);
2374 event = page_address(page);
2375 memset(event, 0, sizeof(*event));
2377 per_cpu(trace_buffered_event, cpu) = event;
2380 if (cpu == smp_processor_id() &&
2381 this_cpu_read(trace_buffered_event) !=
2382 per_cpu(trace_buffered_event, cpu))
2389 trace_buffered_event_disable();
2392 static void enable_trace_buffered_event(void *data)
2394 /* Probably not needed, but do it anyway */
2396 this_cpu_dec(trace_buffered_event_cnt);
2399 static void disable_trace_buffered_event(void *data)
2401 this_cpu_inc(trace_buffered_event_cnt);
2405 * trace_buffered_event_disable - disable buffering events
2407 * When a filter is removed, it is faster to not use the buffered
2408 * events, and to commit directly into the ring buffer. Free up
2409 * the temp buffers when there are no more users. This requires
2410 * special synchronization with current events.
2412 void trace_buffered_event_disable(void)
2416 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2418 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2421 if (--trace_buffered_event_ref)
2425 /* For each CPU, set the buffer as used. */
2426 smp_call_function_many(tracing_buffer_mask,
2427 disable_trace_buffered_event, NULL, 1);
2430 /* Wait for all current users to finish */
2433 for_each_tracing_cpu(cpu) {
2434 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2435 per_cpu(trace_buffered_event, cpu) = NULL;
2438 * Make sure trace_buffered_event is NULL before clearing
2439 * trace_buffered_event_cnt.
2444 /* Do the work on each cpu */
2445 smp_call_function_many(tracing_buffer_mask,
2446 enable_trace_buffered_event, NULL, 1);
2450 static struct ring_buffer *temp_buffer;
2452 struct ring_buffer_event *
2453 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2454 struct trace_event_file *trace_file,
2455 int type, unsigned long len,
2456 unsigned long flags, int pc)
2458 struct ring_buffer_event *entry;
2461 *current_rb = trace_file->tr->trace_buffer.buffer;
2463 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2464 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2465 (entry = this_cpu_read(trace_buffered_event))) {
2466 /* Try to use the per cpu buffer first */
2467 val = this_cpu_inc_return(trace_buffered_event_cnt);
2469 trace_event_setup(entry, type, flags, pc);
2470 entry->array[0] = len;
2473 this_cpu_dec(trace_buffered_event_cnt);
2476 entry = __trace_buffer_lock_reserve(*current_rb,
2477 type, len, flags, pc);
2479 * If tracing is off, but we have triggers enabled
2480 * we still need to look at the event data. Use the temp_buffer
2481 * to store the trace event for the tigger to use. It's recusive
2482 * safe and will not be recorded anywhere.
2484 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2485 *current_rb = temp_buffer;
2486 entry = __trace_buffer_lock_reserve(*current_rb,
2487 type, len, flags, pc);
2491 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2493 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2494 static DEFINE_MUTEX(tracepoint_printk_mutex);
2496 static void output_printk(struct trace_event_buffer *fbuffer)
2498 struct trace_event_call *event_call;
2499 struct trace_event *event;
2500 unsigned long flags;
2501 struct trace_iterator *iter = tracepoint_print_iter;
2503 /* We should never get here if iter is NULL */
2504 if (WARN_ON_ONCE(!iter))
2507 event_call = fbuffer->trace_file->event_call;
2508 if (!event_call || !event_call->event.funcs ||
2509 !event_call->event.funcs->trace)
2512 event = &fbuffer->trace_file->event_call->event;
2514 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2515 trace_seq_init(&iter->seq);
2516 iter->ent = fbuffer->entry;
2517 event_call->event.funcs->trace(iter, 0, event);
2518 trace_seq_putc(&iter->seq, 0);
2519 printk("%s", iter->seq.buffer);
2521 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2524 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2525 void __user *buffer, size_t *lenp,
2528 int save_tracepoint_printk;
2531 mutex_lock(&tracepoint_printk_mutex);
2532 save_tracepoint_printk = tracepoint_printk;
2534 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2537 * This will force exiting early, as tracepoint_printk
2538 * is always zero when tracepoint_printk_iter is not allocated
2540 if (!tracepoint_print_iter)
2541 tracepoint_printk = 0;
2543 if (save_tracepoint_printk == tracepoint_printk)
2546 if (tracepoint_printk)
2547 static_key_enable(&tracepoint_printk_key.key);
2549 static_key_disable(&tracepoint_printk_key.key);
2552 mutex_unlock(&tracepoint_printk_mutex);
2557 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2559 if (static_key_false(&tracepoint_printk_key.key))
2560 output_printk(fbuffer);
2562 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2563 fbuffer->event, fbuffer->entry,
2564 fbuffer->flags, fbuffer->pc);
2566 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2571 * trace_buffer_unlock_commit_regs()
2572 * trace_event_buffer_commit()
2573 * trace_event_raw_event_xxx()
2575 # define STACK_SKIP 3
2577 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2578 struct ring_buffer *buffer,
2579 struct ring_buffer_event *event,
2580 unsigned long flags, int pc,
2581 struct pt_regs *regs)
2583 __buffer_unlock_commit(buffer, event);
2586 * If regs is not set, then skip the necessary functions.
2587 * Note, we can still get here via blktrace, wakeup tracer
2588 * and mmiotrace, but that's ok if they lose a function or
2589 * two. They are not that meaningful.
2591 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2592 ftrace_trace_userstack(buffer, flags, pc);
2596 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2599 trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2600 struct ring_buffer_event *event)
2602 __buffer_unlock_commit(buffer, event);
2606 trace_process_export(struct trace_export *export,
2607 struct ring_buffer_event *event)
2609 struct trace_entry *entry;
2610 unsigned int size = 0;
2612 entry = ring_buffer_event_data(event);
2613 size = ring_buffer_event_length(event);
2614 export->write(export, entry, size);
2617 static DEFINE_MUTEX(ftrace_export_lock);
2619 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2621 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2623 static inline void ftrace_exports_enable(void)
2625 static_branch_enable(&ftrace_exports_enabled);
2628 static inline void ftrace_exports_disable(void)
2630 static_branch_disable(&ftrace_exports_enabled);
2633 static void ftrace_exports(struct ring_buffer_event *event)
2635 struct trace_export *export;
2637 preempt_disable_notrace();
2639 export = rcu_dereference_raw_notrace(ftrace_exports_list);
2641 trace_process_export(export, event);
2642 export = rcu_dereference_raw_notrace(export->next);
2645 preempt_enable_notrace();
2649 add_trace_export(struct trace_export **list, struct trace_export *export)
2651 rcu_assign_pointer(export->next, *list);
2653 * We are entering export into the list but another
2654 * CPU might be walking that list. We need to make sure
2655 * the export->next pointer is valid before another CPU sees
2656 * the export pointer included into the list.
2658 rcu_assign_pointer(*list, export);
2662 rm_trace_export(struct trace_export **list, struct trace_export *export)
2664 struct trace_export **p;
2666 for (p = list; *p != NULL; p = &(*p)->next)
2673 rcu_assign_pointer(*p, (*p)->next);
2679 add_ftrace_export(struct trace_export **list, struct trace_export *export)
2682 ftrace_exports_enable();
2684 add_trace_export(list, export);
2688 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2692 ret = rm_trace_export(list, export);
2694 ftrace_exports_disable();
2699 int register_ftrace_export(struct trace_export *export)
2701 if (WARN_ON_ONCE(!export->write))
2704 mutex_lock(&ftrace_export_lock);
2706 add_ftrace_export(&ftrace_exports_list, export);
2708 mutex_unlock(&ftrace_export_lock);
2712 EXPORT_SYMBOL_GPL(register_ftrace_export);
2714 int unregister_ftrace_export(struct trace_export *export)
2718 mutex_lock(&ftrace_export_lock);
2720 ret = rm_ftrace_export(&ftrace_exports_list, export);
2722 mutex_unlock(&ftrace_export_lock);
2726 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2729 trace_function(struct trace_array *tr,
2730 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2733 struct trace_event_call *call = &event_function;
2734 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2735 struct ring_buffer_event *event;
2736 struct ftrace_entry *entry;
2738 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2742 entry = ring_buffer_event_data(event);
2744 entry->parent_ip = parent_ip;
2746 if (!call_filter_check_discard(call, entry, buffer, event)) {
2747 if (static_branch_unlikely(&ftrace_exports_enabled))
2748 ftrace_exports(event);
2749 __buffer_unlock_commit(buffer, event);
2753 #ifdef CONFIG_STACKTRACE
2755 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2756 struct ftrace_stack {
2757 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
2760 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2761 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2763 static void __ftrace_trace_stack(struct ring_buffer *buffer,
2764 unsigned long flags,
2765 int skip, int pc, struct pt_regs *regs)
2767 struct trace_event_call *call = &event_kernel_stack;
2768 struct ring_buffer_event *event;
2769 struct stack_entry *entry;
2770 struct stack_trace trace;
2772 int size = FTRACE_STACK_ENTRIES;
2774 trace.nr_entries = 0;
2778 * Add one, for this function and the call to save_stack_trace()
2779 * If regs is set, then these functions will not be in the way.
2781 #ifndef CONFIG_UNWINDER_ORC
2787 * Since events can happen in NMIs there's no safe way to
2788 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2789 * or NMI comes in, it will just have to use the default
2790 * FTRACE_STACK_SIZE.
2792 preempt_disable_notrace();
2794 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
2796 * We don't need any atomic variables, just a barrier.
2797 * If an interrupt comes in, we don't care, because it would
2798 * have exited and put the counter back to what we want.
2799 * We just need a barrier to keep gcc from moving things
2803 if (use_stack == 1) {
2804 trace.entries = this_cpu_ptr(ftrace_stack.calls);
2805 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2808 save_stack_trace_regs(regs, &trace);
2810 save_stack_trace(&trace);
2812 if (trace.nr_entries > size)
2813 size = trace.nr_entries;
2815 /* From now on, use_stack is a boolean */
2818 size *= sizeof(unsigned long);
2820 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2821 sizeof(*entry) + size, flags, pc);
2824 entry = ring_buffer_event_data(event);
2826 memset(&entry->caller, 0, size);
2829 memcpy(&entry->caller, trace.entries,
2830 trace.nr_entries * sizeof(unsigned long));
2832 trace.max_entries = FTRACE_STACK_ENTRIES;
2833 trace.entries = entry->caller;
2835 save_stack_trace_regs(regs, &trace);
2837 save_stack_trace(&trace);
2840 entry->size = trace.nr_entries;
2842 if (!call_filter_check_discard(call, entry, buffer, event))
2843 __buffer_unlock_commit(buffer, event);
2846 /* Again, don't let gcc optimize things here */
2848 __this_cpu_dec(ftrace_stack_reserve);
2849 preempt_enable_notrace();
2853 static inline void ftrace_trace_stack(struct trace_array *tr,
2854 struct ring_buffer *buffer,
2855 unsigned long flags,
2856 int skip, int pc, struct pt_regs *regs)
2858 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2861 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2864 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2867 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2869 if (rcu_is_watching()) {
2870 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2875 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2876 * but if the above rcu_is_watching() failed, then the NMI
2877 * triggered someplace critical, and rcu_irq_enter() should
2878 * not be called from NMI.
2880 if (unlikely(in_nmi()))
2883 rcu_irq_enter_irqson();
2884 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2885 rcu_irq_exit_irqson();
2889 * trace_dump_stack - record a stack back trace in the trace buffer
2890 * @skip: Number of functions to skip (helper handlers)
2892 void trace_dump_stack(int skip)
2894 unsigned long flags;
2896 if (tracing_disabled || tracing_selftest_running)
2899 local_save_flags(flags);
2901 #ifndef CONFIG_UNWINDER_ORC
2902 /* Skip 1 to skip this function. */
2905 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2906 flags, skip, preempt_count(), NULL);
2908 EXPORT_SYMBOL_GPL(trace_dump_stack);
2910 static DEFINE_PER_CPU(int, user_stack_count);
2913 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
2915 struct trace_event_call *call = &event_user_stack;
2916 struct ring_buffer_event *event;
2917 struct userstack_entry *entry;
2918 struct stack_trace trace;
2920 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
2924 * NMIs can not handle page faults, even with fix ups.
2925 * The save user stack can (and often does) fault.
2927 if (unlikely(in_nmi()))
2931 * prevent recursion, since the user stack tracing may
2932 * trigger other kernel events.
2935 if (__this_cpu_read(user_stack_count))
2938 __this_cpu_inc(user_stack_count);
2940 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2941 sizeof(*entry), flags, pc);
2943 goto out_drop_count;
2944 entry = ring_buffer_event_data(event);
2946 entry->tgid = current->tgid;
2947 memset(&entry->caller, 0, sizeof(entry->caller));
2949 trace.nr_entries = 0;
2950 trace.max_entries = FTRACE_STACK_ENTRIES;
2952 trace.entries = entry->caller;
2954 save_stack_trace_user(&trace);
2955 if (!call_filter_check_discard(call, entry, buffer, event))
2956 __buffer_unlock_commit(buffer, event);
2959 __this_cpu_dec(user_stack_count);
2965 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2967 ftrace_trace_userstack(tr, flags, preempt_count());
2971 #endif /* CONFIG_STACKTRACE */
2973 /* created for use with alloc_percpu */
2974 struct trace_buffer_struct {
2976 char buffer[4][TRACE_BUF_SIZE];
2979 static struct trace_buffer_struct *trace_percpu_buffer;
2982 * Thise allows for lockless recording. If we're nested too deeply, then
2983 * this returns NULL.
2985 static char *get_trace_buf(void)
2987 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
2989 if (!buffer || buffer->nesting >= 4)
2994 /* Interrupts must see nesting incremented before we use the buffer */
2996 return &buffer->buffer[buffer->nesting][0];
2999 static void put_trace_buf(void)
3001 /* Don't let the decrement of nesting leak before this */
3003 this_cpu_dec(trace_percpu_buffer->nesting);
3006 static int alloc_percpu_trace_buffer(void)
3008 struct trace_buffer_struct *buffers;
3010 buffers = alloc_percpu(struct trace_buffer_struct);
3011 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
3014 trace_percpu_buffer = buffers;
3018 static int buffers_allocated;
3020 void trace_printk_init_buffers(void)
3022 if (buffers_allocated)
3025 if (alloc_percpu_trace_buffer())
3028 /* trace_printk() is for debug use only. Don't use it in production. */
3031 pr_warn("**********************************************************\n");
3032 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3034 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3036 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3037 pr_warn("** unsafe for production use. **\n");
3039 pr_warn("** If you see this message and you are not debugging **\n");
3040 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3042 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3043 pr_warn("**********************************************************\n");
3045 /* Expand the buffers to set size */
3046 tracing_update_buffers();
3048 buffers_allocated = 1;
3051 * trace_printk_init_buffers() can be called by modules.
3052 * If that happens, then we need to start cmdline recording
3053 * directly here. If the global_trace.buffer is already
3054 * allocated here, then this was called by module code.
3056 if (global_trace.trace_buffer.buffer)
3057 tracing_start_cmdline_record();
3060 void trace_printk_start_comm(void)
3062 /* Start tracing comms if trace printk is set */
3063 if (!buffers_allocated)
3065 tracing_start_cmdline_record();
3068 static void trace_printk_start_stop_comm(int enabled)
3070 if (!buffers_allocated)
3074 tracing_start_cmdline_record();
3076 tracing_stop_cmdline_record();
3080 * trace_vbprintk - write binary msg to tracing buffer
3083 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3085 struct trace_event_call *call = &event_bprint;
3086 struct ring_buffer_event *event;
3087 struct ring_buffer *buffer;
3088 struct trace_array *tr = &global_trace;
3089 struct bprint_entry *entry;
3090 unsigned long flags;
3092 int len = 0, size, pc;
3094 if (unlikely(tracing_selftest_running || tracing_disabled))
3097 /* Don't pollute graph traces with trace_vprintk internals */
3098 pause_graph_tracing();
3100 pc = preempt_count();
3101 preempt_disable_notrace();
3103 tbuffer = get_trace_buf();
3109 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3111 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3114 local_save_flags(flags);
3115 size = sizeof(*entry) + sizeof(u32) * len;
3116 buffer = tr->trace_buffer.buffer;
3117 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3121 entry = ring_buffer_event_data(event);
3125 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3126 if (!call_filter_check_discard(call, entry, buffer, event)) {
3127 __buffer_unlock_commit(buffer, event);
3128 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
3135 preempt_enable_notrace();
3136 unpause_graph_tracing();
3140 EXPORT_SYMBOL_GPL(trace_vbprintk);
3144 __trace_array_vprintk(struct ring_buffer *buffer,
3145 unsigned long ip, const char *fmt, va_list args)
3147 struct trace_event_call *call = &event_print;
3148 struct ring_buffer_event *event;
3149 int len = 0, size, pc;
3150 struct print_entry *entry;
3151 unsigned long flags;
3154 if (tracing_disabled || tracing_selftest_running)
3157 /* Don't pollute graph traces with trace_vprintk internals */
3158 pause_graph_tracing();
3160 pc = preempt_count();
3161 preempt_disable_notrace();
3164 tbuffer = get_trace_buf();
3170 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3172 local_save_flags(flags);
3173 size = sizeof(*entry) + len + 1;
3174 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3178 entry = ring_buffer_event_data(event);
3181 memcpy(&entry->buf, tbuffer, len + 1);
3182 if (!call_filter_check_discard(call, entry, buffer, event)) {
3183 __buffer_unlock_commit(buffer, event);
3184 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3191 preempt_enable_notrace();
3192 unpause_graph_tracing();
3198 int trace_array_vprintk(struct trace_array *tr,
3199 unsigned long ip, const char *fmt, va_list args)
3201 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
3205 int trace_array_printk(struct trace_array *tr,
3206 unsigned long ip, const char *fmt, ...)
3211 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3215 ret = trace_array_vprintk(tr, ip, fmt, ap);
3221 int trace_array_printk_buf(struct ring_buffer *buffer,
3222 unsigned long ip, const char *fmt, ...)
3227 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3231 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3237 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3239 return trace_array_vprintk(&global_trace, ip, fmt, args);
3241 EXPORT_SYMBOL_GPL(trace_vprintk);
3243 static void trace_iterator_increment(struct trace_iterator *iter)
3245 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3249 ring_buffer_read(buf_iter, NULL);
3252 static struct trace_entry *
3253 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3254 unsigned long *lost_events)
3256 struct ring_buffer_event *event;
3257 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3260 event = ring_buffer_iter_peek(buf_iter, ts);
3262 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
3266 iter->ent_size = ring_buffer_event_length(event);
3267 return ring_buffer_event_data(event);
3273 static struct trace_entry *
3274 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3275 unsigned long *missing_events, u64 *ent_ts)
3277 struct ring_buffer *buffer = iter->trace_buffer->buffer;
3278 struct trace_entry *ent, *next = NULL;
3279 unsigned long lost_events = 0, next_lost = 0;
3280 int cpu_file = iter->cpu_file;
3281 u64 next_ts = 0, ts;
3287 * If we are in a per_cpu trace file, don't bother by iterating over
3288 * all cpu and peek directly.
3290 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3291 if (ring_buffer_empty_cpu(buffer, cpu_file))
3293 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3295 *ent_cpu = cpu_file;
3300 for_each_tracing_cpu(cpu) {
3302 if (ring_buffer_empty_cpu(buffer, cpu))
3305 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3308 * Pick the entry with the smallest timestamp:
3310 if (ent && (!next || ts < next_ts)) {
3314 next_lost = lost_events;
3315 next_size = iter->ent_size;
3319 iter->ent_size = next_size;
3322 *ent_cpu = next_cpu;
3328 *missing_events = next_lost;
3333 /* Find the next real entry, without updating the iterator itself */
3334 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3335 int *ent_cpu, u64 *ent_ts)
3337 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3340 /* Find the next real entry, and increment the iterator to the next entry */
3341 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3343 iter->ent = __find_next_entry(iter, &iter->cpu,
3344 &iter->lost_events, &iter->ts);
3347 trace_iterator_increment(iter);
3349 return iter->ent ? iter : NULL;
3352 static void trace_consume(struct trace_iterator *iter)
3354 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
3355 &iter->lost_events);
3358 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3360 struct trace_iterator *iter = m->private;
3364 WARN_ON_ONCE(iter->leftover);
3368 /* can't go backwards */
3373 ent = trace_find_next_entry_inc(iter);
3377 while (ent && iter->idx < i)
3378 ent = trace_find_next_entry_inc(iter);
3385 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3387 struct ring_buffer_event *event;
3388 struct ring_buffer_iter *buf_iter;
3389 unsigned long entries = 0;
3392 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
3394 buf_iter = trace_buffer_iter(iter, cpu);
3398 ring_buffer_iter_reset(buf_iter);
3401 * We could have the case with the max latency tracers
3402 * that a reset never took place on a cpu. This is evident
3403 * by the timestamp being before the start of the buffer.
3405 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3406 if (ts >= iter->trace_buffer->time_start)
3409 ring_buffer_read(buf_iter, NULL);
3412 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
3416 * The current tracer is copied to avoid a global locking
3419 static void *s_start(struct seq_file *m, loff_t *pos)
3421 struct trace_iterator *iter = m->private;
3422 struct trace_array *tr = iter->tr;
3423 int cpu_file = iter->cpu_file;
3429 * copy the tracer to avoid using a global lock all around.
3430 * iter->trace is a copy of current_trace, the pointer to the
3431 * name may be used instead of a strcmp(), as iter->trace->name
3432 * will point to the same string as current_trace->name.
3434 mutex_lock(&trace_types_lock);
3435 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3436 *iter->trace = *tr->current_trace;
3437 mutex_unlock(&trace_types_lock);
3439 #ifdef CONFIG_TRACER_MAX_TRACE
3440 if (iter->snapshot && iter->trace->use_max_tr)
3441 return ERR_PTR(-EBUSY);
3444 if (!iter->snapshot)
3445 atomic_inc(&trace_record_taskinfo_disabled);
3447 if (*pos != iter->pos) {
3452 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3453 for_each_tracing_cpu(cpu)
3454 tracing_iter_reset(iter, cpu);
3456 tracing_iter_reset(iter, cpu_file);
3459 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3464 * If we overflowed the seq_file before, then we want
3465 * to just reuse the trace_seq buffer again.
3471 p = s_next(m, p, &l);
3475 trace_event_read_lock();
3476 trace_access_lock(cpu_file);
3480 static void s_stop(struct seq_file *m, void *p)
3482 struct trace_iterator *iter = m->private;
3484 #ifdef CONFIG_TRACER_MAX_TRACE
3485 if (iter->snapshot && iter->trace->use_max_tr)
3489 if (!iter->snapshot)
3490 atomic_dec(&trace_record_taskinfo_disabled);
3492 trace_access_unlock(iter->cpu_file);
3493 trace_event_read_unlock();
3497 get_total_entries(struct trace_buffer *buf,
3498 unsigned long *total, unsigned long *entries)
3500 unsigned long count;
3506 for_each_tracing_cpu(cpu) {
3507 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3509 * If this buffer has skipped entries, then we hold all
3510 * entries for the trace and we need to ignore the
3511 * ones before the time stamp.
3513 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3514 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3515 /* total is the same as the entries */
3519 ring_buffer_overrun_cpu(buf->buffer, cpu);
3524 static void print_lat_help_header(struct seq_file *m)
3526 seq_puts(m, "# _------=> CPU# \n"
3527 "# / _-----=> irqs-off \n"
3528 "# | / _----=> need-resched \n"
3529 "# || / _---=> hardirq/softirq \n"
3530 "# ||| / _--=> preempt-depth \n"
3532 "# cmd pid ||||| time | caller \n"
3533 "# \\ / ||||| \\ | / \n");
3536 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
3538 unsigned long total;
3539 unsigned long entries;
3541 get_total_entries(buf, &total, &entries);
3542 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3543 entries, total, num_online_cpus());
3547 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m,
3550 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3552 print_event_info(buf, m);
3554 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3555 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
3558 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m,
3561 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3562 const char tgid_space[] = " ";
3563 const char space[] = " ";
3565 print_event_info(buf, m);
3567 seq_printf(m, "# %s _-----=> irqs-off\n",
3568 tgid ? tgid_space : space);
3569 seq_printf(m, "# %s / _----=> need-resched\n",
3570 tgid ? tgid_space : space);
3571 seq_printf(m, "# %s| / _---=> hardirq/softirq\n",
3572 tgid ? tgid_space : space);
3573 seq_printf(m, "# %s|| / _--=> preempt-depth\n",
3574 tgid ? tgid_space : space);
3575 seq_printf(m, "# %s||| / delay\n",
3576 tgid ? tgid_space : space);
3577 seq_printf(m, "# TASK-PID %sCPU# |||| TIMESTAMP FUNCTION\n",
3578 tgid ? " TGID " : space);
3579 seq_printf(m, "# | | %s | |||| | |\n",
3580 tgid ? " | " : space);
3584 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3586 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3587 struct trace_buffer *buf = iter->trace_buffer;
3588 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3589 struct tracer *type = iter->trace;
3590 unsigned long entries;
3591 unsigned long total;
3592 const char *name = "preemption";
3596 get_total_entries(buf, &total, &entries);
3598 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3600 seq_puts(m, "# -----------------------------------"
3601 "---------------------------------\n");
3602 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3603 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3604 nsecs_to_usecs(data->saved_latency),
3608 #if defined(CONFIG_PREEMPT_NONE)
3610 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3612 #elif defined(CONFIG_PREEMPT)
3617 /* These are reserved for later use */
3620 seq_printf(m, " #P:%d)\n", num_online_cpus());
3624 seq_puts(m, "# -----------------\n");
3625 seq_printf(m, "# | task: %.16s-%d "
3626 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3627 data->comm, data->pid,
3628 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3629 data->policy, data->rt_priority);
3630 seq_puts(m, "# -----------------\n");
3632 if (data->critical_start) {
3633 seq_puts(m, "# => started at: ");
3634 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3635 trace_print_seq(m, &iter->seq);
3636 seq_puts(m, "\n# => ended at: ");
3637 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3638 trace_print_seq(m, &iter->seq);
3639 seq_puts(m, "\n#\n");
3645 static void test_cpu_buff_start(struct trace_iterator *iter)
3647 struct trace_seq *s = &iter->seq;
3648 struct trace_array *tr = iter->tr;
3650 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3653 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3656 if (cpumask_available(iter->started) &&
3657 cpumask_test_cpu(iter->cpu, iter->started))
3660 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3663 if (cpumask_available(iter->started))
3664 cpumask_set_cpu(iter->cpu, iter->started);
3666 /* Don't print started cpu buffer for the first entry of the trace */
3668 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3672 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3674 struct trace_array *tr = iter->tr;
3675 struct trace_seq *s = &iter->seq;
3676 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3677 struct trace_entry *entry;
3678 struct trace_event *event;
3682 test_cpu_buff_start(iter);
3684 event = ftrace_find_event(entry->type);
3686 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3687 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3688 trace_print_lat_context(iter);
3690 trace_print_context(iter);
3693 if (trace_seq_has_overflowed(s))
3694 return TRACE_TYPE_PARTIAL_LINE;
3697 return event->funcs->trace(iter, sym_flags, event);
3699 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3701 return trace_handle_return(s);
3704 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3706 struct trace_array *tr = iter->tr;
3707 struct trace_seq *s = &iter->seq;
3708 struct trace_entry *entry;
3709 struct trace_event *event;
3713 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3714 trace_seq_printf(s, "%d %d %llu ",
3715 entry->pid, iter->cpu, iter->ts);
3717 if (trace_seq_has_overflowed(s))
3718 return TRACE_TYPE_PARTIAL_LINE;
3720 event = ftrace_find_event(entry->type);
3722 return event->funcs->raw(iter, 0, event);
3724 trace_seq_printf(s, "%d ?\n", entry->type);
3726 return trace_handle_return(s);
3729 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3731 struct trace_array *tr = iter->tr;
3732 struct trace_seq *s = &iter->seq;
3733 unsigned char newline = '\n';
3734 struct trace_entry *entry;
3735 struct trace_event *event;
3739 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3740 SEQ_PUT_HEX_FIELD(s, entry->pid);
3741 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3742 SEQ_PUT_HEX_FIELD(s, iter->ts);
3743 if (trace_seq_has_overflowed(s))
3744 return TRACE_TYPE_PARTIAL_LINE;
3747 event = ftrace_find_event(entry->type);
3749 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3750 if (ret != TRACE_TYPE_HANDLED)
3754 SEQ_PUT_FIELD(s, newline);
3756 return trace_handle_return(s);
3759 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3761 struct trace_array *tr = iter->tr;
3762 struct trace_seq *s = &iter->seq;
3763 struct trace_entry *entry;
3764 struct trace_event *event;
3768 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3769 SEQ_PUT_FIELD(s, entry->pid);
3770 SEQ_PUT_FIELD(s, iter->cpu);
3771 SEQ_PUT_FIELD(s, iter->ts);
3772 if (trace_seq_has_overflowed(s))
3773 return TRACE_TYPE_PARTIAL_LINE;
3776 event = ftrace_find_event(entry->type);
3777 return event ? event->funcs->binary(iter, 0, event) :
3781 int trace_empty(struct trace_iterator *iter)
3783 struct ring_buffer_iter *buf_iter;
3786 /* If we are looking at one CPU buffer, only check that one */
3787 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3788 cpu = iter->cpu_file;
3789 buf_iter = trace_buffer_iter(iter, cpu);
3791 if (!ring_buffer_iter_empty(buf_iter))
3794 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3800 for_each_tracing_cpu(cpu) {
3801 buf_iter = trace_buffer_iter(iter, cpu);
3803 if (!ring_buffer_iter_empty(buf_iter))
3806 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3814 /* Called with trace_event_read_lock() held. */
3815 enum print_line_t print_trace_line(struct trace_iterator *iter)
3817 struct trace_array *tr = iter->tr;
3818 unsigned long trace_flags = tr->trace_flags;
3819 enum print_line_t ret;
3821 if (iter->lost_events) {
3822 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3823 iter->cpu, iter->lost_events);
3824 if (trace_seq_has_overflowed(&iter->seq))
3825 return TRACE_TYPE_PARTIAL_LINE;
3828 if (iter->trace && iter->trace->print_line) {
3829 ret = iter->trace->print_line(iter);
3830 if (ret != TRACE_TYPE_UNHANDLED)
3834 if (iter->ent->type == TRACE_BPUTS &&
3835 trace_flags & TRACE_ITER_PRINTK &&
3836 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3837 return trace_print_bputs_msg_only(iter);
3839 if (iter->ent->type == TRACE_BPRINT &&
3840 trace_flags & TRACE_ITER_PRINTK &&
3841 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3842 return trace_print_bprintk_msg_only(iter);
3844 if (iter->ent->type == TRACE_PRINT &&
3845 trace_flags & TRACE_ITER_PRINTK &&
3846 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3847 return trace_print_printk_msg_only(iter);
3849 if (trace_flags & TRACE_ITER_BIN)
3850 return print_bin_fmt(iter);
3852 if (trace_flags & TRACE_ITER_HEX)
3853 return print_hex_fmt(iter);
3855 if (trace_flags & TRACE_ITER_RAW)
3856 return print_raw_fmt(iter);
3858 return print_trace_fmt(iter);
3861 void trace_latency_header(struct seq_file *m)
3863 struct trace_iterator *iter = m->private;
3864 struct trace_array *tr = iter->tr;
3866 /* print nothing if the buffers are empty */
3867 if (trace_empty(iter))
3870 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3871 print_trace_header(m, iter);
3873 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
3874 print_lat_help_header(m);
3877 void trace_default_header(struct seq_file *m)
3879 struct trace_iterator *iter = m->private;
3880 struct trace_array *tr = iter->tr;
3881 unsigned long trace_flags = tr->trace_flags;
3883 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3886 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3887 /* print nothing if the buffers are empty */
3888 if (trace_empty(iter))
3890 print_trace_header(m, iter);
3891 if (!(trace_flags & TRACE_ITER_VERBOSE))
3892 print_lat_help_header(m);
3894 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3895 if (trace_flags & TRACE_ITER_IRQ_INFO)
3896 print_func_help_header_irq(iter->trace_buffer,
3899 print_func_help_header(iter->trace_buffer, m,
3905 static void test_ftrace_alive(struct seq_file *m)
3907 if (!ftrace_is_dead())
3909 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3910 "# MAY BE MISSING FUNCTION EVENTS\n");
3913 #ifdef CONFIG_TRACER_MAX_TRACE
3914 static void show_snapshot_main_help(struct seq_file *m)
3916 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3917 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3918 "# Takes a snapshot of the main buffer.\n"
3919 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3920 "# (Doesn't have to be '2' works with any number that\n"
3921 "# is not a '0' or '1')\n");
3924 static void show_snapshot_percpu_help(struct seq_file *m)
3926 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3927 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3928 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3929 "# Takes a snapshot of the main buffer for this cpu.\n");
3931 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3932 "# Must use main snapshot file to allocate.\n");
3934 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3935 "# (Doesn't have to be '2' works with any number that\n"
3936 "# is not a '0' or '1')\n");
3939 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3941 if (iter->tr->allocated_snapshot)
3942 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3944 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3946 seq_puts(m, "# Snapshot commands:\n");
3947 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3948 show_snapshot_main_help(m);
3950 show_snapshot_percpu_help(m);
3953 /* Should never be called */
3954 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3957 static int s_show(struct seq_file *m, void *v)
3959 struct trace_iterator *iter = v;
3962 if (iter->ent == NULL) {
3964 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3966 test_ftrace_alive(m);
3968 if (iter->snapshot && trace_empty(iter))
3969 print_snapshot_help(m, iter);
3970 else if (iter->trace && iter->trace->print_header)
3971 iter->trace->print_header(m);
3973 trace_default_header(m);
3975 } else if (iter->leftover) {
3977 * If we filled the seq_file buffer earlier, we
3978 * want to just show it now.
3980 ret = trace_print_seq(m, &iter->seq);
3982 /* ret should this time be zero, but you never know */
3983 iter->leftover = ret;
3986 print_trace_line(iter);
3987 ret = trace_print_seq(m, &iter->seq);
3989 * If we overflow the seq_file buffer, then it will
3990 * ask us for this data again at start up.
3992 * ret is 0 if seq_file write succeeded.
3995 iter->leftover = ret;
4002 * Should be used after trace_array_get(), trace_types_lock
4003 * ensures that i_cdev was already initialized.
4005 static inline int tracing_get_cpu(struct inode *inode)
4007 if (inode->i_cdev) /* See trace_create_cpu_file() */
4008 return (long)inode->i_cdev - 1;
4009 return RING_BUFFER_ALL_CPUS;
4012 static const struct seq_operations tracer_seq_ops = {
4019 static struct trace_iterator *
4020 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
4022 struct trace_array *tr = inode->i_private;
4023 struct trace_iterator *iter;
4026 if (tracing_disabled)
4027 return ERR_PTR(-ENODEV);
4029 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4031 return ERR_PTR(-ENOMEM);
4033 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
4035 if (!iter->buffer_iter)
4039 * We make a copy of the current tracer to avoid concurrent
4040 * changes on it while we are reading.
4042 mutex_lock(&trace_types_lock);
4043 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
4047 *iter->trace = *tr->current_trace;
4049 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
4054 #ifdef CONFIG_TRACER_MAX_TRACE
4055 /* Currently only the top directory has a snapshot */
4056 if (tr->current_trace->print_max || snapshot)
4057 iter->trace_buffer = &tr->max_buffer;
4060 iter->trace_buffer = &tr->trace_buffer;
4061 iter->snapshot = snapshot;
4063 iter->cpu_file = tracing_get_cpu(inode);
4064 mutex_init(&iter->mutex);
4066 /* Notify the tracer early; before we stop tracing. */
4067 if (iter->trace && iter->trace->open)
4068 iter->trace->open(iter);
4070 /* Annotate start of buffers if we had overruns */
4071 if (ring_buffer_overruns(iter->trace_buffer->buffer))
4072 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4074 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4075 if (trace_clocks[tr->clock_id].in_ns)
4076 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4078 /* stop the trace while dumping if we are not opening "snapshot" */
4079 if (!iter->snapshot)
4080 tracing_stop_tr(tr);
4082 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4083 for_each_tracing_cpu(cpu) {
4084 iter->buffer_iter[cpu] =
4085 ring_buffer_read_prepare(iter->trace_buffer->buffer,
4088 ring_buffer_read_prepare_sync();
4089 for_each_tracing_cpu(cpu) {
4090 ring_buffer_read_start(iter->buffer_iter[cpu]);
4091 tracing_iter_reset(iter, cpu);
4094 cpu = iter->cpu_file;
4095 iter->buffer_iter[cpu] =
4096 ring_buffer_read_prepare(iter->trace_buffer->buffer,
4098 ring_buffer_read_prepare_sync();
4099 ring_buffer_read_start(iter->buffer_iter[cpu]);
4100 tracing_iter_reset(iter, cpu);
4103 mutex_unlock(&trace_types_lock);
4108 mutex_unlock(&trace_types_lock);
4110 kfree(iter->buffer_iter);
4112 seq_release_private(inode, file);
4113 return ERR_PTR(-ENOMEM);
4116 int tracing_open_generic(struct inode *inode, struct file *filp)
4118 if (tracing_disabled)
4121 filp->private_data = inode->i_private;
4125 bool tracing_is_disabled(void)
4127 return (tracing_disabled) ? true: false;
4131 * Open and update trace_array ref count.
4132 * Must have the current trace_array passed to it.
4134 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4136 struct trace_array *tr = inode->i_private;
4138 if (tracing_disabled)
4141 if (trace_array_get(tr) < 0)
4144 filp->private_data = inode->i_private;
4149 static int tracing_release(struct inode *inode, struct file *file)
4151 struct trace_array *tr = inode->i_private;
4152 struct seq_file *m = file->private_data;
4153 struct trace_iterator *iter;
4156 if (!(file->f_mode & FMODE_READ)) {
4157 trace_array_put(tr);
4161 /* Writes do not use seq_file */
4163 mutex_lock(&trace_types_lock);
4165 for_each_tracing_cpu(cpu) {
4166 if (iter->buffer_iter[cpu])
4167 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4170 if (iter->trace && iter->trace->close)
4171 iter->trace->close(iter);
4173 if (!iter->snapshot)
4174 /* reenable tracing if it was previously enabled */
4175 tracing_start_tr(tr);
4177 __trace_array_put(tr);
4179 mutex_unlock(&trace_types_lock);
4181 mutex_destroy(&iter->mutex);
4182 free_cpumask_var(iter->started);
4184 kfree(iter->buffer_iter);
4185 seq_release_private(inode, file);
4190 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4192 struct trace_array *tr = inode->i_private;
4194 trace_array_put(tr);
4198 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4200 struct trace_array *tr = inode->i_private;
4202 trace_array_put(tr);
4204 return single_release(inode, file);
4207 static int tracing_open(struct inode *inode, struct file *file)
4209 struct trace_array *tr = inode->i_private;
4210 struct trace_iterator *iter;
4213 if (trace_array_get(tr) < 0)
4216 /* If this file was open for write, then erase contents */
4217 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4218 int cpu = tracing_get_cpu(inode);
4219 struct trace_buffer *trace_buf = &tr->trace_buffer;
4221 #ifdef CONFIG_TRACER_MAX_TRACE
4222 if (tr->current_trace->print_max)
4223 trace_buf = &tr->max_buffer;
4226 if (cpu == RING_BUFFER_ALL_CPUS)
4227 tracing_reset_online_cpus(trace_buf);
4229 tracing_reset(trace_buf, cpu);
4232 if (file->f_mode & FMODE_READ) {
4233 iter = __tracing_open(inode, file, false);
4235 ret = PTR_ERR(iter);
4236 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4237 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4241 trace_array_put(tr);
4247 * Some tracers are not suitable for instance buffers.
4248 * A tracer is always available for the global array (toplevel)
4249 * or if it explicitly states that it is.
4252 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4254 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4257 /* Find the next tracer that this trace array may use */
4258 static struct tracer *
4259 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4261 while (t && !trace_ok_for_array(t, tr))
4268 t_next(struct seq_file *m, void *v, loff_t *pos)
4270 struct trace_array *tr = m->private;
4271 struct tracer *t = v;
4276 t = get_tracer_for_array(tr, t->next);
4281 static void *t_start(struct seq_file *m, loff_t *pos)
4283 struct trace_array *tr = m->private;
4287 mutex_lock(&trace_types_lock);
4289 t = get_tracer_for_array(tr, trace_types);
4290 for (; t && l < *pos; t = t_next(m, t, &l))
4296 static void t_stop(struct seq_file *m, void *p)
4298 mutex_unlock(&trace_types_lock);
4301 static int t_show(struct seq_file *m, void *v)
4303 struct tracer *t = v;
4308 seq_puts(m, t->name);
4317 static const struct seq_operations show_traces_seq_ops = {
4324 static int show_traces_open(struct inode *inode, struct file *file)
4326 struct trace_array *tr = inode->i_private;
4330 if (tracing_disabled)
4333 ret = seq_open(file, &show_traces_seq_ops);
4337 m = file->private_data;
4344 tracing_write_stub(struct file *filp, const char __user *ubuf,
4345 size_t count, loff_t *ppos)
4350 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4354 if (file->f_mode & FMODE_READ)
4355 ret = seq_lseek(file, offset, whence);
4357 file->f_pos = ret = 0;
4362 static const struct file_operations tracing_fops = {
4363 .open = tracing_open,
4365 .write = tracing_write_stub,
4366 .llseek = tracing_lseek,
4367 .release = tracing_release,
4370 static const struct file_operations show_traces_fops = {
4371 .open = show_traces_open,
4373 .release = seq_release,
4374 .llseek = seq_lseek,
4378 tracing_cpumask_read(struct file *filp, char __user *ubuf,
4379 size_t count, loff_t *ppos)
4381 struct trace_array *tr = file_inode(filp)->i_private;
4385 len = snprintf(NULL, 0, "%*pb\n",
4386 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4387 mask_str = kmalloc(len, GFP_KERNEL);
4391 len = snprintf(mask_str, len, "%*pb\n",
4392 cpumask_pr_args(tr->tracing_cpumask));
4397 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4406 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4407 size_t count, loff_t *ppos)
4409 struct trace_array *tr = file_inode(filp)->i_private;
4410 cpumask_var_t tracing_cpumask_new;
4413 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4416 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4420 local_irq_disable();
4421 arch_spin_lock(&tr->max_lock);
4422 for_each_tracing_cpu(cpu) {
4424 * Increase/decrease the disabled counter if we are
4425 * about to flip a bit in the cpumask:
4427 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4428 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4429 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4430 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
4432 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4433 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4434 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4435 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
4438 arch_spin_unlock(&tr->max_lock);
4441 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4442 free_cpumask_var(tracing_cpumask_new);
4447 free_cpumask_var(tracing_cpumask_new);
4452 static const struct file_operations tracing_cpumask_fops = {
4453 .open = tracing_open_generic_tr,
4454 .read = tracing_cpumask_read,
4455 .write = tracing_cpumask_write,
4456 .release = tracing_release_generic_tr,
4457 .llseek = generic_file_llseek,
4460 static int tracing_trace_options_show(struct seq_file *m, void *v)
4462 struct tracer_opt *trace_opts;
4463 struct trace_array *tr = m->private;
4467 mutex_lock(&trace_types_lock);
4468 tracer_flags = tr->current_trace->flags->val;
4469 trace_opts = tr->current_trace->flags->opts;
4471 for (i = 0; trace_options[i]; i++) {
4472 if (tr->trace_flags & (1 << i))
4473 seq_printf(m, "%s\n", trace_options[i]);
4475 seq_printf(m, "no%s\n", trace_options[i]);
4478 for (i = 0; trace_opts[i].name; i++) {
4479 if (tracer_flags & trace_opts[i].bit)
4480 seq_printf(m, "%s\n", trace_opts[i].name);
4482 seq_printf(m, "no%s\n", trace_opts[i].name);
4484 mutex_unlock(&trace_types_lock);
4489 static int __set_tracer_option(struct trace_array *tr,
4490 struct tracer_flags *tracer_flags,
4491 struct tracer_opt *opts, int neg)
4493 struct tracer *trace = tracer_flags->trace;
4496 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4501 tracer_flags->val &= ~opts->bit;
4503 tracer_flags->val |= opts->bit;
4507 /* Try to assign a tracer specific option */
4508 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4510 struct tracer *trace = tr->current_trace;
4511 struct tracer_flags *tracer_flags = trace->flags;
4512 struct tracer_opt *opts = NULL;
4515 for (i = 0; tracer_flags->opts[i].name; i++) {
4516 opts = &tracer_flags->opts[i];
4518 if (strcmp(cmp, opts->name) == 0)
4519 return __set_tracer_option(tr, trace->flags, opts, neg);
4525 /* Some tracers require overwrite to stay enabled */
4526 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4528 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4534 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4536 /* do nothing if flag is already set */
4537 if (!!(tr->trace_flags & mask) == !!enabled)
4540 /* Give the tracer a chance to approve the change */
4541 if (tr->current_trace->flag_changed)
4542 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4546 tr->trace_flags |= mask;
4548 tr->trace_flags &= ~mask;
4550 if (mask == TRACE_ITER_RECORD_CMD)
4551 trace_event_enable_cmd_record(enabled);
4553 if (mask == TRACE_ITER_RECORD_TGID) {
4555 tgid_map = kcalloc(PID_MAX_DEFAULT + 1,
4559 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4563 trace_event_enable_tgid_record(enabled);
4566 if (mask == TRACE_ITER_EVENT_FORK)
4567 trace_event_follow_fork(tr, enabled);
4569 if (mask == TRACE_ITER_FUNC_FORK)
4570 ftrace_pid_follow_fork(tr, enabled);
4572 if (mask == TRACE_ITER_OVERWRITE) {
4573 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
4574 #ifdef CONFIG_TRACER_MAX_TRACE
4575 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4579 if (mask == TRACE_ITER_PRINTK) {
4580 trace_printk_start_stop_comm(enabled);
4581 trace_printk_control(enabled);
4587 static int trace_set_options(struct trace_array *tr, char *option)
4592 size_t orig_len = strlen(option);
4595 cmp = strstrip(option);
4597 len = str_has_prefix(cmp, "no");
4603 mutex_lock(&trace_types_lock);
4605 ret = match_string(trace_options, -1, cmp);
4606 /* If no option could be set, test the specific tracer options */
4608 ret = set_tracer_option(tr, cmp, neg);
4610 ret = set_tracer_flag(tr, 1 << ret, !neg);
4612 mutex_unlock(&trace_types_lock);
4615 * If the first trailing whitespace is replaced with '\0' by strstrip,
4616 * turn it back into a space.
4618 if (orig_len > strlen(option))
4619 option[strlen(option)] = ' ';
4624 static void __init apply_trace_boot_options(void)
4626 char *buf = trace_boot_options_buf;
4630 option = strsep(&buf, ",");
4636 trace_set_options(&global_trace, option);
4638 /* Put back the comma to allow this to be called again */
4645 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4646 size_t cnt, loff_t *ppos)
4648 struct seq_file *m = filp->private_data;
4649 struct trace_array *tr = m->private;
4653 if (cnt >= sizeof(buf))
4656 if (copy_from_user(buf, ubuf, cnt))
4661 ret = trace_set_options(tr, buf);
4670 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4672 struct trace_array *tr = inode->i_private;
4675 if (tracing_disabled)
4678 if (trace_array_get(tr) < 0)
4681 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4683 trace_array_put(tr);
4688 static const struct file_operations tracing_iter_fops = {
4689 .open = tracing_trace_options_open,
4691 .llseek = seq_lseek,
4692 .release = tracing_single_release_tr,
4693 .write = tracing_trace_options_write,
4696 static const char readme_msg[] =
4697 "tracing mini-HOWTO:\n\n"
4698 "# echo 0 > tracing_on : quick way to disable tracing\n"
4699 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4700 " Important files:\n"
4701 " trace\t\t\t- The static contents of the buffer\n"
4702 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4703 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4704 " current_tracer\t- function and latency tracers\n"
4705 " available_tracers\t- list of configured tracers for current_tracer\n"
4706 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4707 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4708 " trace_clock\t\t-change the clock used to order events\n"
4709 " local: Per cpu clock but may not be synced across CPUs\n"
4710 " global: Synced across CPUs but slows tracing down.\n"
4711 " counter: Not a clock, but just an increment\n"
4712 " uptime: Jiffy counter from time of boot\n"
4713 " perf: Same clock that perf events use\n"
4714 #ifdef CONFIG_X86_64
4715 " x86-tsc: TSC cycle counter\n"
4717 "\n timestamp_mode\t-view the mode used to timestamp events\n"
4718 " delta: Delta difference against a buffer-wide timestamp\n"
4719 " absolute: Absolute (standalone) timestamp\n"
4720 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4721 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4722 " tracing_cpumask\t- Limit which CPUs to trace\n"
4723 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4724 "\t\t\t Remove sub-buffer with rmdir\n"
4725 " trace_options\t\t- Set format or modify how tracing happens\n"
4726 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4727 "\t\t\t option name\n"
4728 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4729 #ifdef CONFIG_DYNAMIC_FTRACE
4730 "\n available_filter_functions - list of functions that can be filtered on\n"
4731 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4732 "\t\t\t functions\n"
4733 "\t accepts: func_full_name or glob-matching-pattern\n"
4734 "\t modules: Can select a group via module\n"
4735 "\t Format: :mod:<module-name>\n"
4736 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4737 "\t triggers: a command to perform when function is hit\n"
4738 "\t Format: <function>:<trigger>[:count]\n"
4739 "\t trigger: traceon, traceoff\n"
4740 "\t\t enable_event:<system>:<event>\n"
4741 "\t\t disable_event:<system>:<event>\n"
4742 #ifdef CONFIG_STACKTRACE
4745 #ifdef CONFIG_TRACER_SNAPSHOT
4750 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4751 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4752 "\t The first one will disable tracing every time do_fault is hit\n"
4753 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4754 "\t The first time do trap is hit and it disables tracing, the\n"
4755 "\t counter will decrement to 2. If tracing is already disabled,\n"
4756 "\t the counter will not decrement. It only decrements when the\n"
4757 "\t trigger did work\n"
4758 "\t To remove trigger without count:\n"
4759 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4760 "\t To remove trigger with a count:\n"
4761 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4762 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4763 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4764 "\t modules: Can select a group via module command :mod:\n"
4765 "\t Does not accept triggers\n"
4766 #endif /* CONFIG_DYNAMIC_FTRACE */
4767 #ifdef CONFIG_FUNCTION_TRACER
4768 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4771 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4772 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4773 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4774 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4776 #ifdef CONFIG_TRACER_SNAPSHOT
4777 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4778 "\t\t\t snapshot buffer. Read the contents for more\n"
4779 "\t\t\t information\n"
4781 #ifdef CONFIG_STACK_TRACER
4782 " stack_trace\t\t- Shows the max stack trace when active\n"
4783 " stack_max_size\t- Shows current max stack size that was traced\n"
4784 "\t\t\t Write into this file to reset the max size (trigger a\n"
4785 "\t\t\t new trace)\n"
4786 #ifdef CONFIG_DYNAMIC_FTRACE
4787 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4790 #endif /* CONFIG_STACK_TRACER */
4791 #ifdef CONFIG_DYNAMIC_EVENTS
4792 " dynamic_events\t\t- Add/remove/show the generic dynamic events\n"
4793 "\t\t\t Write into this file to define/undefine new trace events.\n"
4795 #ifdef CONFIG_KPROBE_EVENTS
4796 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4797 "\t\t\t Write into this file to define/undefine new trace events.\n"
4799 #ifdef CONFIG_UPROBE_EVENTS
4800 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4801 "\t\t\t Write into this file to define/undefine new trace events.\n"
4803 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4804 "\t accepts: event-definitions (one definition per line)\n"
4805 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4806 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
4807 #ifdef CONFIG_HIST_TRIGGERS
4808 "\t s:[synthetic/]<event> <field> [<field>]\n"
4810 "\t -:[<group>/]<event>\n"
4811 #ifdef CONFIG_KPROBE_EVENTS
4812 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4813 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4815 #ifdef CONFIG_UPROBE_EVENTS
4816 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
4818 "\t args: <name>=fetcharg[:type]\n"
4819 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4820 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4821 "\t $stack<index>, $stack, $retval, $comm, $arg<N>\n"
4823 "\t $stack<index>, $stack, $retval, $comm\n"
4825 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
4826 "\t b<bit-width>@<bit-offset>/<container-size>,\n"
4827 "\t <type>\\[<array-size>\\]\n"
4828 #ifdef CONFIG_HIST_TRIGGERS
4829 "\t field: <stype> <name>;\n"
4830 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4831 "\t [unsigned] char/int/long\n"
4834 " events/\t\t- Directory containing all trace event subsystems:\n"
4835 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4836 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4837 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4839 " filter\t\t- If set, only events passing filter are traced\n"
4840 " events/<system>/<event>/\t- Directory containing control files for\n"
4842 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4843 " filter\t\t- If set, only events passing filter are traced\n"
4844 " trigger\t\t- If set, a command to perform when event is hit\n"
4845 "\t Format: <trigger>[:count][if <filter>]\n"
4846 "\t trigger: traceon, traceoff\n"
4847 "\t enable_event:<system>:<event>\n"
4848 "\t disable_event:<system>:<event>\n"
4849 #ifdef CONFIG_HIST_TRIGGERS
4850 "\t enable_hist:<system>:<event>\n"
4851 "\t disable_hist:<system>:<event>\n"
4853 #ifdef CONFIG_STACKTRACE
4856 #ifdef CONFIG_TRACER_SNAPSHOT
4859 #ifdef CONFIG_HIST_TRIGGERS
4860 "\t\t hist (see below)\n"
4862 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4863 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4864 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4865 "\t events/block/block_unplug/trigger\n"
4866 "\t The first disables tracing every time block_unplug is hit.\n"
4867 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4868 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4869 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4870 "\t Like function triggers, the counter is only decremented if it\n"
4871 "\t enabled or disabled tracing.\n"
4872 "\t To remove a trigger without a count:\n"
4873 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4874 "\t To remove a trigger with a count:\n"
4875 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4876 "\t Filters can be ignored when removing a trigger.\n"
4877 #ifdef CONFIG_HIST_TRIGGERS
4878 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4879 "\t Format: hist:keys=<field1[,field2,...]>\n"
4880 "\t [:values=<field1[,field2,...]>]\n"
4881 "\t [:sort=<field1[,field2,...]>]\n"
4882 "\t [:size=#entries]\n"
4883 "\t [:pause][:continue][:clear]\n"
4884 "\t [:name=histname1]\n"
4885 "\t [:<handler>.<action>]\n"
4886 "\t [if <filter>]\n\n"
4887 "\t When a matching event is hit, an entry is added to a hash\n"
4888 "\t table using the key(s) and value(s) named, and the value of a\n"
4889 "\t sum called 'hitcount' is incremented. Keys and values\n"
4890 "\t correspond to fields in the event's format description. Keys\n"
4891 "\t can be any field, or the special string 'stacktrace'.\n"
4892 "\t Compound keys consisting of up to two fields can be specified\n"
4893 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4894 "\t fields. Sort keys consisting of up to two fields can be\n"
4895 "\t specified using the 'sort' keyword. The sort direction can\n"
4896 "\t be modified by appending '.descending' or '.ascending' to a\n"
4897 "\t sort field. The 'size' parameter can be used to specify more\n"
4898 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4899 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4900 "\t its histogram data will be shared with other triggers of the\n"
4901 "\t same name, and trigger hits will update this common data.\n\n"
4902 "\t Reading the 'hist' file for the event will dump the hash\n"
4903 "\t table in its entirety to stdout. If there are multiple hist\n"
4904 "\t triggers attached to an event, there will be a table for each\n"
4905 "\t trigger in the output. The table displayed for a named\n"
4906 "\t trigger will be the same as any other instance having the\n"
4907 "\t same name. The default format used to display a given field\n"
4908 "\t can be modified by appending any of the following modifiers\n"
4909 "\t to the field name, as applicable:\n\n"
4910 "\t .hex display a number as a hex value\n"
4911 "\t .sym display an address as a symbol\n"
4912 "\t .sym-offset display an address as a symbol and offset\n"
4913 "\t .execname display a common_pid as a program name\n"
4914 "\t .syscall display a syscall id as a syscall name\n"
4915 "\t .log2 display log2 value rather than raw number\n"
4916 "\t .usecs display a common_timestamp in microseconds\n\n"
4917 "\t The 'pause' parameter can be used to pause an existing hist\n"
4918 "\t trigger or to start a hist trigger but not log any events\n"
4919 "\t until told to do so. 'continue' can be used to start or\n"
4920 "\t restart a paused hist trigger.\n\n"
4921 "\t The 'clear' parameter will clear the contents of a running\n"
4922 "\t hist trigger and leave its current paused/active state\n"
4924 "\t The enable_hist and disable_hist triggers can be used to\n"
4925 "\t have one event conditionally start and stop another event's\n"
4926 "\t already-attached hist trigger. The syntax is analogous to\n"
4927 "\t the enable_event and disable_event triggers.\n\n"
4928 "\t Hist trigger handlers and actions are executed whenever a\n"
4929 "\t a histogram entry is added or updated. They take the form:\n\n"
4930 "\t <handler>.<action>\n\n"
4931 "\t The available handlers are:\n\n"
4932 "\t onmatch(matching.event) - invoke on addition or update\n"
4933 "\t onmax(var) - invoke if var exceeds current max\n"
4934 "\t onchange(var) - invoke action if var changes\n\n"
4935 "\t The available actions are:\n\n"
4936 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
4937 "\t save(field,...) - save current event fields\n"
4938 #ifdef CONFIG_TRACER_SNAPSHOT
4939 "\t snapshot() - snapshot the trace buffer\n"
4945 tracing_readme_read(struct file *filp, char __user *ubuf,
4946 size_t cnt, loff_t *ppos)
4948 return simple_read_from_buffer(ubuf, cnt, ppos,
4949 readme_msg, strlen(readme_msg));
4952 static const struct file_operations tracing_readme_fops = {
4953 .open = tracing_open_generic,
4954 .read = tracing_readme_read,
4955 .llseek = generic_file_llseek,
4958 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
4962 if (*pos || m->count)
4967 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
4968 if (trace_find_tgid(*ptr))
4975 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
4985 v = saved_tgids_next(m, v, &l);
4993 static void saved_tgids_stop(struct seq_file *m, void *v)
4997 static int saved_tgids_show(struct seq_file *m, void *v)
4999 int pid = (int *)v - tgid_map;
5001 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5005 static const struct seq_operations tracing_saved_tgids_seq_ops = {
5006 .start = saved_tgids_start,
5007 .stop = saved_tgids_stop,
5008 .next = saved_tgids_next,
5009 .show = saved_tgids_show,
5012 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5014 if (tracing_disabled)
5017 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5021 static const struct file_operations tracing_saved_tgids_fops = {
5022 .open = tracing_saved_tgids_open,
5024 .llseek = seq_lseek,
5025 .release = seq_release,
5028 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5030 unsigned int *ptr = v;
5032 if (*pos || m->count)
5037 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5039 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5048 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5054 arch_spin_lock(&trace_cmdline_lock);
5056 v = &savedcmd->map_cmdline_to_pid[0];
5058 v = saved_cmdlines_next(m, v, &l);
5066 static void saved_cmdlines_stop(struct seq_file *m, void *v)
5068 arch_spin_unlock(&trace_cmdline_lock);
5072 static int saved_cmdlines_show(struct seq_file *m, void *v)
5074 char buf[TASK_COMM_LEN];
5075 unsigned int *pid = v;
5077 __trace_find_cmdline(*pid, buf);
5078 seq_printf(m, "%d %s\n", *pid, buf);
5082 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5083 .start = saved_cmdlines_start,
5084 .next = saved_cmdlines_next,
5085 .stop = saved_cmdlines_stop,
5086 .show = saved_cmdlines_show,
5089 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5091 if (tracing_disabled)
5094 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
5097 static const struct file_operations tracing_saved_cmdlines_fops = {
5098 .open = tracing_saved_cmdlines_open,
5100 .llseek = seq_lseek,
5101 .release = seq_release,
5105 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5106 size_t cnt, loff_t *ppos)
5111 arch_spin_lock(&trace_cmdline_lock);
5112 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
5113 arch_spin_unlock(&trace_cmdline_lock);
5115 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5118 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5120 kfree(s->saved_cmdlines);
5121 kfree(s->map_cmdline_to_pid);
5125 static int tracing_resize_saved_cmdlines(unsigned int val)
5127 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5129 s = kmalloc(sizeof(*s), GFP_KERNEL);
5133 if (allocate_cmdlines_buffer(val, s) < 0) {
5138 arch_spin_lock(&trace_cmdline_lock);
5139 savedcmd_temp = savedcmd;
5141 arch_spin_unlock(&trace_cmdline_lock);
5142 free_saved_cmdlines_buffer(savedcmd_temp);
5148 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5149 size_t cnt, loff_t *ppos)
5154 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5158 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5159 if (!val || val > PID_MAX_DEFAULT)
5162 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5171 static const struct file_operations tracing_saved_cmdlines_size_fops = {
5172 .open = tracing_open_generic,
5173 .read = tracing_saved_cmdlines_size_read,
5174 .write = tracing_saved_cmdlines_size_write,
5177 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5178 static union trace_eval_map_item *
5179 update_eval_map(union trace_eval_map_item *ptr)
5181 if (!ptr->map.eval_string) {
5182 if (ptr->tail.next) {
5183 ptr = ptr->tail.next;
5184 /* Set ptr to the next real item (skip head) */
5192 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5194 union trace_eval_map_item *ptr = v;
5197 * Paranoid! If ptr points to end, we don't want to increment past it.
5198 * This really should never happen.
5200 ptr = update_eval_map(ptr);
5201 if (WARN_ON_ONCE(!ptr))
5208 ptr = update_eval_map(ptr);
5213 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5215 union trace_eval_map_item *v;
5218 mutex_lock(&trace_eval_mutex);
5220 v = trace_eval_maps;
5224 while (v && l < *pos) {
5225 v = eval_map_next(m, v, &l);
5231 static void eval_map_stop(struct seq_file *m, void *v)
5233 mutex_unlock(&trace_eval_mutex);
5236 static int eval_map_show(struct seq_file *m, void *v)
5238 union trace_eval_map_item *ptr = v;
5240 seq_printf(m, "%s %ld (%s)\n",
5241 ptr->map.eval_string, ptr->map.eval_value,
5247 static const struct seq_operations tracing_eval_map_seq_ops = {
5248 .start = eval_map_start,
5249 .next = eval_map_next,
5250 .stop = eval_map_stop,
5251 .show = eval_map_show,
5254 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5256 if (tracing_disabled)
5259 return seq_open(filp, &tracing_eval_map_seq_ops);
5262 static const struct file_operations tracing_eval_map_fops = {
5263 .open = tracing_eval_map_open,
5265 .llseek = seq_lseek,
5266 .release = seq_release,
5269 static inline union trace_eval_map_item *
5270 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5272 /* Return tail of array given the head */
5273 return ptr + ptr->head.length + 1;
5277 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5280 struct trace_eval_map **stop;
5281 struct trace_eval_map **map;
5282 union trace_eval_map_item *map_array;
5283 union trace_eval_map_item *ptr;
5288 * The trace_eval_maps contains the map plus a head and tail item,
5289 * where the head holds the module and length of array, and the
5290 * tail holds a pointer to the next list.
5292 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
5294 pr_warn("Unable to allocate trace eval mapping\n");
5298 mutex_lock(&trace_eval_mutex);
5300 if (!trace_eval_maps)
5301 trace_eval_maps = map_array;
5303 ptr = trace_eval_maps;
5305 ptr = trace_eval_jmp_to_tail(ptr);
5306 if (!ptr->tail.next)
5308 ptr = ptr->tail.next;
5311 ptr->tail.next = map_array;
5313 map_array->head.mod = mod;
5314 map_array->head.length = len;
5317 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5318 map_array->map = **map;
5321 memset(map_array, 0, sizeof(*map_array));
5323 mutex_unlock(&trace_eval_mutex);
5326 static void trace_create_eval_file(struct dentry *d_tracer)
5328 trace_create_file("eval_map", 0444, d_tracer,
5329 NULL, &tracing_eval_map_fops);
5332 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5333 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5334 static inline void trace_insert_eval_map_file(struct module *mod,
5335 struct trace_eval_map **start, int len) { }
5336 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5338 static void trace_insert_eval_map(struct module *mod,
5339 struct trace_eval_map **start, int len)
5341 struct trace_eval_map **map;
5348 trace_event_eval_update(map, len);
5350 trace_insert_eval_map_file(mod, start, len);
5354 tracing_set_trace_read(struct file *filp, char __user *ubuf,
5355 size_t cnt, loff_t *ppos)
5357 struct trace_array *tr = filp->private_data;
5358 char buf[MAX_TRACER_SIZE+2];
5361 mutex_lock(&trace_types_lock);
5362 r = sprintf(buf, "%s\n", tr->current_trace->name);
5363 mutex_unlock(&trace_types_lock);
5365 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5368 int tracer_init(struct tracer *t, struct trace_array *tr)
5370 tracing_reset_online_cpus(&tr->trace_buffer);
5374 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
5378 for_each_tracing_cpu(cpu)
5379 per_cpu_ptr(buf->data, cpu)->entries = val;
5382 #ifdef CONFIG_TRACER_MAX_TRACE
5383 /* resize @tr's buffer to the size of @size_tr's entries */
5384 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5385 struct trace_buffer *size_buf, int cpu_id)
5389 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5390 for_each_tracing_cpu(cpu) {
5391 ret = ring_buffer_resize(trace_buf->buffer,
5392 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5395 per_cpu_ptr(trace_buf->data, cpu)->entries =
5396 per_cpu_ptr(size_buf->data, cpu)->entries;
5399 ret = ring_buffer_resize(trace_buf->buffer,
5400 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5402 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5403 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5408 #endif /* CONFIG_TRACER_MAX_TRACE */
5410 static int __tracing_resize_ring_buffer(struct trace_array *tr,
5411 unsigned long size, int cpu)
5416 * If kernel or user changes the size of the ring buffer
5417 * we use the size that was given, and we can forget about
5418 * expanding it later.
5420 ring_buffer_expanded = true;
5422 /* May be called before buffers are initialized */
5423 if (!tr->trace_buffer.buffer)
5426 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
5430 #ifdef CONFIG_TRACER_MAX_TRACE
5431 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5432 !tr->current_trace->use_max_tr)
5435 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5437 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5438 &tr->trace_buffer, cpu);
5441 * AARGH! We are left with different
5442 * size max buffer!!!!
5443 * The max buffer is our "snapshot" buffer.
5444 * When a tracer needs a snapshot (one of the
5445 * latency tracers), it swaps the max buffer
5446 * with the saved snap shot. We succeeded to
5447 * update the size of the main buffer, but failed to
5448 * update the size of the max buffer. But when we tried
5449 * to reset the main buffer to the original size, we
5450 * failed there too. This is very unlikely to
5451 * happen, but if it does, warn and kill all
5455 tracing_disabled = 1;
5460 if (cpu == RING_BUFFER_ALL_CPUS)
5461 set_buffer_entries(&tr->max_buffer, size);
5463 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5466 #endif /* CONFIG_TRACER_MAX_TRACE */
5468 if (cpu == RING_BUFFER_ALL_CPUS)
5469 set_buffer_entries(&tr->trace_buffer, size);
5471 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
5476 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5477 unsigned long size, int cpu_id)
5481 mutex_lock(&trace_types_lock);
5483 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5484 /* make sure, this cpu is enabled in the mask */
5485 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5491 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5496 mutex_unlock(&trace_types_lock);
5503 * tracing_update_buffers - used by tracing facility to expand ring buffers
5505 * To save on memory when the tracing is never used on a system with it
5506 * configured in. The ring buffers are set to a minimum size. But once
5507 * a user starts to use the tracing facility, then they need to grow
5508 * to their default size.
5510 * This function is to be called when a tracer is about to be used.
5512 int tracing_update_buffers(void)
5516 mutex_lock(&trace_types_lock);
5517 if (!ring_buffer_expanded)
5518 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5519 RING_BUFFER_ALL_CPUS);
5520 mutex_unlock(&trace_types_lock);
5525 struct trace_option_dentry;
5528 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5531 * Used to clear out the tracer before deletion of an instance.
5532 * Must have trace_types_lock held.
5534 static void tracing_set_nop(struct trace_array *tr)
5536 if (tr->current_trace == &nop_trace)
5539 tr->current_trace->enabled--;
5541 if (tr->current_trace->reset)
5542 tr->current_trace->reset(tr);
5544 tr->current_trace = &nop_trace;
5547 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5549 /* Only enable if the directory has been created already. */
5553 create_trace_option_files(tr, t);
5556 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5559 #ifdef CONFIG_TRACER_MAX_TRACE
5564 mutex_lock(&trace_types_lock);
5566 if (!ring_buffer_expanded) {
5567 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5568 RING_BUFFER_ALL_CPUS);
5574 for (t = trace_types; t; t = t->next) {
5575 if (strcmp(t->name, buf) == 0)
5582 if (t == tr->current_trace)
5585 #ifdef CONFIG_TRACER_SNAPSHOT
5586 if (t->use_max_tr) {
5587 arch_spin_lock(&tr->max_lock);
5588 if (tr->cond_snapshot)
5590 arch_spin_unlock(&tr->max_lock);
5595 /* Some tracers won't work on kernel command line */
5596 if (system_state < SYSTEM_RUNNING && t->noboot) {
5597 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5602 /* Some tracers are only allowed for the top level buffer */
5603 if (!trace_ok_for_array(t, tr)) {
5608 /* If trace pipe files are being read, we can't change the tracer */
5609 if (tr->current_trace->ref) {
5614 trace_branch_disable();
5616 tr->current_trace->enabled--;
5618 if (tr->current_trace->reset)
5619 tr->current_trace->reset(tr);
5621 /* Current trace needs to be nop_trace before synchronize_rcu */
5622 tr->current_trace = &nop_trace;
5624 #ifdef CONFIG_TRACER_MAX_TRACE
5625 had_max_tr = tr->allocated_snapshot;
5627 if (had_max_tr && !t->use_max_tr) {
5629 * We need to make sure that the update_max_tr sees that
5630 * current_trace changed to nop_trace to keep it from
5631 * swapping the buffers after we resize it.
5632 * The update_max_tr is called from interrupts disabled
5633 * so a synchronized_sched() is sufficient.
5640 #ifdef CONFIG_TRACER_MAX_TRACE
5641 if (t->use_max_tr && !had_max_tr) {
5642 ret = tracing_alloc_snapshot_instance(tr);
5649 ret = tracer_init(t, tr);
5654 tr->current_trace = t;
5655 tr->current_trace->enabled++;
5656 trace_branch_enable(tr);
5658 mutex_unlock(&trace_types_lock);
5664 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5665 size_t cnt, loff_t *ppos)
5667 struct trace_array *tr = filp->private_data;
5668 char buf[MAX_TRACER_SIZE+1];
5675 if (cnt > MAX_TRACER_SIZE)
5676 cnt = MAX_TRACER_SIZE;
5678 if (copy_from_user(buf, ubuf, cnt))
5683 /* strip ending whitespace. */
5684 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5687 err = tracing_set_tracer(tr, buf);
5697 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5698 size_t cnt, loff_t *ppos)
5703 r = snprintf(buf, sizeof(buf), "%ld\n",
5704 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5705 if (r > sizeof(buf))
5707 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5711 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5712 size_t cnt, loff_t *ppos)
5717 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5727 tracing_thresh_read(struct file *filp, char __user *ubuf,
5728 size_t cnt, loff_t *ppos)
5730 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5734 tracing_thresh_write(struct file *filp, const char __user *ubuf,
5735 size_t cnt, loff_t *ppos)
5737 struct trace_array *tr = filp->private_data;
5740 mutex_lock(&trace_types_lock);
5741 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5745 if (tr->current_trace->update_thresh) {
5746 ret = tr->current_trace->update_thresh(tr);
5753 mutex_unlock(&trace_types_lock);
5758 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5761 tracing_max_lat_read(struct file *filp, char __user *ubuf,
5762 size_t cnt, loff_t *ppos)
5764 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5768 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5769 size_t cnt, loff_t *ppos)
5771 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5776 static int tracing_open_pipe(struct inode *inode, struct file *filp)
5778 struct trace_array *tr = inode->i_private;
5779 struct trace_iterator *iter;
5782 if (tracing_disabled)
5785 if (trace_array_get(tr) < 0)
5788 mutex_lock(&trace_types_lock);
5790 /* create a buffer to store the information to pass to userspace */
5791 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5794 __trace_array_put(tr);
5798 trace_seq_init(&iter->seq);
5799 iter->trace = tr->current_trace;
5801 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5806 /* trace pipe does not show start of buffer */
5807 cpumask_setall(iter->started);
5809 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5810 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5812 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5813 if (trace_clocks[tr->clock_id].in_ns)
5814 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5817 iter->trace_buffer = &tr->trace_buffer;
5818 iter->cpu_file = tracing_get_cpu(inode);
5819 mutex_init(&iter->mutex);
5820 filp->private_data = iter;
5822 if (iter->trace->pipe_open)
5823 iter->trace->pipe_open(iter);
5825 nonseekable_open(inode, filp);
5827 tr->current_trace->ref++;
5829 mutex_unlock(&trace_types_lock);
5834 __trace_array_put(tr);
5835 mutex_unlock(&trace_types_lock);
5839 static int tracing_release_pipe(struct inode *inode, struct file *file)
5841 struct trace_iterator *iter = file->private_data;
5842 struct trace_array *tr = inode->i_private;
5844 mutex_lock(&trace_types_lock);
5846 tr->current_trace->ref--;
5848 if (iter->trace->pipe_close)
5849 iter->trace->pipe_close(iter);
5851 mutex_unlock(&trace_types_lock);
5853 free_cpumask_var(iter->started);
5854 mutex_destroy(&iter->mutex);
5857 trace_array_put(tr);
5863 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5865 struct trace_array *tr = iter->tr;
5867 /* Iterators are static, they should be filled or empty */
5868 if (trace_buffer_iter(iter, iter->cpu_file))
5869 return EPOLLIN | EPOLLRDNORM;
5871 if (tr->trace_flags & TRACE_ITER_BLOCK)
5873 * Always select as readable when in blocking mode
5875 return EPOLLIN | EPOLLRDNORM;
5877 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5882 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5884 struct trace_iterator *iter = filp->private_data;
5886 return trace_poll(iter, filp, poll_table);
5889 /* Must be called with iter->mutex held. */
5890 static int tracing_wait_pipe(struct file *filp)
5892 struct trace_iterator *iter = filp->private_data;
5895 while (trace_empty(iter)) {
5897 if ((filp->f_flags & O_NONBLOCK)) {
5902 * We block until we read something and tracing is disabled.
5903 * We still block if tracing is disabled, but we have never
5904 * read anything. This allows a user to cat this file, and
5905 * then enable tracing. But after we have read something,
5906 * we give an EOF when tracing is again disabled.
5908 * iter->pos will be 0 if we haven't read anything.
5910 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5913 mutex_unlock(&iter->mutex);
5915 ret = wait_on_pipe(iter, 0);
5917 mutex_lock(&iter->mutex);
5930 tracing_read_pipe(struct file *filp, char __user *ubuf,
5931 size_t cnt, loff_t *ppos)
5933 struct trace_iterator *iter = filp->private_data;
5937 * Avoid more than one consumer on a single file descriptor
5938 * This is just a matter of traces coherency, the ring buffer itself
5941 mutex_lock(&iter->mutex);
5943 /* return any leftover data */
5944 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5948 trace_seq_init(&iter->seq);
5950 if (iter->trace->read) {
5951 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5957 sret = tracing_wait_pipe(filp);
5961 /* stop when tracing is finished */
5962 if (trace_empty(iter)) {
5967 if (cnt >= PAGE_SIZE)
5968 cnt = PAGE_SIZE - 1;
5970 /* reset all but tr, trace, and overruns */
5971 memset(&iter->seq, 0,
5972 sizeof(struct trace_iterator) -
5973 offsetof(struct trace_iterator, seq));
5974 cpumask_clear(iter->started);
5977 trace_event_read_lock();
5978 trace_access_lock(iter->cpu_file);
5979 while (trace_find_next_entry_inc(iter) != NULL) {
5980 enum print_line_t ret;
5981 int save_len = iter->seq.seq.len;
5983 ret = print_trace_line(iter);
5984 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5985 /* don't print partial lines */
5986 iter->seq.seq.len = save_len;
5989 if (ret != TRACE_TYPE_NO_CONSUME)
5990 trace_consume(iter);
5992 if (trace_seq_used(&iter->seq) >= cnt)
5996 * Setting the full flag means we reached the trace_seq buffer
5997 * size and we should leave by partial output condition above.
5998 * One of the trace_seq_* functions is not used properly.
6000 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6003 trace_access_unlock(iter->cpu_file);
6004 trace_event_read_unlock();
6006 /* Now copy what we have to the user */
6007 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
6008 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
6009 trace_seq_init(&iter->seq);
6012 * If there was nothing to send to user, in spite of consuming trace
6013 * entries, go back to wait for more entries.
6019 mutex_unlock(&iter->mutex);
6024 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6027 __free_page(spd->pages[idx]);
6030 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
6031 .confirm = generic_pipe_buf_confirm,
6032 .release = generic_pipe_buf_release,
6033 .steal = generic_pipe_buf_steal,
6034 .get = generic_pipe_buf_get,
6038 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6044 /* Seq buffer is page-sized, exactly what we need. */
6046 save_len = iter->seq.seq.len;
6047 ret = print_trace_line(iter);
6049 if (trace_seq_has_overflowed(&iter->seq)) {
6050 iter->seq.seq.len = save_len;
6055 * This should not be hit, because it should only
6056 * be set if the iter->seq overflowed. But check it
6057 * anyway to be safe.
6059 if (ret == TRACE_TYPE_PARTIAL_LINE) {
6060 iter->seq.seq.len = save_len;
6064 count = trace_seq_used(&iter->seq) - save_len;
6067 iter->seq.seq.len = save_len;
6071 if (ret != TRACE_TYPE_NO_CONSUME)
6072 trace_consume(iter);
6074 if (!trace_find_next_entry_inc(iter)) {
6084 static ssize_t tracing_splice_read_pipe(struct file *filp,
6086 struct pipe_inode_info *pipe,
6090 struct page *pages_def[PIPE_DEF_BUFFERS];
6091 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6092 struct trace_iterator *iter = filp->private_data;
6093 struct splice_pipe_desc spd = {
6095 .partial = partial_def,
6096 .nr_pages = 0, /* This gets updated below. */
6097 .nr_pages_max = PIPE_DEF_BUFFERS,
6098 .ops = &tracing_pipe_buf_ops,
6099 .spd_release = tracing_spd_release_pipe,
6105 if (splice_grow_spd(pipe, &spd))
6108 mutex_lock(&iter->mutex);
6110 if (iter->trace->splice_read) {
6111 ret = iter->trace->splice_read(iter, filp,
6112 ppos, pipe, len, flags);
6117 ret = tracing_wait_pipe(filp);
6121 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6126 trace_event_read_lock();
6127 trace_access_lock(iter->cpu_file);
6129 /* Fill as many pages as possible. */
6130 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6131 spd.pages[i] = alloc_page(GFP_KERNEL);
6135 rem = tracing_fill_pipe_page(rem, iter);
6137 /* Copy the data into the page, so we can start over. */
6138 ret = trace_seq_to_buffer(&iter->seq,
6139 page_address(spd.pages[i]),
6140 trace_seq_used(&iter->seq));
6142 __free_page(spd.pages[i]);
6145 spd.partial[i].offset = 0;
6146 spd.partial[i].len = trace_seq_used(&iter->seq);
6148 trace_seq_init(&iter->seq);
6151 trace_access_unlock(iter->cpu_file);
6152 trace_event_read_unlock();
6153 mutex_unlock(&iter->mutex);
6158 ret = splice_to_pipe(pipe, &spd);
6162 splice_shrink_spd(&spd);
6166 mutex_unlock(&iter->mutex);
6171 tracing_entries_read(struct file *filp, char __user *ubuf,
6172 size_t cnt, loff_t *ppos)
6174 struct inode *inode = file_inode(filp);
6175 struct trace_array *tr = inode->i_private;
6176 int cpu = tracing_get_cpu(inode);
6181 mutex_lock(&trace_types_lock);
6183 if (cpu == RING_BUFFER_ALL_CPUS) {
6184 int cpu, buf_size_same;
6189 /* check if all cpu sizes are same */
6190 for_each_tracing_cpu(cpu) {
6191 /* fill in the size from first enabled cpu */
6193 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
6194 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
6200 if (buf_size_same) {
6201 if (!ring_buffer_expanded)
6202 r = sprintf(buf, "%lu (expanded: %lu)\n",
6204 trace_buf_size >> 10);
6206 r = sprintf(buf, "%lu\n", size >> 10);
6208 r = sprintf(buf, "X\n");
6210 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
6212 mutex_unlock(&trace_types_lock);
6214 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6219 tracing_entries_write(struct file *filp, const char __user *ubuf,
6220 size_t cnt, loff_t *ppos)
6222 struct inode *inode = file_inode(filp);
6223 struct trace_array *tr = inode->i_private;
6227 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6231 /* must have at least 1 entry */
6235 /* value is in KB */
6237 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6247 tracing_total_entries_read(struct file *filp, char __user *ubuf,
6248 size_t cnt, loff_t *ppos)
6250 struct trace_array *tr = filp->private_data;
6253 unsigned long size = 0, expanded_size = 0;
6255 mutex_lock(&trace_types_lock);
6256 for_each_tracing_cpu(cpu) {
6257 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
6258 if (!ring_buffer_expanded)
6259 expanded_size += trace_buf_size >> 10;
6261 if (ring_buffer_expanded)
6262 r = sprintf(buf, "%lu\n", size);
6264 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6265 mutex_unlock(&trace_types_lock);
6267 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6271 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6272 size_t cnt, loff_t *ppos)
6275 * There is no need to read what the user has written, this function
6276 * is just to make sure that there is no error when "echo" is used
6285 tracing_free_buffer_release(struct inode *inode, struct file *filp)
6287 struct trace_array *tr = inode->i_private;
6289 /* disable tracing ? */
6290 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6291 tracer_tracing_off(tr);
6292 /* resize the ring buffer to 0 */
6293 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6295 trace_array_put(tr);
6301 tracing_mark_write(struct file *filp, const char __user *ubuf,
6302 size_t cnt, loff_t *fpos)
6304 struct trace_array *tr = filp->private_data;
6305 struct ring_buffer_event *event;
6306 enum event_trigger_type tt = ETT_NONE;
6307 struct ring_buffer *buffer;
6308 struct print_entry *entry;
6309 unsigned long irq_flags;
6310 const char faulted[] = "<faulted>";
6315 /* Used in tracing_mark_raw_write() as well */
6316 #define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */
6318 if (tracing_disabled)
6321 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6324 if (cnt > TRACE_BUF_SIZE)
6325 cnt = TRACE_BUF_SIZE;
6327 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6329 local_save_flags(irq_flags);
6330 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6332 /* If less than "<faulted>", then make sure we can still add that */
6333 if (cnt < FAULTED_SIZE)
6334 size += FAULTED_SIZE - cnt;
6336 buffer = tr->trace_buffer.buffer;
6337 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6338 irq_flags, preempt_count());
6339 if (unlikely(!event))
6340 /* Ring buffer disabled, return as if not open for write */
6343 entry = ring_buffer_event_data(event);
6344 entry->ip = _THIS_IP_;
6346 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6348 memcpy(&entry->buf, faulted, FAULTED_SIZE);
6355 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6356 /* do not add \n before testing triggers, but add \0 */
6357 entry->buf[cnt] = '\0';
6358 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6361 if (entry->buf[cnt - 1] != '\n') {
6362 entry->buf[cnt] = '\n';
6363 entry->buf[cnt + 1] = '\0';
6365 entry->buf[cnt] = '\0';
6367 __buffer_unlock_commit(buffer, event);
6370 event_triggers_post_call(tr->trace_marker_file, tt);
6378 /* Limit it for now to 3K (including tag) */
6379 #define RAW_DATA_MAX_SIZE (1024*3)
6382 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6383 size_t cnt, loff_t *fpos)
6385 struct trace_array *tr = filp->private_data;
6386 struct ring_buffer_event *event;
6387 struct ring_buffer *buffer;
6388 struct raw_data_entry *entry;
6389 const char faulted[] = "<faulted>";
6390 unsigned long irq_flags;
6395 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6397 if (tracing_disabled)
6400 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6403 /* The marker must at least have a tag id */
6404 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6407 if (cnt > TRACE_BUF_SIZE)
6408 cnt = TRACE_BUF_SIZE;
6410 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6412 local_save_flags(irq_flags);
6413 size = sizeof(*entry) + cnt;
6414 if (cnt < FAULT_SIZE_ID)
6415 size += FAULT_SIZE_ID - cnt;
6417 buffer = tr->trace_buffer.buffer;
6418 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6419 irq_flags, preempt_count());
6421 /* Ring buffer disabled, return as if not open for write */
6424 entry = ring_buffer_event_data(event);
6426 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6429 memcpy(&entry->buf, faulted, FAULTED_SIZE);
6434 __buffer_unlock_commit(buffer, event);
6442 static int tracing_clock_show(struct seq_file *m, void *v)
6444 struct trace_array *tr = m->private;
6447 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6449 "%s%s%s%s", i ? " " : "",
6450 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6451 i == tr->clock_id ? "]" : "");
6457 int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6461 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6462 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6465 if (i == ARRAY_SIZE(trace_clocks))
6468 mutex_lock(&trace_types_lock);
6472 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
6475 * New clock may not be consistent with the previous clock.
6476 * Reset the buffer so that it doesn't have incomparable timestamps.
6478 tracing_reset_online_cpus(&tr->trace_buffer);
6480 #ifdef CONFIG_TRACER_MAX_TRACE
6481 if (tr->max_buffer.buffer)
6482 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6483 tracing_reset_online_cpus(&tr->max_buffer);
6486 mutex_unlock(&trace_types_lock);
6491 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6492 size_t cnt, loff_t *fpos)
6494 struct seq_file *m = filp->private_data;
6495 struct trace_array *tr = m->private;
6497 const char *clockstr;
6500 if (cnt >= sizeof(buf))
6503 if (copy_from_user(buf, ubuf, cnt))
6508 clockstr = strstrip(buf);
6510 ret = tracing_set_clock(tr, clockstr);
6519 static int tracing_clock_open(struct inode *inode, struct file *file)
6521 struct trace_array *tr = inode->i_private;
6524 if (tracing_disabled)
6527 if (trace_array_get(tr))
6530 ret = single_open(file, tracing_clock_show, inode->i_private);
6532 trace_array_put(tr);
6537 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6539 struct trace_array *tr = m->private;
6541 mutex_lock(&trace_types_lock);
6543 if (ring_buffer_time_stamp_abs(tr->trace_buffer.buffer))
6544 seq_puts(m, "delta [absolute]\n");
6546 seq_puts(m, "[delta] absolute\n");
6548 mutex_unlock(&trace_types_lock);
6553 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6555 struct trace_array *tr = inode->i_private;
6558 if (tracing_disabled)
6561 if (trace_array_get(tr))
6564 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6566 trace_array_put(tr);
6571 int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6575 mutex_lock(&trace_types_lock);
6577 if (abs && tr->time_stamp_abs_ref++)
6581 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6586 if (--tr->time_stamp_abs_ref)
6590 ring_buffer_set_time_stamp_abs(tr->trace_buffer.buffer, abs);
6592 #ifdef CONFIG_TRACER_MAX_TRACE
6593 if (tr->max_buffer.buffer)
6594 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6597 mutex_unlock(&trace_types_lock);
6602 struct ftrace_buffer_info {
6603 struct trace_iterator iter;
6605 unsigned int spare_cpu;
6609 #ifdef CONFIG_TRACER_SNAPSHOT
6610 static int tracing_snapshot_open(struct inode *inode, struct file *file)
6612 struct trace_array *tr = inode->i_private;
6613 struct trace_iterator *iter;
6617 if (trace_array_get(tr) < 0)
6620 if (file->f_mode & FMODE_READ) {
6621 iter = __tracing_open(inode, file, true);
6623 ret = PTR_ERR(iter);
6625 /* Writes still need the seq_file to hold the private data */
6627 m = kzalloc(sizeof(*m), GFP_KERNEL);
6630 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6638 iter->trace_buffer = &tr->max_buffer;
6639 iter->cpu_file = tracing_get_cpu(inode);
6641 file->private_data = m;
6645 trace_array_put(tr);
6651 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6654 struct seq_file *m = filp->private_data;
6655 struct trace_iterator *iter = m->private;
6656 struct trace_array *tr = iter->tr;
6660 ret = tracing_update_buffers();
6664 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6668 mutex_lock(&trace_types_lock);
6670 if (tr->current_trace->use_max_tr) {
6675 arch_spin_lock(&tr->max_lock);
6676 if (tr->cond_snapshot)
6678 arch_spin_unlock(&tr->max_lock);
6684 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6688 if (tr->allocated_snapshot)
6692 /* Only allow per-cpu swap if the ring buffer supports it */
6693 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6694 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6699 if (!tr->allocated_snapshot) {
6700 ret = tracing_alloc_snapshot_instance(tr);
6704 local_irq_disable();
6705 /* Now, we're going to swap */
6706 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6707 update_max_tr(tr, current, smp_processor_id(), NULL);
6709 update_max_tr_single(tr, current, iter->cpu_file);
6713 if (tr->allocated_snapshot) {
6714 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6715 tracing_reset_online_cpus(&tr->max_buffer);
6717 tracing_reset(&tr->max_buffer, iter->cpu_file);
6727 mutex_unlock(&trace_types_lock);
6731 static int tracing_snapshot_release(struct inode *inode, struct file *file)
6733 struct seq_file *m = file->private_data;
6736 ret = tracing_release(inode, file);
6738 if (file->f_mode & FMODE_READ)
6741 /* If write only, the seq_file is just a stub */
6749 static int tracing_buffers_open(struct inode *inode, struct file *filp);
6750 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6751 size_t count, loff_t *ppos);
6752 static int tracing_buffers_release(struct inode *inode, struct file *file);
6753 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6754 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6756 static int snapshot_raw_open(struct inode *inode, struct file *filp)
6758 struct ftrace_buffer_info *info;
6761 ret = tracing_buffers_open(inode, filp);
6765 info = filp->private_data;
6767 if (info->iter.trace->use_max_tr) {
6768 tracing_buffers_release(inode, filp);
6772 info->iter.snapshot = true;
6773 info->iter.trace_buffer = &info->iter.tr->max_buffer;
6778 #endif /* CONFIG_TRACER_SNAPSHOT */
6781 static const struct file_operations tracing_thresh_fops = {
6782 .open = tracing_open_generic,
6783 .read = tracing_thresh_read,
6784 .write = tracing_thresh_write,
6785 .llseek = generic_file_llseek,
6788 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6789 static const struct file_operations tracing_max_lat_fops = {
6790 .open = tracing_open_generic,
6791 .read = tracing_max_lat_read,
6792 .write = tracing_max_lat_write,
6793 .llseek = generic_file_llseek,
6797 static const struct file_operations set_tracer_fops = {
6798 .open = tracing_open_generic,
6799 .read = tracing_set_trace_read,
6800 .write = tracing_set_trace_write,
6801 .llseek = generic_file_llseek,
6804 static const struct file_operations tracing_pipe_fops = {
6805 .open = tracing_open_pipe,
6806 .poll = tracing_poll_pipe,
6807 .read = tracing_read_pipe,
6808 .splice_read = tracing_splice_read_pipe,
6809 .release = tracing_release_pipe,
6810 .llseek = no_llseek,
6813 static const struct file_operations tracing_entries_fops = {
6814 .open = tracing_open_generic_tr,
6815 .read = tracing_entries_read,
6816 .write = tracing_entries_write,
6817 .llseek = generic_file_llseek,
6818 .release = tracing_release_generic_tr,
6821 static const struct file_operations tracing_total_entries_fops = {
6822 .open = tracing_open_generic_tr,
6823 .read = tracing_total_entries_read,
6824 .llseek = generic_file_llseek,
6825 .release = tracing_release_generic_tr,
6828 static const struct file_operations tracing_free_buffer_fops = {
6829 .open = tracing_open_generic_tr,
6830 .write = tracing_free_buffer_write,
6831 .release = tracing_free_buffer_release,
6834 static const struct file_operations tracing_mark_fops = {
6835 .open = tracing_open_generic_tr,
6836 .write = tracing_mark_write,
6837 .llseek = generic_file_llseek,
6838 .release = tracing_release_generic_tr,
6841 static const struct file_operations tracing_mark_raw_fops = {
6842 .open = tracing_open_generic_tr,
6843 .write = tracing_mark_raw_write,
6844 .llseek = generic_file_llseek,
6845 .release = tracing_release_generic_tr,
6848 static const struct file_operations trace_clock_fops = {
6849 .open = tracing_clock_open,
6851 .llseek = seq_lseek,
6852 .release = tracing_single_release_tr,
6853 .write = tracing_clock_write,
6856 static const struct file_operations trace_time_stamp_mode_fops = {
6857 .open = tracing_time_stamp_mode_open,
6859 .llseek = seq_lseek,
6860 .release = tracing_single_release_tr,
6863 #ifdef CONFIG_TRACER_SNAPSHOT
6864 static const struct file_operations snapshot_fops = {
6865 .open = tracing_snapshot_open,
6867 .write = tracing_snapshot_write,
6868 .llseek = tracing_lseek,
6869 .release = tracing_snapshot_release,
6872 static const struct file_operations snapshot_raw_fops = {
6873 .open = snapshot_raw_open,
6874 .read = tracing_buffers_read,
6875 .release = tracing_buffers_release,
6876 .splice_read = tracing_buffers_splice_read,
6877 .llseek = no_llseek,
6880 #endif /* CONFIG_TRACER_SNAPSHOT */
6882 static int tracing_buffers_open(struct inode *inode, struct file *filp)
6884 struct trace_array *tr = inode->i_private;
6885 struct ftrace_buffer_info *info;
6888 if (tracing_disabled)
6891 if (trace_array_get(tr) < 0)
6894 info = kzalloc(sizeof(*info), GFP_KERNEL);
6896 trace_array_put(tr);
6900 mutex_lock(&trace_types_lock);
6903 info->iter.cpu_file = tracing_get_cpu(inode);
6904 info->iter.trace = tr->current_trace;
6905 info->iter.trace_buffer = &tr->trace_buffer;
6907 /* Force reading ring buffer for first read */
6908 info->read = (unsigned int)-1;
6910 filp->private_data = info;
6912 tr->current_trace->ref++;
6914 mutex_unlock(&trace_types_lock);
6916 ret = nonseekable_open(inode, filp);
6918 trace_array_put(tr);
6924 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6926 struct ftrace_buffer_info *info = filp->private_data;
6927 struct trace_iterator *iter = &info->iter;
6929 return trace_poll(iter, filp, poll_table);
6933 tracing_buffers_read(struct file *filp, char __user *ubuf,
6934 size_t count, loff_t *ppos)
6936 struct ftrace_buffer_info *info = filp->private_data;
6937 struct trace_iterator *iter = &info->iter;
6944 #ifdef CONFIG_TRACER_MAX_TRACE
6945 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6950 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6952 if (IS_ERR(info->spare)) {
6953 ret = PTR_ERR(info->spare);
6956 info->spare_cpu = iter->cpu_file;
6962 /* Do we have previous read data to read? */
6963 if (info->read < PAGE_SIZE)
6967 trace_access_lock(iter->cpu_file);
6968 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
6972 trace_access_unlock(iter->cpu_file);
6975 if (trace_empty(iter)) {
6976 if ((filp->f_flags & O_NONBLOCK))
6979 ret = wait_on_pipe(iter, 0);
6990 size = PAGE_SIZE - info->read;
6994 ret = copy_to_user(ubuf, info->spare + info->read, size);
7006 static int tracing_buffers_release(struct inode *inode, struct file *file)
7008 struct ftrace_buffer_info *info = file->private_data;
7009 struct trace_iterator *iter = &info->iter;
7011 mutex_lock(&trace_types_lock);
7013 iter->tr->current_trace->ref--;
7015 __trace_array_put(iter->tr);
7018 ring_buffer_free_read_page(iter->trace_buffer->buffer,
7019 info->spare_cpu, info->spare);
7022 mutex_unlock(&trace_types_lock);
7028 struct ring_buffer *buffer;
7031 refcount_t refcount;
7034 static void buffer_ref_release(struct buffer_ref *ref)
7036 if (!refcount_dec_and_test(&ref->refcount))
7038 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7042 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7043 struct pipe_buffer *buf)
7045 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7047 buffer_ref_release(ref);
7051 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
7052 struct pipe_buffer *buf)
7054 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7056 if (refcount_read(&ref->refcount) > INT_MAX/2)
7059 refcount_inc(&ref->refcount);
7063 /* Pipe buffer operations for a buffer. */
7064 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
7065 .confirm = generic_pipe_buf_confirm,
7066 .release = buffer_pipe_buf_release,
7067 .steal = generic_pipe_buf_nosteal,
7068 .get = buffer_pipe_buf_get,
7072 * Callback from splice_to_pipe(), if we need to release some pages
7073 * at the end of the spd in case we error'ed out in filling the pipe.
7075 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7077 struct buffer_ref *ref =
7078 (struct buffer_ref *)spd->partial[i].private;
7080 buffer_ref_release(ref);
7081 spd->partial[i].private = 0;
7085 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7086 struct pipe_inode_info *pipe, size_t len,
7089 struct ftrace_buffer_info *info = file->private_data;
7090 struct trace_iterator *iter = &info->iter;
7091 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7092 struct page *pages_def[PIPE_DEF_BUFFERS];
7093 struct splice_pipe_desc spd = {
7095 .partial = partial_def,
7096 .nr_pages_max = PIPE_DEF_BUFFERS,
7097 .ops = &buffer_pipe_buf_ops,
7098 .spd_release = buffer_spd_release,
7100 struct buffer_ref *ref;
7104 #ifdef CONFIG_TRACER_MAX_TRACE
7105 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7109 if (*ppos & (PAGE_SIZE - 1))
7112 if (len & (PAGE_SIZE - 1)) {
7113 if (len < PAGE_SIZE)
7118 if (splice_grow_spd(pipe, &spd))
7122 trace_access_lock(iter->cpu_file);
7123 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
7125 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
7129 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
7135 refcount_set(&ref->refcount, 1);
7136 ref->buffer = iter->trace_buffer->buffer;
7137 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
7138 if (IS_ERR(ref->page)) {
7139 ret = PTR_ERR(ref->page);
7144 ref->cpu = iter->cpu_file;
7146 r = ring_buffer_read_page(ref->buffer, &ref->page,
7147 len, iter->cpu_file, 1);
7149 ring_buffer_free_read_page(ref->buffer, ref->cpu,
7155 page = virt_to_page(ref->page);
7157 spd.pages[i] = page;
7158 spd.partial[i].len = PAGE_SIZE;
7159 spd.partial[i].offset = 0;
7160 spd.partial[i].private = (unsigned long)ref;
7164 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
7167 trace_access_unlock(iter->cpu_file);
7170 /* did we read anything? */
7171 if (!spd.nr_pages) {
7176 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
7179 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
7186 ret = splice_to_pipe(pipe, &spd);
7188 splice_shrink_spd(&spd);
7193 static const struct file_operations tracing_buffers_fops = {
7194 .open = tracing_buffers_open,
7195 .read = tracing_buffers_read,
7196 .poll = tracing_buffers_poll,
7197 .release = tracing_buffers_release,
7198 .splice_read = tracing_buffers_splice_read,
7199 .llseek = no_llseek,
7203 tracing_stats_read(struct file *filp, char __user *ubuf,
7204 size_t count, loff_t *ppos)
7206 struct inode *inode = file_inode(filp);
7207 struct trace_array *tr = inode->i_private;
7208 struct trace_buffer *trace_buf = &tr->trace_buffer;
7209 int cpu = tracing_get_cpu(inode);
7210 struct trace_seq *s;
7212 unsigned long long t;
7213 unsigned long usec_rem;
7215 s = kmalloc(sizeof(*s), GFP_KERNEL);
7221 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
7222 trace_seq_printf(s, "entries: %ld\n", cnt);
7224 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
7225 trace_seq_printf(s, "overrun: %ld\n", cnt);
7227 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
7228 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7230 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
7231 trace_seq_printf(s, "bytes: %ld\n", cnt);
7233 if (trace_clocks[tr->clock_id].in_ns) {
7234 /* local or global for trace_clock */
7235 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7236 usec_rem = do_div(t, USEC_PER_SEC);
7237 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7240 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
7241 usec_rem = do_div(t, USEC_PER_SEC);
7242 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7244 /* counter or tsc mode for trace_clock */
7245 trace_seq_printf(s, "oldest event ts: %llu\n",
7246 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
7248 trace_seq_printf(s, "now ts: %llu\n",
7249 ring_buffer_time_stamp(trace_buf->buffer, cpu));
7252 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
7253 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7255 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
7256 trace_seq_printf(s, "read events: %ld\n", cnt);
7258 count = simple_read_from_buffer(ubuf, count, ppos,
7259 s->buffer, trace_seq_used(s));
7266 static const struct file_operations tracing_stats_fops = {
7267 .open = tracing_open_generic_tr,
7268 .read = tracing_stats_read,
7269 .llseek = generic_file_llseek,
7270 .release = tracing_release_generic_tr,
7273 #ifdef CONFIG_DYNAMIC_FTRACE
7276 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
7277 size_t cnt, loff_t *ppos)
7279 unsigned long *p = filp->private_data;
7280 char buf[64]; /* Not too big for a shallow stack */
7283 r = scnprintf(buf, 63, "%ld", *p);
7286 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7289 static const struct file_operations tracing_dyn_info_fops = {
7290 .open = tracing_open_generic,
7291 .read = tracing_read_dyn_info,
7292 .llseek = generic_file_llseek,
7294 #endif /* CONFIG_DYNAMIC_FTRACE */
7296 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7298 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7299 struct trace_array *tr, struct ftrace_probe_ops *ops,
7302 tracing_snapshot_instance(tr);
7306 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7307 struct trace_array *tr, struct ftrace_probe_ops *ops,
7310 struct ftrace_func_mapper *mapper = data;
7314 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7324 tracing_snapshot_instance(tr);
7328 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7329 struct ftrace_probe_ops *ops, void *data)
7331 struct ftrace_func_mapper *mapper = data;
7334 seq_printf(m, "%ps:", (void *)ip);
7336 seq_puts(m, "snapshot");
7339 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7342 seq_printf(m, ":count=%ld\n", *count);
7344 seq_puts(m, ":unlimited\n");
7350 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7351 unsigned long ip, void *init_data, void **data)
7353 struct ftrace_func_mapper *mapper = *data;
7356 mapper = allocate_ftrace_func_mapper();
7362 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7366 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7367 unsigned long ip, void *data)
7369 struct ftrace_func_mapper *mapper = data;
7374 free_ftrace_func_mapper(mapper, NULL);
7378 ftrace_func_mapper_remove_ip(mapper, ip);
7381 static struct ftrace_probe_ops snapshot_probe_ops = {
7382 .func = ftrace_snapshot,
7383 .print = ftrace_snapshot_print,
7386 static struct ftrace_probe_ops snapshot_count_probe_ops = {
7387 .func = ftrace_count_snapshot,
7388 .print = ftrace_snapshot_print,
7389 .init = ftrace_snapshot_init,
7390 .free = ftrace_snapshot_free,
7394 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7395 char *glob, char *cmd, char *param, int enable)
7397 struct ftrace_probe_ops *ops;
7398 void *count = (void *)-1;
7405 /* hash funcs only work with set_ftrace_filter */
7409 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7412 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
7417 number = strsep(¶m, ":");
7419 if (!strlen(number))
7423 * We use the callback data field (which is a pointer)
7426 ret = kstrtoul(number, 0, (unsigned long *)&count);
7431 ret = tracing_alloc_snapshot_instance(tr);
7435 ret = register_ftrace_function_probe(glob, tr, ops, count);
7438 return ret < 0 ? ret : 0;
7441 static struct ftrace_func_command ftrace_snapshot_cmd = {
7443 .func = ftrace_trace_snapshot_callback,
7446 static __init int register_snapshot_cmd(void)
7448 return register_ftrace_command(&ftrace_snapshot_cmd);
7451 static inline __init int register_snapshot_cmd(void) { return 0; }
7452 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7454 static struct dentry *tracing_get_dentry(struct trace_array *tr)
7456 if (WARN_ON(!tr->dir))
7457 return ERR_PTR(-ENODEV);
7459 /* Top directory uses NULL as the parent */
7460 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7463 /* All sub buffers have a descriptor */
7467 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7469 struct dentry *d_tracer;
7472 return tr->percpu_dir;
7474 d_tracer = tracing_get_dentry(tr);
7475 if (IS_ERR(d_tracer))
7478 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7480 WARN_ONCE(!tr->percpu_dir,
7481 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7483 return tr->percpu_dir;
7486 static struct dentry *
7487 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7488 void *data, long cpu, const struct file_operations *fops)
7490 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7492 if (ret) /* See tracing_get_cpu() */
7493 d_inode(ret)->i_cdev = (void *)(cpu + 1);
7498 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
7500 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
7501 struct dentry *d_cpu;
7502 char cpu_dir[30]; /* 30 characters should be more than enough */
7507 snprintf(cpu_dir, 30, "cpu%ld", cpu);
7508 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
7510 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
7514 /* per cpu trace_pipe */
7515 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
7516 tr, cpu, &tracing_pipe_fops);
7519 trace_create_cpu_file("trace", 0644, d_cpu,
7520 tr, cpu, &tracing_fops);
7522 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
7523 tr, cpu, &tracing_buffers_fops);
7525 trace_create_cpu_file("stats", 0444, d_cpu,
7526 tr, cpu, &tracing_stats_fops);
7528 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
7529 tr, cpu, &tracing_entries_fops);
7531 #ifdef CONFIG_TRACER_SNAPSHOT
7532 trace_create_cpu_file("snapshot", 0644, d_cpu,
7533 tr, cpu, &snapshot_fops);
7535 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
7536 tr, cpu, &snapshot_raw_fops);
7540 #ifdef CONFIG_FTRACE_SELFTEST
7541 /* Let selftest have access to static functions in this file */
7542 #include "trace_selftest.c"
7546 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7549 struct trace_option_dentry *topt = filp->private_data;
7552 if (topt->flags->val & topt->opt->bit)
7557 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7561 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7564 struct trace_option_dentry *topt = filp->private_data;
7568 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7572 if (val != 0 && val != 1)
7575 if (!!(topt->flags->val & topt->opt->bit) != val) {
7576 mutex_lock(&trace_types_lock);
7577 ret = __set_tracer_option(topt->tr, topt->flags,
7579 mutex_unlock(&trace_types_lock);
7590 static const struct file_operations trace_options_fops = {
7591 .open = tracing_open_generic,
7592 .read = trace_options_read,
7593 .write = trace_options_write,
7594 .llseek = generic_file_llseek,
7598 * In order to pass in both the trace_array descriptor as well as the index
7599 * to the flag that the trace option file represents, the trace_array
7600 * has a character array of trace_flags_index[], which holds the index
7601 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7602 * The address of this character array is passed to the flag option file
7603 * read/write callbacks.
7605 * In order to extract both the index and the trace_array descriptor,
7606 * get_tr_index() uses the following algorithm.
7610 * As the pointer itself contains the address of the index (remember
7613 * Then to get the trace_array descriptor, by subtracting that index
7614 * from the ptr, we get to the start of the index itself.
7616 * ptr - idx == &index[0]
7618 * Then a simple container_of() from that pointer gets us to the
7619 * trace_array descriptor.
7621 static void get_tr_index(void *data, struct trace_array **ptr,
7622 unsigned int *pindex)
7624 *pindex = *(unsigned char *)data;
7626 *ptr = container_of(data - *pindex, struct trace_array,
7631 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7634 void *tr_index = filp->private_data;
7635 struct trace_array *tr;
7639 get_tr_index(tr_index, &tr, &index);
7641 if (tr->trace_flags & (1 << index))
7646 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7650 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7653 void *tr_index = filp->private_data;
7654 struct trace_array *tr;
7659 get_tr_index(tr_index, &tr, &index);
7661 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7665 if (val != 0 && val != 1)
7668 mutex_lock(&trace_types_lock);
7669 ret = set_tracer_flag(tr, 1 << index, val);
7670 mutex_unlock(&trace_types_lock);
7680 static const struct file_operations trace_options_core_fops = {
7681 .open = tracing_open_generic,
7682 .read = trace_options_core_read,
7683 .write = trace_options_core_write,
7684 .llseek = generic_file_llseek,
7687 struct dentry *trace_create_file(const char *name,
7689 struct dentry *parent,
7691 const struct file_operations *fops)
7695 ret = tracefs_create_file(name, mode, parent, data, fops);
7697 pr_warn("Could not create tracefs '%s' entry\n", name);
7703 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
7705 struct dentry *d_tracer;
7710 d_tracer = tracing_get_dentry(tr);
7711 if (IS_ERR(d_tracer))
7714 tr->options = tracefs_create_dir("options", d_tracer);
7716 pr_warn("Could not create tracefs directory 'options'\n");
7724 create_trace_option_file(struct trace_array *tr,
7725 struct trace_option_dentry *topt,
7726 struct tracer_flags *flags,
7727 struct tracer_opt *opt)
7729 struct dentry *t_options;
7731 t_options = trace_options_init_dentry(tr);
7735 topt->flags = flags;
7739 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
7740 &trace_options_fops);
7745 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
7747 struct trace_option_dentry *topts;
7748 struct trace_options *tr_topts;
7749 struct tracer_flags *flags;
7750 struct tracer_opt *opts;
7757 flags = tracer->flags;
7759 if (!flags || !flags->opts)
7763 * If this is an instance, only create flags for tracers
7764 * the instance may have.
7766 if (!trace_ok_for_array(tracer, tr))
7769 for (i = 0; i < tr->nr_topts; i++) {
7770 /* Make sure there's no duplicate flags. */
7771 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
7777 for (cnt = 0; opts[cnt].name; cnt++)
7780 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
7784 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
7791 tr->topts = tr_topts;
7792 tr->topts[tr->nr_topts].tracer = tracer;
7793 tr->topts[tr->nr_topts].topts = topts;
7796 for (cnt = 0; opts[cnt].name; cnt++) {
7797 create_trace_option_file(tr, &topts[cnt], flags,
7799 WARN_ONCE(topts[cnt].entry == NULL,
7800 "Failed to create trace option: %s",
7805 static struct dentry *
7806 create_trace_option_core_file(struct trace_array *tr,
7807 const char *option, long index)
7809 struct dentry *t_options;
7811 t_options = trace_options_init_dentry(tr);
7815 return trace_create_file(option, 0644, t_options,
7816 (void *)&tr->trace_flags_index[index],
7817 &trace_options_core_fops);
7820 static void create_trace_options_dir(struct trace_array *tr)
7822 struct dentry *t_options;
7823 bool top_level = tr == &global_trace;
7826 t_options = trace_options_init_dentry(tr);
7830 for (i = 0; trace_options[i]; i++) {
7832 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
7833 create_trace_option_core_file(tr, trace_options[i], i);
7838 rb_simple_read(struct file *filp, char __user *ubuf,
7839 size_t cnt, loff_t *ppos)
7841 struct trace_array *tr = filp->private_data;
7845 r = tracer_tracing_is_on(tr);
7846 r = sprintf(buf, "%d\n", r);
7848 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7852 rb_simple_write(struct file *filp, const char __user *ubuf,
7853 size_t cnt, loff_t *ppos)
7855 struct trace_array *tr = filp->private_data;
7856 struct ring_buffer *buffer = tr->trace_buffer.buffer;
7860 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7865 mutex_lock(&trace_types_lock);
7866 if (!!val == tracer_tracing_is_on(tr)) {
7867 val = 0; /* do nothing */
7869 tracer_tracing_on(tr);
7870 if (tr->current_trace->start)
7871 tr->current_trace->start(tr);
7873 tracer_tracing_off(tr);
7874 if (tr->current_trace->stop)
7875 tr->current_trace->stop(tr);
7877 mutex_unlock(&trace_types_lock);
7885 static const struct file_operations rb_simple_fops = {
7886 .open = tracing_open_generic_tr,
7887 .read = rb_simple_read,
7888 .write = rb_simple_write,
7889 .release = tracing_release_generic_tr,
7890 .llseek = default_llseek,
7894 buffer_percent_read(struct file *filp, char __user *ubuf,
7895 size_t cnt, loff_t *ppos)
7897 struct trace_array *tr = filp->private_data;
7901 r = tr->buffer_percent;
7902 r = sprintf(buf, "%d\n", r);
7904 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7908 buffer_percent_write(struct file *filp, const char __user *ubuf,
7909 size_t cnt, loff_t *ppos)
7911 struct trace_array *tr = filp->private_data;
7915 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7925 tr->buffer_percent = val;
7932 static const struct file_operations buffer_percent_fops = {
7933 .open = tracing_open_generic_tr,
7934 .read = buffer_percent_read,
7935 .write = buffer_percent_write,
7936 .release = tracing_release_generic_tr,
7937 .llseek = default_llseek,
7940 struct dentry *trace_instance_dir;
7943 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
7946 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
7948 enum ring_buffer_flags rb_flags;
7950 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
7954 buf->buffer = ring_buffer_alloc(size, rb_flags);
7958 buf->data = alloc_percpu(struct trace_array_cpu);
7960 ring_buffer_free(buf->buffer);
7965 /* Allocate the first page for all buffers */
7966 set_buffer_entries(&tr->trace_buffer,
7967 ring_buffer_size(tr->trace_buffer.buffer, 0));
7972 static int allocate_trace_buffers(struct trace_array *tr, int size)
7976 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
7980 #ifdef CONFIG_TRACER_MAX_TRACE
7981 ret = allocate_trace_buffer(tr, &tr->max_buffer,
7982 allocate_snapshot ? size : 1);
7984 ring_buffer_free(tr->trace_buffer.buffer);
7985 tr->trace_buffer.buffer = NULL;
7986 free_percpu(tr->trace_buffer.data);
7987 tr->trace_buffer.data = NULL;
7990 tr->allocated_snapshot = allocate_snapshot;
7993 * Only the top level trace array gets its snapshot allocated
7994 * from the kernel command line.
7996 allocate_snapshot = false;
8001 static void free_trace_buffer(struct trace_buffer *buf)
8004 ring_buffer_free(buf->buffer);
8006 free_percpu(buf->data);
8011 static void free_trace_buffers(struct trace_array *tr)
8016 free_trace_buffer(&tr->trace_buffer);
8018 #ifdef CONFIG_TRACER_MAX_TRACE
8019 free_trace_buffer(&tr->max_buffer);
8023 static void init_trace_flags_index(struct trace_array *tr)
8027 /* Used by the trace options files */
8028 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8029 tr->trace_flags_index[i] = i;
8032 static void __update_tracer_options(struct trace_array *tr)
8036 for (t = trace_types; t; t = t->next)
8037 add_tracer_options(tr, t);
8040 static void update_tracer_options(struct trace_array *tr)
8042 mutex_lock(&trace_types_lock);
8043 __update_tracer_options(tr);
8044 mutex_unlock(&trace_types_lock);
8047 static int instance_mkdir(const char *name)
8049 struct trace_array *tr;
8052 mutex_lock(&event_mutex);
8053 mutex_lock(&trace_types_lock);
8056 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8057 if (tr->name && strcmp(tr->name, name) == 0)
8062 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8066 tr->name = kstrdup(name, GFP_KERNEL);
8070 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8073 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
8075 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8077 raw_spin_lock_init(&tr->start_lock);
8079 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8081 tr->current_trace = &nop_trace;
8083 INIT_LIST_HEAD(&tr->systems);
8084 INIT_LIST_HEAD(&tr->events);
8085 INIT_LIST_HEAD(&tr->hist_vars);
8087 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
8090 tr->dir = tracefs_create_dir(name, trace_instance_dir);
8094 ret = event_trace_add_tracer(tr->dir, tr);
8096 tracefs_remove_recursive(tr->dir);
8100 ftrace_init_trace_array(tr);
8102 init_tracer_tracefs(tr, tr->dir);
8103 init_trace_flags_index(tr);
8104 __update_tracer_options(tr);
8106 list_add(&tr->list, &ftrace_trace_arrays);
8108 mutex_unlock(&trace_types_lock);
8109 mutex_unlock(&event_mutex);
8114 free_trace_buffers(tr);
8115 free_cpumask_var(tr->tracing_cpumask);
8120 mutex_unlock(&trace_types_lock);
8121 mutex_unlock(&event_mutex);
8127 static int instance_rmdir(const char *name)
8129 struct trace_array *tr;
8134 mutex_lock(&event_mutex);
8135 mutex_lock(&trace_types_lock);
8138 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8139 if (tr->name && strcmp(tr->name, name) == 0) {
8148 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
8151 list_del(&tr->list);
8153 /* Disable all the flags that were enabled coming in */
8154 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8155 if ((1 << i) & ZEROED_TRACE_FLAGS)
8156 set_tracer_flag(tr, 1 << i, 0);
8159 tracing_set_nop(tr);
8160 clear_ftrace_function_probes(tr);
8161 event_trace_del_tracer(tr);
8162 ftrace_clear_pids(tr);
8163 ftrace_destroy_function_files(tr);
8164 tracefs_remove_recursive(tr->dir);
8165 free_trace_buffers(tr);
8167 for (i = 0; i < tr->nr_topts; i++) {
8168 kfree(tr->topts[i].topts);
8172 free_cpumask_var(tr->tracing_cpumask);
8179 mutex_unlock(&trace_types_lock);
8180 mutex_unlock(&event_mutex);
8185 static __init void create_trace_instances(struct dentry *d_tracer)
8187 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8190 if (WARN_ON(!trace_instance_dir))
8195 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8197 struct trace_event_file *file;
8200 trace_create_file("available_tracers", 0444, d_tracer,
8201 tr, &show_traces_fops);
8203 trace_create_file("current_tracer", 0644, d_tracer,
8204 tr, &set_tracer_fops);
8206 trace_create_file("tracing_cpumask", 0644, d_tracer,
8207 tr, &tracing_cpumask_fops);
8209 trace_create_file("trace_options", 0644, d_tracer,
8210 tr, &tracing_iter_fops);
8212 trace_create_file("trace", 0644, d_tracer,
8215 trace_create_file("trace_pipe", 0444, d_tracer,
8216 tr, &tracing_pipe_fops);
8218 trace_create_file("buffer_size_kb", 0644, d_tracer,
8219 tr, &tracing_entries_fops);
8221 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
8222 tr, &tracing_total_entries_fops);
8224 trace_create_file("free_buffer", 0200, d_tracer,
8225 tr, &tracing_free_buffer_fops);
8227 trace_create_file("trace_marker", 0220, d_tracer,
8228 tr, &tracing_mark_fops);
8230 file = __find_event_file(tr, "ftrace", "print");
8231 if (file && file->dir)
8232 trace_create_file("trigger", 0644, file->dir, file,
8233 &event_trigger_fops);
8234 tr->trace_marker_file = file;
8236 trace_create_file("trace_marker_raw", 0220, d_tracer,
8237 tr, &tracing_mark_raw_fops);
8239 trace_create_file("trace_clock", 0644, d_tracer, tr,
8242 trace_create_file("tracing_on", 0644, d_tracer,
8243 tr, &rb_simple_fops);
8245 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8246 &trace_time_stamp_mode_fops);
8248 tr->buffer_percent = 50;
8250 trace_create_file("buffer_percent", 0444, d_tracer,
8251 tr, &buffer_percent_fops);
8253 create_trace_options_dir(tr);
8255 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
8256 trace_create_file("tracing_max_latency", 0644, d_tracer,
8257 &tr->max_latency, &tracing_max_lat_fops);
8260 if (ftrace_create_function_files(tr, d_tracer))
8261 WARN(1, "Could not allocate function filter files");
8263 #ifdef CONFIG_TRACER_SNAPSHOT
8264 trace_create_file("snapshot", 0644, d_tracer,
8265 tr, &snapshot_fops);
8268 for_each_tracing_cpu(cpu)
8269 tracing_init_tracefs_percpu(tr, cpu);
8271 ftrace_init_tracefs(tr, d_tracer);
8274 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
8276 struct vfsmount *mnt;
8277 struct file_system_type *type;
8280 * To maintain backward compatibility for tools that mount
8281 * debugfs to get to the tracing facility, tracefs is automatically
8282 * mounted to the debugfs/tracing directory.
8284 type = get_fs_type("tracefs");
8287 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
8288 put_filesystem(type);
8297 * tracing_init_dentry - initialize top level trace array
8299 * This is called when creating files or directories in the tracing
8300 * directory. It is called via fs_initcall() by any of the boot up code
8301 * and expects to return the dentry of the top level tracing directory.
8303 struct dentry *tracing_init_dentry(void)
8305 struct trace_array *tr = &global_trace;
8307 /* The top level trace array uses NULL as parent */
8311 if (WARN_ON(!tracefs_initialized()) ||
8312 (IS_ENABLED(CONFIG_DEBUG_FS) &&
8313 WARN_ON(!debugfs_initialized())))
8314 return ERR_PTR(-ENODEV);
8317 * As there may still be users that expect the tracing
8318 * files to exist in debugfs/tracing, we must automount
8319 * the tracefs file system there, so older tools still
8320 * work with the newer kerenl.
8322 tr->dir = debugfs_create_automount("tracing", NULL,
8323 trace_automount, NULL);
8325 pr_warn_once("Could not create debugfs directory 'tracing'\n");
8326 return ERR_PTR(-ENOMEM);
8332 extern struct trace_eval_map *__start_ftrace_eval_maps[];
8333 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8335 static void __init trace_eval_init(void)
8339 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8340 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8343 #ifdef CONFIG_MODULES
8344 static void trace_module_add_evals(struct module *mod)
8346 if (!mod->num_trace_evals)
8350 * Modules with bad taint do not have events created, do
8351 * not bother with enums either.
8353 if (trace_module_has_bad_taint(mod))
8356 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
8359 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
8360 static void trace_module_remove_evals(struct module *mod)
8362 union trace_eval_map_item *map;
8363 union trace_eval_map_item **last = &trace_eval_maps;
8365 if (!mod->num_trace_evals)
8368 mutex_lock(&trace_eval_mutex);
8370 map = trace_eval_maps;
8373 if (map->head.mod == mod)
8375 map = trace_eval_jmp_to_tail(map);
8376 last = &map->tail.next;
8377 map = map->tail.next;
8382 *last = trace_eval_jmp_to_tail(map)->tail.next;
8385 mutex_unlock(&trace_eval_mutex);
8388 static inline void trace_module_remove_evals(struct module *mod) { }
8389 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
8391 static int trace_module_notify(struct notifier_block *self,
8392 unsigned long val, void *data)
8394 struct module *mod = data;
8397 case MODULE_STATE_COMING:
8398 trace_module_add_evals(mod);
8400 case MODULE_STATE_GOING:
8401 trace_module_remove_evals(mod);
8408 static struct notifier_block trace_module_nb = {
8409 .notifier_call = trace_module_notify,
8412 #endif /* CONFIG_MODULES */
8414 static __init int tracer_init_tracefs(void)
8416 struct dentry *d_tracer;
8418 trace_access_lock_init();
8420 d_tracer = tracing_init_dentry();
8421 if (IS_ERR(d_tracer))
8426 init_tracer_tracefs(&global_trace, d_tracer);
8427 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
8429 trace_create_file("tracing_thresh", 0644, d_tracer,
8430 &global_trace, &tracing_thresh_fops);
8432 trace_create_file("README", 0444, d_tracer,
8433 NULL, &tracing_readme_fops);
8435 trace_create_file("saved_cmdlines", 0444, d_tracer,
8436 NULL, &tracing_saved_cmdlines_fops);
8438 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
8439 NULL, &tracing_saved_cmdlines_size_fops);
8441 trace_create_file("saved_tgids", 0444, d_tracer,
8442 NULL, &tracing_saved_tgids_fops);
8446 trace_create_eval_file(d_tracer);
8448 #ifdef CONFIG_MODULES
8449 register_module_notifier(&trace_module_nb);
8452 #ifdef CONFIG_DYNAMIC_FTRACE
8453 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8454 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
8457 create_trace_instances(d_tracer);
8459 update_tracer_options(&global_trace);
8464 static int trace_panic_handler(struct notifier_block *this,
8465 unsigned long event, void *unused)
8467 if (ftrace_dump_on_oops)
8468 ftrace_dump(ftrace_dump_on_oops);
8472 static struct notifier_block trace_panic_notifier = {
8473 .notifier_call = trace_panic_handler,
8475 .priority = 150 /* priority: INT_MAX >= x >= 0 */
8478 static int trace_die_handler(struct notifier_block *self,
8484 if (ftrace_dump_on_oops)
8485 ftrace_dump(ftrace_dump_on_oops);
8493 static struct notifier_block trace_die_notifier = {
8494 .notifier_call = trace_die_handler,
8499 * printk is set to max of 1024, we really don't need it that big.
8500 * Nothing should be printing 1000 characters anyway.
8502 #define TRACE_MAX_PRINT 1000
8505 * Define here KERN_TRACE so that we have one place to modify
8506 * it if we decide to change what log level the ftrace dump
8509 #define KERN_TRACE KERN_EMERG
8512 trace_printk_seq(struct trace_seq *s)
8514 /* Probably should print a warning here. */
8515 if (s->seq.len >= TRACE_MAX_PRINT)
8516 s->seq.len = TRACE_MAX_PRINT;
8519 * More paranoid code. Although the buffer size is set to
8520 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
8521 * an extra layer of protection.
8523 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
8524 s->seq.len = s->seq.size - 1;
8526 /* should be zero ended, but we are paranoid. */
8527 s->buffer[s->seq.len] = 0;
8529 printk(KERN_TRACE "%s", s->buffer);
8534 void trace_init_global_iter(struct trace_iterator *iter)
8536 iter->tr = &global_trace;
8537 iter->trace = iter->tr->current_trace;
8538 iter->cpu_file = RING_BUFFER_ALL_CPUS;
8539 iter->trace_buffer = &global_trace.trace_buffer;
8541 if (iter->trace && iter->trace->open)
8542 iter->trace->open(iter);
8544 /* Annotate start of buffers if we had overruns */
8545 if (ring_buffer_overruns(iter->trace_buffer->buffer))
8546 iter->iter_flags |= TRACE_FILE_ANNOTATE;
8548 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
8549 if (trace_clocks[iter->tr->clock_id].in_ns)
8550 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
8553 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
8555 /* use static because iter can be a bit big for the stack */
8556 static struct trace_iterator iter;
8557 static atomic_t dump_running;
8558 struct trace_array *tr = &global_trace;
8559 unsigned int old_userobj;
8560 unsigned long flags;
8563 /* Only allow one dump user at a time. */
8564 if (atomic_inc_return(&dump_running) != 1) {
8565 atomic_dec(&dump_running);
8570 * Always turn off tracing when we dump.
8571 * We don't need to show trace output of what happens
8572 * between multiple crashes.
8574 * If the user does a sysrq-z, then they can re-enable
8575 * tracing with echo 1 > tracing_on.
8579 local_irq_save(flags);
8580 printk_nmi_direct_enter();
8582 /* Simulate the iterator */
8583 trace_init_global_iter(&iter);
8585 for_each_tracing_cpu(cpu) {
8586 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
8589 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
8591 /* don't look at user memory in panic mode */
8592 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
8594 switch (oops_dump_mode) {
8596 iter.cpu_file = RING_BUFFER_ALL_CPUS;
8599 iter.cpu_file = raw_smp_processor_id();
8604 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
8605 iter.cpu_file = RING_BUFFER_ALL_CPUS;
8608 printk(KERN_TRACE "Dumping ftrace buffer:\n");
8610 /* Did function tracer already get disabled? */
8611 if (ftrace_is_dead()) {
8612 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
8613 printk("# MAY BE MISSING FUNCTION EVENTS\n");
8617 * We need to stop all tracing on all CPUS to read the
8618 * the next buffer. This is a bit expensive, but is
8619 * not done often. We fill all what we can read,
8620 * and then release the locks again.
8623 while (!trace_empty(&iter)) {
8626 printk(KERN_TRACE "---------------------------------\n");
8630 /* reset all but tr, trace, and overruns */
8631 memset(&iter.seq, 0,
8632 sizeof(struct trace_iterator) -
8633 offsetof(struct trace_iterator, seq));
8634 iter.iter_flags |= TRACE_FILE_LAT_FMT;
8637 if (trace_find_next_entry_inc(&iter) != NULL) {
8640 ret = print_trace_line(&iter);
8641 if (ret != TRACE_TYPE_NO_CONSUME)
8642 trace_consume(&iter);
8644 touch_nmi_watchdog();
8646 trace_printk_seq(&iter.seq);
8650 printk(KERN_TRACE " (ftrace buffer empty)\n");
8652 printk(KERN_TRACE "---------------------------------\n");
8655 tr->trace_flags |= old_userobj;
8657 for_each_tracing_cpu(cpu) {
8658 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
8660 atomic_dec(&dump_running);
8661 printk_nmi_direct_exit();
8662 local_irq_restore(flags);
8664 EXPORT_SYMBOL_GPL(ftrace_dump);
8666 int trace_run_command(const char *buf, int (*createfn)(int, char **))
8673 argv = argv_split(GFP_KERNEL, buf, &argc);
8678 ret = createfn(argc, argv);
8685 #define WRITE_BUFSIZE 4096
8687 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
8688 size_t count, loff_t *ppos,
8689 int (*createfn)(int, char **))
8691 char *kbuf, *buf, *tmp;
8696 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
8700 while (done < count) {
8701 size = count - done;
8703 if (size >= WRITE_BUFSIZE)
8704 size = WRITE_BUFSIZE - 1;
8706 if (copy_from_user(kbuf, buffer + done, size)) {
8713 tmp = strchr(buf, '\n');
8716 size = tmp - buf + 1;
8719 if (done + size < count) {
8722 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
8723 pr_warn("Line length is too long: Should be less than %d\n",
8731 /* Remove comments */
8732 tmp = strchr(buf, '#');
8737 ret = trace_run_command(buf, createfn);
8742 } while (done < count);
8752 __init static int tracer_alloc_buffers(void)
8758 * Make sure we don't accidently add more trace options
8759 * than we have bits for.
8761 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
8763 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
8766 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
8767 goto out_free_buffer_mask;
8769 /* Only allocate trace_printk buffers if a trace_printk exists */
8770 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
8771 /* Must be called before global_trace.buffer is allocated */
8772 trace_printk_init_buffers();
8774 /* To save memory, keep the ring buffer size to its minimum */
8775 if (ring_buffer_expanded)
8776 ring_buf_size = trace_buf_size;
8780 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
8781 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
8783 raw_spin_lock_init(&global_trace.start_lock);
8786 * The prepare callbacks allocates some memory for the ring buffer. We
8787 * don't free the buffer if the if the CPU goes down. If we were to free
8788 * the buffer, then the user would lose any trace that was in the
8789 * buffer. The memory will be removed once the "instance" is removed.
8791 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
8792 "trace/RB:preapre", trace_rb_cpu_prepare,
8795 goto out_free_cpumask;
8796 /* Used for event triggers */
8798 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
8800 goto out_rm_hp_state;
8802 if (trace_create_savedcmd() < 0)
8803 goto out_free_temp_buffer;
8805 /* TODO: make the number of buffers hot pluggable with CPUS */
8806 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
8807 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
8809 goto out_free_savedcmd;
8812 if (global_trace.buffer_disabled)
8815 if (trace_boot_clock) {
8816 ret = tracing_set_clock(&global_trace, trace_boot_clock);
8818 pr_warn("Trace clock %s not defined, going back to default\n",
8823 * register_tracer() might reference current_trace, so it
8824 * needs to be set before we register anything. This is
8825 * just a bootstrap of current_trace anyway.
8827 global_trace.current_trace = &nop_trace;
8829 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8831 ftrace_init_global_array_ops(&global_trace);
8833 init_trace_flags_index(&global_trace);
8835 register_tracer(&nop_trace);
8837 /* Function tracing may start here (via kernel command line) */
8838 init_function_trace();
8840 /* All seems OK, enable tracing */
8841 tracing_disabled = 0;
8843 atomic_notifier_chain_register(&panic_notifier_list,
8844 &trace_panic_notifier);
8846 register_die_notifier(&trace_die_notifier);
8848 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
8850 INIT_LIST_HEAD(&global_trace.systems);
8851 INIT_LIST_HEAD(&global_trace.events);
8852 INIT_LIST_HEAD(&global_trace.hist_vars);
8853 list_add(&global_trace.list, &ftrace_trace_arrays);
8855 apply_trace_boot_options();
8857 register_snapshot_cmd();
8862 free_saved_cmdlines_buffer(savedcmd);
8863 out_free_temp_buffer:
8864 ring_buffer_free(temp_buffer);
8866 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
8868 free_cpumask_var(global_trace.tracing_cpumask);
8869 out_free_buffer_mask:
8870 free_cpumask_var(tracing_buffer_mask);
8875 void __init early_trace_init(void)
8877 if (tracepoint_printk) {
8878 tracepoint_print_iter =
8879 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
8880 if (WARN_ON(!tracepoint_print_iter))
8881 tracepoint_printk = 0;
8883 static_key_enable(&tracepoint_printk_key.key);
8885 tracer_alloc_buffers();
8888 void __init trace_init(void)
8893 __init static int clear_boot_tracer(void)
8896 * The default tracer at boot buffer is an init section.
8897 * This function is called in lateinit. If we did not
8898 * find the boot tracer, then clear it out, to prevent
8899 * later registration from accessing the buffer that is
8900 * about to be freed.
8902 if (!default_bootup_tracer)
8905 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
8906 default_bootup_tracer);
8907 default_bootup_tracer = NULL;
8912 fs_initcall(tracer_init_tracefs);
8913 late_initcall_sync(clear_boot_tracer);
8915 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
8916 __init static int tracing_set_default_clock(void)
8918 /* sched_clock_stable() is determined in late_initcall */
8919 if (!trace_boot_clock && !sched_clock_stable()) {
8921 "Unstable clock detected, switching default tracing clock to \"global\"\n"
8922 "If you want to keep using the local clock, then add:\n"
8923 " \"trace_clock=local\"\n"
8924 "on the kernel command line\n");
8925 tracing_set_clock(&global_trace, "global");
8930 late_initcall_sync(tracing_set_default_clock);