1 // SPDX-License-Identifier: GPL-2.0
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
19 #include <linux/kthread.h>
20 #include <linux/tracefs.h>
21 #include <linux/uaccess.h>
22 #include <linux/cpumask.h>
23 #include <linux/delay.h>
24 #include <linux/sched/clock.h>
25 #include <uapi/linux/sched/types.h>
26 #include <linux/sched.h>
29 #ifdef CONFIG_X86_LOCAL_APIC
30 #include <asm/trace/irq_vectors.h>
31 #undef TRACE_INCLUDE_PATH
32 #undef TRACE_INCLUDE_FILE
33 #endif /* CONFIG_X86_LOCAL_APIC */
35 #include <trace/events/irq.h>
36 #include <trace/events/sched.h>
38 #define CREATE_TRACE_POINTS
39 #include <trace/events/osnoise.h>
41 static struct trace_array *osnoise_trace;
46 #define BANNER "osnoise: "
47 #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
48 #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
50 #define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
51 #define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
71 #define THREAD_CONTEXT 1
73 * sofirq runtime info.
82 * thread runtime info.
91 * Runtime information: this structure saves the runtime information used by
92 * one sampling thread.
94 struct osnoise_variables {
95 struct task_struct *kthread;
100 struct osn_softirq softirq;
101 struct osn_thread thread;
106 * Per-cpu runtime information.
108 DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
111 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
113 static inline struct osnoise_variables *this_cpu_osn_var(void)
115 return this_cpu_ptr(&per_cpu_osnoise_var);
118 #ifdef CONFIG_TIMERLAT_TRACER
120 * Runtime information for the timer mode.
122 struct timerlat_variables {
123 struct task_struct *kthread;
124 struct hrtimer timer;
131 DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
134 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
136 static inline struct timerlat_variables *this_cpu_tmr_var(void)
138 return this_cpu_ptr(&per_cpu_timerlat_var);
142 * tlat_var_reset - Reset the values of the given timerlat_variables
144 static inline void tlat_var_reset(void)
146 struct timerlat_variables *tlat_var;
149 * So far, all the values are initialized as 0, so
150 * zeroing the structure is perfect.
152 for_each_cpu(cpu, cpu_online_mask) {
153 tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
154 memset(tlat_var, 0, sizeof(*tlat_var));
157 #else /* CONFIG_TIMERLAT_TRACER */
158 #define tlat_var_reset() do {} while (0)
159 #endif /* CONFIG_TIMERLAT_TRACER */
162 * osn_var_reset - Reset the values of the given osnoise_variables
164 static inline void osn_var_reset(void)
166 struct osnoise_variables *osn_var;
170 * So far, all the values are initialized as 0, so
171 * zeroing the structure is perfect.
173 for_each_cpu(cpu, cpu_online_mask) {
174 osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
175 memset(osn_var, 0, sizeof(*osn_var));
180 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
182 static inline void osn_var_reset_all(void)
189 * Tells NMIs to call back to the osnoise tracer to record timestamps.
191 bool trace_osnoise_callback_enabled;
194 * osnoise sample structure definition. Used to store the statistics of a
197 struct osnoise_sample {
198 u64 runtime; /* runtime */
199 u64 noise; /* noise */
200 u64 max_sample; /* max single noise sample */
201 int hw_count; /* # HW (incl. hypervisor) interference */
202 int nmi_count; /* # NMIs during this sample */
203 int irq_count; /* # IRQs during this sample */
204 int softirq_count; /* # softirqs during this sample */
205 int thread_count; /* # threads during this sample */
208 #ifdef CONFIG_TIMERLAT_TRACER
210 * timerlat sample structure definition. Used to store the statistics of
213 struct timerlat_sample {
214 u64 timer_latency; /* timer_latency */
215 unsigned int seqnum; /* unique sequence */
216 int context; /* timer context */
221 * Protect the interface.
223 struct mutex interface_lock;
228 static struct osnoise_data {
229 u64 sample_period; /* total sampling period */
230 u64 sample_runtime; /* active sampling portion of period */
231 u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
232 u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
233 #ifdef CONFIG_TIMERLAT_TRACER
234 u64 timerlat_period; /* timerlat period */
235 u64 print_stack; /* print IRQ stack if total > */
236 int timerlat_tracer; /* timerlat tracer */
238 bool tainted; /* infor users and developers about a problem */
240 .sample_period = DEFAULT_SAMPLE_PERIOD,
241 .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
243 .stop_tracing_total = 0,
244 #ifdef CONFIG_TIMERLAT_TRACER
246 .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
247 .timerlat_tracer = 0,
252 * Boolean variable used to inform that the tracer is currently sampling.
254 static bool osnoise_busy;
256 #ifdef CONFIG_PREEMPT_RT
258 * Print the osnoise header info.
260 static void print_osnoise_headers(struct seq_file *s)
262 if (osnoise_data.tainted)
263 seq_puts(s, "# osnoise is tainted!\n");
265 seq_puts(s, "# _-------=> irqs-off\n");
266 seq_puts(s, "# / _------=> need-resched\n");
267 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
268 seq_puts(s, "# || / _----=> hardirq/softirq\n");
269 seq_puts(s, "# ||| / _---=> preempt-depth\n");
270 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
271 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
273 seq_puts(s, "# |||||| / ");
274 seq_puts(s, " MAX\n");
276 seq_puts(s, "# ||||| / ");
277 seq_puts(s, " SINGLE Interference counters:\n");
279 seq_puts(s, "# ||||||| RUNTIME ");
280 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
282 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
283 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
285 seq_puts(s, "# | | | ||||||| | | ");
286 seq_puts(s, " | | | | | | | |\n");
288 #else /* CONFIG_PREEMPT_RT */
289 static void print_osnoise_headers(struct seq_file *s)
291 if (osnoise_data.tainted)
292 seq_puts(s, "# osnoise is tainted!\n");
294 seq_puts(s, "# _-----=> irqs-off\n");
295 seq_puts(s, "# / _----=> need-resched\n");
296 seq_puts(s, "# | / _---=> hardirq/softirq\n");
297 seq_puts(s, "# || / _--=> preempt-depth\n");
298 seq_puts(s, "# ||| / _-=> migrate-disable ");
299 seq_puts(s, " MAX\n");
300 seq_puts(s, "# |||| / delay ");
301 seq_puts(s, " SINGLE Interference counters:\n");
303 seq_puts(s, "# ||||| RUNTIME ");
304 seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
306 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
307 seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
309 seq_puts(s, "# | | | ||||| | | ");
310 seq_puts(s, " | | | | | | | |\n");
312 #endif /* CONFIG_PREEMPT_RT */
315 * osnoise_taint - report an osnoise error.
317 #define osnoise_taint(msg) ({ \
318 struct trace_array *tr = osnoise_trace; \
320 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg); \
321 osnoise_data.tainted = true; \
325 * Record an osnoise_sample into the tracer buffer.
327 static void trace_osnoise_sample(struct osnoise_sample *sample)
329 struct trace_array *tr = osnoise_trace;
330 struct trace_buffer *buffer = tr->array_buffer.buffer;
331 struct trace_event_call *call = &event_osnoise;
332 struct ring_buffer_event *event;
333 struct osnoise_entry *entry;
335 event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
339 entry = ring_buffer_event_data(event);
340 entry->runtime = sample->runtime;
341 entry->noise = sample->noise;
342 entry->max_sample = sample->max_sample;
343 entry->hw_count = sample->hw_count;
344 entry->nmi_count = sample->nmi_count;
345 entry->irq_count = sample->irq_count;
346 entry->softirq_count = sample->softirq_count;
347 entry->thread_count = sample->thread_count;
349 if (!call_filter_check_discard(call, entry, buffer, event))
350 trace_buffer_unlock_commit_nostack(buffer, event);
353 #ifdef CONFIG_TIMERLAT_TRACER
355 * Print the timerlat header info.
357 #ifdef CONFIG_PREEMPT_RT
358 static void print_timerlat_headers(struct seq_file *s)
360 seq_puts(s, "# _-------=> irqs-off\n");
361 seq_puts(s, "# / _------=> need-resched\n");
362 seq_puts(s, "# | / _-----=> need-resched-lazy\n");
363 seq_puts(s, "# || / _----=> hardirq/softirq\n");
364 seq_puts(s, "# ||| / _---=> preempt-depth\n");
365 seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
366 seq_puts(s, "# ||||| / _-=> migrate-disable\n");
367 seq_puts(s, "# |||||| /\n");
368 seq_puts(s, "# ||||||| ACTIVATION\n");
369 seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
370 seq_puts(s, " CONTEXT LATENCY\n");
371 seq_puts(s, "# | | | ||||||| | | ");
372 seq_puts(s, " | |\n");
374 #else /* CONFIG_PREEMPT_RT */
375 static void print_timerlat_headers(struct seq_file *s)
377 seq_puts(s, "# _-----=> irqs-off\n");
378 seq_puts(s, "# / _----=> need-resched\n");
379 seq_puts(s, "# | / _---=> hardirq/softirq\n");
380 seq_puts(s, "# || / _--=> preempt-depth\n");
381 seq_puts(s, "# ||| / _-=> migrate-disable\n");
382 seq_puts(s, "# |||| / delay\n");
383 seq_puts(s, "# ||||| ACTIVATION\n");
384 seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
385 seq_puts(s, " CONTEXT LATENCY\n");
386 seq_puts(s, "# | | | ||||| | | ");
387 seq_puts(s, " | |\n");
389 #endif /* CONFIG_PREEMPT_RT */
392 * Record an timerlat_sample into the tracer buffer.
394 static void trace_timerlat_sample(struct timerlat_sample *sample)
396 struct trace_array *tr = osnoise_trace;
397 struct trace_event_call *call = &event_osnoise;
398 struct trace_buffer *buffer = tr->array_buffer.buffer;
399 struct ring_buffer_event *event;
400 struct timerlat_entry *entry;
402 event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
406 entry = ring_buffer_event_data(event);
407 entry->seqnum = sample->seqnum;
408 entry->context = sample->context;
409 entry->timer_latency = sample->timer_latency;
411 if (!call_filter_check_discard(call, entry, buffer, event))
412 trace_buffer_unlock_commit_nostack(buffer, event);
415 #ifdef CONFIG_STACKTRACE
417 #define MAX_CALLS 256
420 * Stack trace will take place only at IRQ level, so, no need
421 * to control nesting here.
426 unsigned long calls[MAX_CALLS];
429 static DEFINE_PER_CPU(struct trace_stack, trace_stack);
432 * timerlat_save_stack - save a stack trace without printing
434 * Save the current stack trace without printing. The
435 * stack will be printed later, after the end of the measurement.
437 static void timerlat_save_stack(int skip)
439 unsigned int size, nr_entries;
440 struct trace_stack *fstack;
442 fstack = this_cpu_ptr(&trace_stack);
444 size = ARRAY_SIZE(fstack->calls);
446 nr_entries = stack_trace_save(fstack->calls, size, skip);
448 fstack->stack_size = nr_entries * sizeof(unsigned long);
449 fstack->nr_entries = nr_entries;
455 * timerlat_dump_stack - dump a stack trace previously saved
457 * Dump a saved stack trace into the trace buffer.
459 static void timerlat_dump_stack(void)
461 struct trace_event_call *call = &event_osnoise;
462 struct trace_array *tr = osnoise_trace;
463 struct trace_buffer *buffer = tr->array_buffer.buffer;
464 struct ring_buffer_event *event;
465 struct trace_stack *fstack;
466 struct stack_entry *entry;
469 preempt_disable_notrace();
470 fstack = this_cpu_ptr(&trace_stack);
471 size = fstack->stack_size;
473 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
478 entry = ring_buffer_event_data(event);
480 memcpy(&entry->caller, fstack->calls, size);
481 entry->size = fstack->nr_entries;
483 if (!call_filter_check_discard(call, entry, buffer, event))
484 trace_buffer_unlock_commit_nostack(buffer, event);
487 preempt_enable_notrace();
490 #define timerlat_dump_stack() do {} while (0)
491 #define timerlat_save_stack(a) do {} while (0)
492 #endif /* CONFIG_STACKTRACE */
493 #endif /* CONFIG_TIMERLAT_TRACER */
496 * Macros to encapsulate the time capturing infrastructure.
498 #define time_get() trace_clock_local()
499 #define time_to_us(x) div_u64(x, 1000)
500 #define time_sub(a, b) ((a) - (b))
503 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
505 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
506 * to discount the NMI interference.
508 * See get_int_safe_duration().
511 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
513 if (osn_var->irq.delta_start)
514 osn_var->irq.delta_start += duration;
517 #ifndef CONFIG_PREEMPT_RT
519 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
521 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
522 * forward to discount the interference.
524 * See get_int_safe_duration().
527 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
529 if (osn_var->softirq.delta_start)
530 osn_var->softirq.delta_start += duration;
532 #else /* CONFIG_PREEMPT_RT */
533 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
537 * cond_move_thread_delta_start - Forward the delta_start of a running thread
539 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
540 * is pushed forward to discount the interference.
542 * See get_int_safe_duration().
545 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
547 if (osn_var->thread.delta_start)
548 osn_var->thread.delta_start += duration;
552 * get_int_safe_duration - Get the duration of a window
554 * The irq, softirq and thread varaibles need to have its duration without
555 * the interference from higher priority interrupts. Instead of keeping a
556 * variable to discount the interrupt interference from these variables, the
557 * starting time of these variables are pushed forward with the interrupt's
558 * duration. In this way, a single variable is used to:
560 * - Know if a given window is being measured.
561 * - Account its duration.
562 * - Discount the interference.
564 * To avoid getting inconsistent values, e.g.,:
568 * delta_start -= int duration;
570 * duration = now - delta_start;
572 * result: negative duration if the variable duration before the
573 * interrupt was smaller than the interrupt execution.
575 * A counter of interrupts is used. If the counter increased, try
576 * to capture an interference safe duration.
579 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
581 u64 int_counter, now;
585 int_counter = local_read(&osn_var->int_counter);
586 /* synchronize with interrupts */
590 duration = (now - *delta_start);
592 /* synchronize with interrupts */
594 } while (int_counter != local_read(&osn_var->int_counter));
597 * This is an evidence of race conditions that cause
598 * a value to be "discounted" too much.
601 osnoise_taint("Negative duration!\n");
610 * set_int_safe_time - Save the current time on *time, aware of interference
612 * Get the time, taking into consideration a possible interference from
613 * higher priority interrupts.
615 * See get_int_safe_duration() for an explanation.
618 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
623 int_counter = local_read(&osn_var->int_counter);
624 /* synchronize with interrupts */
629 /* synchronize with interrupts */
631 } while (int_counter != local_read(&osn_var->int_counter));
636 #ifdef CONFIG_TIMERLAT_TRACER
638 * copy_int_safe_time - Copy *src into *desc aware of interference
641 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
646 int_counter = local_read(&osn_var->int_counter);
647 /* synchronize with interrupts */
652 /* synchronize with interrupts */
654 } while (int_counter != local_read(&osn_var->int_counter));
658 #endif /* CONFIG_TIMERLAT_TRACER */
661 * trace_osnoise_callback - NMI entry/exit callback
663 * This function is called at the entry and exit NMI code. The bool enter
664 * distinguishes between either case. This function is used to note a NMI
665 * occurrence, compute the noise caused by the NMI, and to remove the noise
666 * it is potentially causing on other interference variables.
668 void trace_osnoise_callback(bool enter)
670 struct osnoise_variables *osn_var = this_cpu_osn_var();
673 if (!osn_var->sampling)
677 * Currently trace_clock_local() calls sched_clock() and the
678 * generic version is not NMI safe.
680 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
682 osn_var->nmi.delta_start = time_get();
683 local_inc(&osn_var->int_counter);
685 duration = time_get() - osn_var->nmi.delta_start;
687 trace_nmi_noise(osn_var->nmi.delta_start, duration);
689 cond_move_irq_delta_start(osn_var, duration);
690 cond_move_softirq_delta_start(osn_var, duration);
691 cond_move_thread_delta_start(osn_var, duration);
696 osn_var->nmi.count++;
700 * osnoise_trace_irq_entry - Note the starting of an IRQ
702 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
703 * it is safe to use a single variable (ons_var->irq) to save the statistics.
704 * The arrival_time is used to report... the arrival time. The delta_start
705 * is used to compute the duration at the IRQ exit handler. See
706 * cond_move_irq_delta_start().
708 void osnoise_trace_irq_entry(int id)
710 struct osnoise_variables *osn_var = this_cpu_osn_var();
712 if (!osn_var->sampling)
715 * This value will be used in the report, but not to compute
716 * the execution time, so it is safe to get it unsafe.
718 osn_var->irq.arrival_time = time_get();
719 set_int_safe_time(osn_var, &osn_var->irq.delta_start);
720 osn_var->irq.count++;
722 local_inc(&osn_var->int_counter);
726 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
728 * Computes the duration of the IRQ noise, and trace it. Also discounts the
729 * interference from other sources of noise could be currently being accounted.
731 void osnoise_trace_irq_exit(int id, const char *desc)
733 struct osnoise_variables *osn_var = this_cpu_osn_var();
736 if (!osn_var->sampling)
739 duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
740 trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
741 osn_var->irq.arrival_time = 0;
742 cond_move_softirq_delta_start(osn_var, duration);
743 cond_move_thread_delta_start(osn_var, duration);
747 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
749 * Used to note the starting of an IRQ occurece.
751 static void trace_irqentry_callback(void *data, int irq,
752 struct irqaction *action)
754 osnoise_trace_irq_entry(irq);
758 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
760 * Used to note the end of an IRQ occurece.
762 static void trace_irqexit_callback(void *data, int irq,
763 struct irqaction *action, int ret)
765 osnoise_trace_irq_exit(irq, action->name);
769 * arch specific register function.
771 int __weak osnoise_arch_register(void)
777 * arch specific unregister function.
779 void __weak osnoise_arch_unregister(void)
785 * hook_irq_events - Hook IRQ handling events
787 * This function hooks the IRQ related callbacks to the respective trace
790 static int hook_irq_events(void)
794 ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
798 ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
800 goto out_unregister_entry;
802 ret = osnoise_arch_register();
809 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
810 out_unregister_entry:
811 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
817 * unhook_irq_events - Unhook IRQ handling events
819 * This function unhooks the IRQ related callbacks to the respective trace
822 static void unhook_irq_events(void)
824 osnoise_arch_unregister();
825 unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
826 unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
829 #ifndef CONFIG_PREEMPT_RT
831 * trace_softirq_entry_callback - Note the starting of a softirq
833 * Save the starting time of a softirq. As softirqs are non-preemptive to
834 * other softirqs, it is safe to use a single variable (ons_var->softirq)
835 * to save the statistics. The arrival_time is used to report... the
836 * arrival time. The delta_start is used to compute the duration at the
837 * softirq exit handler. See cond_move_softirq_delta_start().
839 static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
841 struct osnoise_variables *osn_var = this_cpu_osn_var();
843 if (!osn_var->sampling)
846 * This value will be used in the report, but not to compute
847 * the execution time, so it is safe to get it unsafe.
849 osn_var->softirq.arrival_time = time_get();
850 set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
851 osn_var->softirq.count++;
853 local_inc(&osn_var->int_counter);
857 * trace_softirq_exit_callback - Note the end of an softirq
859 * Computes the duration of the softirq noise, and trace it. Also discounts the
860 * interference from other sources of noise could be currently being accounted.
862 static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
864 struct osnoise_variables *osn_var = this_cpu_osn_var();
867 if (!osn_var->sampling)
870 #ifdef CONFIG_TIMERLAT_TRACER
872 * If the timerlat is enabled, but the irq handler did
873 * not run yet enabling timerlat_tracer, do not trace.
875 if (unlikely(osnoise_data.timerlat_tracer)) {
876 struct timerlat_variables *tlat_var;
877 tlat_var = this_cpu_tmr_var();
878 if (!tlat_var->tracing_thread) {
879 osn_var->softirq.arrival_time = 0;
880 osn_var->softirq.delta_start = 0;
886 duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
887 trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
888 cond_move_thread_delta_start(osn_var, duration);
889 osn_var->softirq.arrival_time = 0;
893 * hook_softirq_events - Hook softirq handling events
895 * This function hooks the softirq related callbacks to the respective trace
898 static int hook_softirq_events(void)
902 ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
906 ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
908 goto out_unreg_entry;
913 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
919 * unhook_softirq_events - Unhook softirq handling events
921 * This function hooks the softirq related callbacks to the respective trace
924 static void unhook_softirq_events(void)
926 unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
927 unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
929 #else /* CONFIG_PREEMPT_RT */
931 * softirq are threads on the PREEMPT_RT mode.
933 static int hook_softirq_events(void)
937 static void unhook_softirq_events(void)
943 * thread_entry - Record the starting of a thread noise window
945 * It saves the context switch time for a noisy thread, and increments
946 * the interference counters.
949 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
951 if (!osn_var->sampling)
954 * The arrival time will be used in the report, but not to compute
955 * the execution time, so it is safe to get it unsafe.
957 osn_var->thread.arrival_time = time_get();
959 set_int_safe_time(osn_var, &osn_var->thread.delta_start);
961 osn_var->thread.count++;
962 local_inc(&osn_var->int_counter);
966 * thread_exit - Report the end of a thread noise window
968 * It computes the total noise from a thread, tracing if needed.
971 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
975 if (!osn_var->sampling)
978 #ifdef CONFIG_TIMERLAT_TRACER
979 if (osnoise_data.timerlat_tracer) {
980 struct timerlat_variables *tlat_var;
981 tlat_var = this_cpu_tmr_var();
982 if (!tlat_var->tracing_thread) {
983 osn_var->thread.delta_start = 0;
984 osn_var->thread.arrival_time = 0;
990 duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
992 trace_thread_noise(t, osn_var->thread.arrival_time, duration);
994 osn_var->thread.arrival_time = 0;
998 * trace_sched_switch - sched:sched_switch trace event handler
1000 * This function is hooked to the sched:sched_switch trace event, and it is
1001 * used to record the beginning and to report the end of a thread noise window.
1004 trace_sched_switch_callback(void *data, bool preempt, struct task_struct *p,
1005 struct task_struct *n)
1007 struct osnoise_variables *osn_var = this_cpu_osn_var();
1009 if (p->pid != osn_var->pid)
1010 thread_exit(osn_var, p);
1012 if (n->pid != osn_var->pid)
1013 thread_entry(osn_var, n);
1017 * hook_thread_events - Hook the insturmentation for thread noise
1019 * Hook the osnoise tracer callbacks to handle the noise from other
1020 * threads on the necessary kernel events.
1022 static int hook_thread_events(void)
1026 ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1034 * unhook_thread_events - *nhook the insturmentation for thread noise
1036 * Unook the osnoise tracer callbacks to handle the noise from other
1037 * threads on the necessary kernel events.
1039 static void unhook_thread_events(void)
1041 unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1045 * save_osn_sample_stats - Save the osnoise_sample statistics
1047 * Save the osnoise_sample statistics before the sampling phase. These
1048 * values will be used later to compute the diff betwneen the statistics
1049 * before and after the osnoise sampling.
1052 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1054 s->nmi_count = osn_var->nmi.count;
1055 s->irq_count = osn_var->irq.count;
1056 s->softirq_count = osn_var->softirq.count;
1057 s->thread_count = osn_var->thread.count;
1061 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1063 * After a sample period, compute the difference on the osnoise_sample
1064 * statistics. The struct osnoise_sample *s contains the statistics saved via
1065 * save_osn_sample_stats() before the osnoise sampling.
1068 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1070 s->nmi_count = osn_var->nmi.count - s->nmi_count;
1071 s->irq_count = osn_var->irq.count - s->irq_count;
1072 s->softirq_count = osn_var->softirq.count - s->softirq_count;
1073 s->thread_count = osn_var->thread.count - s->thread_count;
1077 * osnoise_stop_tracing - Stop tracing and the tracer.
1079 static __always_inline void osnoise_stop_tracing(void)
1081 struct trace_array *tr = osnoise_trace;
1083 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1084 "stop tracing hit on cpu %d\n", smp_processor_id());
1086 tracer_tracing_off(tr);
1090 * run_osnoise - Sample the time and look for osnoise
1092 * Used to capture the time, looking for potential osnoise latency repeatedly.
1093 * Different from hwlat_detector, it is called with preemption and interrupts
1094 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1095 * osnoise sampling thread, as they would do with a regular thread.
1097 static int run_osnoise(void)
1099 struct osnoise_variables *osn_var = this_cpu_osn_var();
1100 struct trace_array *tr = osnoise_trace;
1101 u64 start, sample, last_sample;
1102 u64 last_int_count, int_count;
1103 s64 noise = 0, max_noise = 0;
1104 s64 total, last_total = 0;
1105 struct osnoise_sample s;
1106 unsigned int threshold;
1107 u64 runtime, stop_in;
1113 * Considers the current thread as the workload.
1115 osn_var->pid = current->pid;
1118 * Save the current stats for the diff
1120 save_osn_sample_stats(osn_var, &s);
1123 * if threshold is 0, use the default value of 5 us.
1125 threshold = tracing_thresh ? : 5000;
1128 * Make sure NMIs see sampling first
1130 osn_var->sampling = true;
1134 * Transform the *_us config to nanoseconds to avoid the
1135 * division on the main loop.
1137 runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1138 stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1148 last_int_count = set_int_safe_time(osn_var, &last_sample);
1154 int_count = set_int_safe_time(osn_var, &sample);
1156 noise = time_sub(sample, last_sample);
1159 * This shouldn't happen.
1162 osnoise_taint("negative noise!");
1169 total = time_sub(sample, start);
1172 * Check for possible overflows.
1174 if (total < last_total) {
1175 osnoise_taint("total overflow!");
1181 if (noise >= threshold) {
1182 int interference = int_count - last_int_count;
1184 if (noise > max_noise)
1192 trace_sample_threshold(last_sample, noise, interference);
1194 if (osnoise_data.stop_tracing)
1195 if (noise > stop_in)
1196 osnoise_stop_tracing();
1200 * For the non-preemptive kernel config: let threads runs, if
1205 last_sample = sample;
1206 last_int_count = int_count;
1208 } while (total < runtime && !kthread_should_stop());
1211 * Finish the above in the view for interrupts.
1215 osn_var->sampling = false;
1218 * Make sure sampling data is no longer updated.
1225 s.noise = time_to_us(sum_noise);
1226 s.runtime = time_to_us(total);
1227 s.max_sample = time_to_us(max_noise);
1228 s.hw_count = hw_count;
1230 /* Save interference stats info */
1231 diff_osn_sample_stats(osn_var, &s);
1233 trace_osnoise_sample(&s);
1235 /* Keep a running maximum ever recorded osnoise "latency" */
1236 if (max_noise > tr->max_latency) {
1237 tr->max_latency = max_noise;
1238 latency_fsnotify(tr);
1241 if (osnoise_data.stop_tracing_total)
1242 if (s.noise > osnoise_data.stop_tracing_total)
1243 osnoise_stop_tracing();
1250 static struct cpumask osnoise_cpumask;
1251 static struct cpumask save_cpumask;
1254 * osnoise_main - The osnoise detection kernel thread
1256 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1259 static int osnoise_main(void *data)
1263 while (!kthread_should_stop()) {
1267 mutex_lock(&interface_lock);
1268 interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1269 mutex_unlock(&interface_lock);
1271 do_div(interval, USEC_PER_MSEC);
1274 * differently from hwlat_detector, the osnoise tracer can run
1275 * without a pause because preemption is on.
1278 /* Let synchronize_rcu_tasks() make progress */
1279 cond_resched_tasks_rcu_qs();
1283 if (msleep_interruptible(interval))
1290 #ifdef CONFIG_TIMERLAT_TRACER
1292 * timerlat_irq - hrtimer handler for timerlat.
1294 static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1296 struct osnoise_variables *osn_var = this_cpu_osn_var();
1297 struct trace_array *tr = osnoise_trace;
1298 struct timerlat_variables *tlat;
1299 struct timerlat_sample s;
1304 * I am not sure if the timer was armed for this CPU. So, get
1305 * the timerlat struct from the timer itself, not from this
1308 tlat = container_of(timer, struct timerlat_variables, timer);
1310 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1313 * Enable the osnoise: events for thread an softirq.
1315 tlat->tracing_thread = true;
1317 osn_var->thread.arrival_time = time_get();
1320 * A hardirq is running: the timer IRQ. It is for sure preempting
1321 * a thread, and potentially preempting a softirq.
1323 * At this point, it is not interesting to know the duration of the
1324 * preempted thread (and maybe softirq), but how much time they will
1325 * delay the beginning of the execution of the timer thread.
1327 * To get the correct (net) delay added by the softirq, its delta_start
1328 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1329 * start of the sofitrq will be zeroed, accounting then only the time
1332 * The thread follows the same principle. However, if a softirq is
1333 * running, the thread needs to receive the softirq delta_start. The
1334 * reason being is that the softirq will be the last to be unfolded,
1335 * resseting the thread delay to zero.
1337 #ifndef CONFIG_PREEMPT_RT
1338 if (osn_var->softirq.delta_start) {
1339 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1340 &osn_var->softirq.delta_start);
1342 copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1343 &osn_var->irq.delta_start);
1345 copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1346 &osn_var->irq.delta_start);
1348 #else /* CONFIG_PREEMPT_RT */
1350 * The sofirqs run as threads on RT, so there is not need
1351 * to keep track of it.
1353 copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->irq.delta_start);
1354 #endif /* CONFIG_PREEMPT_RT */
1357 * Compute the current time with the expected time.
1359 diff = now - tlat->abs_period;
1362 s.seqnum = tlat->count;
1363 s.timer_latency = diff;
1364 s.context = IRQ_CONTEXT;
1366 trace_timerlat_sample(&s);
1368 /* Keep a running maximum ever recorded os noise "latency" */
1369 if (diff > tr->max_latency) {
1370 tr->max_latency = diff;
1371 latency_fsnotify(tr);
1374 if (osnoise_data.stop_tracing)
1375 if (time_to_us(diff) >= osnoise_data.stop_tracing)
1376 osnoise_stop_tracing();
1378 wake_up_process(tlat->kthread);
1380 if (osnoise_data.print_stack)
1381 timerlat_save_stack(0);
1383 return HRTIMER_NORESTART;
1387 * wait_next_period - Wait for the next period for timerlat
1389 static int wait_next_period(struct timerlat_variables *tlat)
1391 ktime_t next_abs_period, now;
1392 u64 rel_period = osnoise_data.timerlat_period * 1000;
1394 now = hrtimer_cb_get_time(&tlat->timer);
1395 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1398 * Save the next abs_period.
1400 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1403 * If the new abs_period is in the past, skip the activation.
1405 while (ktime_compare(now, next_abs_period) > 0) {
1406 next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1407 tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1410 set_current_state(TASK_INTERRUPTIBLE);
1412 hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1418 * timerlat_main- Timerlat main
1420 static int timerlat_main(void *data)
1422 struct osnoise_variables *osn_var = this_cpu_osn_var();
1423 struct timerlat_variables *tlat = this_cpu_tmr_var();
1424 struct timerlat_sample s;
1425 struct sched_param sp;
1429 * Make the thread RT, that is how cyclictest is usually used.
1431 sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1432 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1435 tlat->tracing_thread = false;
1437 hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1438 tlat->timer.function = timerlat_irq;
1439 tlat->kthread = current;
1440 osn_var->pid = current->pid;
1442 * Anotate the arrival time.
1444 tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1446 wait_next_period(tlat);
1448 osn_var->sampling = 1;
1450 while (!kthread_should_stop()) {
1451 now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1452 diff = now - tlat->abs_period;
1454 s.seqnum = tlat->count;
1455 s.timer_latency = diff;
1456 s.context = THREAD_CONTEXT;
1458 trace_timerlat_sample(&s);
1460 #ifdef CONFIG_STACKTRACE
1461 if (osnoise_data.print_stack)
1462 if (osnoise_data.print_stack <= time_to_us(diff))
1463 timerlat_dump_stack();
1464 #endif /* CONFIG_STACKTRACE */
1466 tlat->tracing_thread = false;
1467 if (osnoise_data.stop_tracing_total)
1468 if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1469 osnoise_stop_tracing();
1471 wait_next_period(tlat);
1474 hrtimer_cancel(&tlat->timer);
1477 #endif /* CONFIG_TIMERLAT_TRACER */
1480 * stop_kthread - stop a workload thread
1482 static void stop_kthread(unsigned int cpu)
1484 struct task_struct *kthread;
1486 kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
1488 kthread_stop(kthread);
1489 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1493 * stop_per_cpu_kthread - Stop per-cpu threads
1495 * Stop the osnoise sampling htread. Use this on unload and at system
1498 static void stop_per_cpu_kthreads(void)
1504 for_each_online_cpu(cpu)
1511 * start_kthread - Start a workload tread
1513 static int start_kthread(unsigned int cpu)
1515 struct task_struct *kthread;
1516 void *main = osnoise_main;
1519 #ifdef CONFIG_TIMERLAT_TRACER
1520 if (osnoise_data.timerlat_tracer) {
1521 snprintf(comm, 24, "timerlat/%d", cpu);
1522 main = timerlat_main;
1524 snprintf(comm, 24, "osnoise/%d", cpu);
1527 snprintf(comm, 24, "osnoise/%d", cpu);
1529 kthread = kthread_create_on_cpu(main, NULL, cpu, comm);
1531 if (IS_ERR(kthread)) {
1532 pr_err(BANNER "could not start sampling thread\n");
1533 stop_per_cpu_kthreads();
1537 per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
1538 wake_up_process(kthread);
1544 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
1546 * This starts the kernel thread that will look for osnoise on many
1549 static int start_per_cpu_kthreads(struct trace_array *tr)
1551 struct cpumask *current_mask = &save_cpumask;
1557 * Run only on CPUs in which trace and osnoise are allowed to run.
1559 cpumask_and(current_mask, tr->tracing_cpumask, &osnoise_cpumask);
1561 * And the CPU is online.
1563 cpumask_and(current_mask, cpu_online_mask, current_mask);
1565 for_each_possible_cpu(cpu)
1566 per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
1568 for_each_cpu(cpu, current_mask) {
1569 retval = start_kthread(cpu);
1571 stop_per_cpu_kthreads();
1581 #ifdef CONFIG_HOTPLUG_CPU
1582 static void osnoise_hotplug_workfn(struct work_struct *dummy)
1584 struct trace_array *tr = osnoise_trace;
1585 unsigned int cpu = smp_processor_id();
1588 mutex_lock(&trace_types_lock);
1591 goto out_unlock_trace;
1593 mutex_lock(&interface_lock);
1596 if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
1599 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask))
1606 mutex_unlock(&interface_lock);
1608 mutex_unlock(&trace_types_lock);
1611 static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
1614 * osnoise_cpu_init - CPU hotplug online callback function
1616 static int osnoise_cpu_init(unsigned int cpu)
1618 schedule_work_on(cpu, &osnoise_hotplug_work);
1623 * osnoise_cpu_die - CPU hotplug offline callback function
1625 static int osnoise_cpu_die(unsigned int cpu)
1631 static void osnoise_init_hotplug_support(void)
1635 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
1636 osnoise_cpu_init, osnoise_cpu_die);
1638 pr_warn(BANNER "Error to init cpu hotplug support\n");
1642 #else /* CONFIG_HOTPLUG_CPU */
1643 static void osnoise_init_hotplug_support(void)
1647 #endif /* CONFIG_HOTPLUG_CPU */
1650 * osnoise_cpus_read - Read function for reading the "cpus" file
1651 * @filp: The active open file structure
1652 * @ubuf: The userspace provided buffer to read value into
1653 * @cnt: The maximum number of bytes to read
1654 * @ppos: The current "file" position
1656 * Prints the "cpus" output into the user-provided buffer.
1659 osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
1665 mutex_lock(&interface_lock);
1667 len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
1668 mask_str = kmalloc(len, GFP_KERNEL);
1674 len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
1680 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
1685 mutex_unlock(&interface_lock);
1690 static void osnoise_tracer_start(struct trace_array *tr);
1691 static void osnoise_tracer_stop(struct trace_array *tr);
1694 * osnoise_cpus_write - Write function for "cpus" entry
1695 * @filp: The active open file structure
1696 * @ubuf: The user buffer that contains the value to write
1697 * @cnt: The maximum number of bytes to write to "file"
1698 * @ppos: The current position in @file
1700 * This function provides a write implementation for the "cpus"
1701 * interface to the osnoise trace. By default, it lists all CPUs,
1702 * in this way, allowing osnoise threads to run on any online CPU
1703 * of the system. It serves to restrict the execution of osnoise to the
1704 * set of CPUs writing via this interface. Note that osnoise also
1705 * respects the "tracing_cpumask." Hence, osnoise threads will run only
1706 * on the set of CPUs allowed here AND on "tracing_cpumask." Why not
1707 * have just "tracing_cpumask?" Because the user might be interested
1708 * in tracing what is running on other CPUs. For instance, one might
1709 * run osnoise in one HT CPU while observing what is running on the
1713 osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
1716 struct trace_array *tr = osnoise_trace;
1717 cpumask_var_t osnoise_cpumask_new;
1724 if (copy_from_user(buf, ubuf, count))
1727 if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
1730 err = cpulist_parse(buf, osnoise_cpumask_new);
1735 * trace_types_lock is taken to avoid concurrency on start/stop
1738 mutex_lock(&trace_types_lock);
1739 running = osnoise_busy;
1741 osnoise_tracer_stop(tr);
1743 mutex_lock(&interface_lock);
1745 * osnoise_cpumask is read by CPU hotplug operations.
1749 cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
1752 mutex_unlock(&interface_lock);
1755 osnoise_tracer_start(tr);
1756 mutex_unlock(&trace_types_lock);
1758 free_cpumask_var(osnoise_cpumask_new);
1762 free_cpumask_var(osnoise_cpumask_new);
1768 * osnoise/runtime_us: cannot be greater than the period.
1770 static struct trace_min_max_param osnoise_runtime = {
1771 .lock = &interface_lock,
1772 .val = &osnoise_data.sample_runtime,
1773 .max = &osnoise_data.sample_period,
1778 * osnoise/period_us: cannot be smaller than the runtime.
1780 static struct trace_min_max_param osnoise_period = {
1781 .lock = &interface_lock,
1782 .val = &osnoise_data.sample_period,
1784 .min = &osnoise_data.sample_runtime,
1788 * osnoise/stop_tracing_us: no limit.
1790 static struct trace_min_max_param osnoise_stop_tracing_in = {
1791 .lock = &interface_lock,
1792 .val = &osnoise_data.stop_tracing,
1798 * osnoise/stop_tracing_total_us: no limit.
1800 static struct trace_min_max_param osnoise_stop_tracing_total = {
1801 .lock = &interface_lock,
1802 .val = &osnoise_data.stop_tracing_total,
1807 #ifdef CONFIG_TIMERLAT_TRACER
1809 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
1810 * latency is higher than val.
1812 static struct trace_min_max_param osnoise_print_stack = {
1813 .lock = &interface_lock,
1814 .val = &osnoise_data.print_stack,
1820 * osnoise/timerlat_period: min 100 us, max 1 s
1822 u64 timerlat_min_period = 100;
1823 u64 timerlat_max_period = 1000000;
1824 static struct trace_min_max_param timerlat_period = {
1825 .lock = &interface_lock,
1826 .val = &osnoise_data.timerlat_period,
1827 .max = &timerlat_max_period,
1828 .min = &timerlat_min_period,
1832 static const struct file_operations cpus_fops = {
1833 .open = tracing_open_generic,
1834 .read = osnoise_cpus_read,
1835 .write = osnoise_cpus_write,
1836 .llseek = generic_file_llseek,
1840 * init_tracefs - A function to initialize the tracefs interface files
1842 * This function creates entries in tracefs for "osnoise" and "timerlat".
1843 * It creates these directories in the tracing directory, and within that
1844 * directory the use can change and view the configs.
1846 static int init_tracefs(void)
1848 struct dentry *top_dir;
1852 ret = tracing_init_dentry();
1856 top_dir = tracefs_create_dir("osnoise", NULL);
1860 tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
1861 &osnoise_period, &trace_min_max_fops);
1865 tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
1866 &osnoise_runtime, &trace_min_max_fops);
1870 tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
1871 &osnoise_stop_tracing_in, &trace_min_max_fops);
1875 tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
1876 &osnoise_stop_tracing_total, &trace_min_max_fops);
1880 tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
1883 #ifdef CONFIG_TIMERLAT_TRACER
1884 #ifdef CONFIG_STACKTRACE
1885 tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
1886 &osnoise_print_stack, &trace_min_max_fops);
1891 tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
1892 &timerlat_period, &trace_min_max_fops);
1900 tracefs_remove(top_dir);
1904 static int osnoise_hook_events(void)
1909 * Trace is already hooked, we are re-enabling from
1912 if (trace_osnoise_callback_enabled)
1915 retval = hook_irq_events();
1919 retval = hook_softirq_events();
1921 goto out_unhook_irq;
1923 retval = hook_thread_events();
1930 unhook_softirq_events();
1932 unhook_irq_events();
1936 static int __osnoise_tracer_start(struct trace_array *tr)
1940 osn_var_reset_all();
1942 retval = osnoise_hook_events();
1946 * Make sure NMIs see reseted values.
1949 trace_osnoise_callback_enabled = true;
1951 retval = start_per_cpu_kthreads(tr);
1953 unhook_irq_events();
1957 osnoise_busy = true;
1962 static void osnoise_tracer_start(struct trace_array *tr)
1969 retval = __osnoise_tracer_start(tr);
1971 pr_err(BANNER "Error starting osnoise tracer\n");
1975 static void osnoise_tracer_stop(struct trace_array *tr)
1980 trace_osnoise_callback_enabled = false;
1983 stop_per_cpu_kthreads();
1985 unhook_irq_events();
1986 unhook_softirq_events();
1987 unhook_thread_events();
1989 osnoise_busy = false;
1992 static int osnoise_tracer_init(struct trace_array *tr)
1995 /* Only allow one instance to enable this */
2000 tr->max_latency = 0;
2002 osnoise_tracer_start(tr);
2007 static void osnoise_tracer_reset(struct trace_array *tr)
2009 osnoise_tracer_stop(tr);
2012 static struct tracer osnoise_tracer __read_mostly = {
2014 .init = osnoise_tracer_init,
2015 .reset = osnoise_tracer_reset,
2016 .start = osnoise_tracer_start,
2017 .stop = osnoise_tracer_stop,
2018 .print_header = print_osnoise_headers,
2019 .allow_instances = true,
2022 #ifdef CONFIG_TIMERLAT_TRACER
2023 static void timerlat_tracer_start(struct trace_array *tr)
2030 osnoise_data.timerlat_tracer = 1;
2032 retval = __osnoise_tracer_start(tr);
2038 pr_err(BANNER "Error starting timerlat tracer\n");
2041 static void timerlat_tracer_stop(struct trace_array *tr)
2048 for_each_online_cpu(cpu)
2049 per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
2051 osnoise_tracer_stop(tr);
2053 osnoise_data.timerlat_tracer = 0;
2056 static int timerlat_tracer_init(struct trace_array *tr)
2058 /* Only allow one instance to enable this */
2064 tr->max_latency = 0;
2066 timerlat_tracer_start(tr);
2071 static void timerlat_tracer_reset(struct trace_array *tr)
2073 timerlat_tracer_stop(tr);
2076 static struct tracer timerlat_tracer __read_mostly = {
2078 .init = timerlat_tracer_init,
2079 .reset = timerlat_tracer_reset,
2080 .start = timerlat_tracer_start,
2081 .stop = timerlat_tracer_stop,
2082 .print_header = print_timerlat_headers,
2083 .allow_instances = true,
2085 #endif /* CONFIG_TIMERLAT_TRACER */
2087 __init static int init_osnoise_tracer(void)
2091 mutex_init(&interface_lock);
2093 cpumask_copy(&osnoise_cpumask, cpu_all_mask);
2095 ret = register_tracer(&osnoise_tracer);
2097 pr_err(BANNER "Error registering osnoise!\n");
2101 #ifdef CONFIG_TIMERLAT_TRACER
2102 ret = register_tracer(&timerlat_tracer);
2104 pr_err(BANNER "Error registering timerlat\n");
2108 osnoise_init_hotplug_support();
2114 late_initcall(init_osnoise_tracer);