include/linux/hardirq.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef LINUX_HARDIRQ_H
   3 #define LINUX_HARDIRQ_H
   4
   5 #include <linux/context_tracking_state.h>
   6 #include <linux/preempt.h>
   7 #include <linux/lockdep.h>
   8 #include <linux/ftrace_irq.h>
   9 #include <linux/vtime.h>
  10 #include <asm/hardirq.h>
  11
  12 extern void synchronize_irq(unsigned int irq);
  13 extern bool synchronize_hardirq(unsigned int irq);
  14
  15 #ifdef CONFIG_NO_HZ_FULL
  16 void __rcu_irq_enter_check_tick(void);
  17 #else
  18 static inline void __rcu_irq_enter_check_tick(void) { }
  19 #endif
  20
  21 static __always_inline void rcu_irq_enter_check_tick(void)
  22 {
  23         if (context_tracking_enabled())
  24                 __rcu_irq_enter_check_tick();
  25 }
  26
  27 /*
  28  * It is safe to do non-atomic ops on ->hardirq_context,
  29  * because NMI handlers may not preempt and the ops are
  30  * always balanced, so the interrupted value of ->hardirq_context
  31  * will always be restored.
  32  */
  33 #define __irq_enter()                                   \
  34         do {                                            \
  35                 account_irq_enter_time(current);        \
  36                 preempt_count_add(HARDIRQ_OFFSET);      \
  37                 lockdep_hardirq_enter();                \
  38         } while (0)
  39
  40 /*
  41  * Enter irq context (on NO_HZ, update jiffies):
  42  */
  43 extern void irq_enter(void);
  44
  45 /*
  46  * Exit irq context without processing softirqs:
  47  */
  48 #define __irq_exit()                                    \
  49         do {                                            \
  50                 lockdep_hardirq_exit();                 \
  51                 account_irq_exit_time(current);         \
  52                 preempt_count_sub(HARDIRQ_OFFSET);      \
  53         } while (0)
  54
  55 /*
  56  * Exit irq context and process softirqs if needed:
  57  */
  58 extern void irq_exit(void);
  59
  60 #ifndef arch_nmi_enter
  61 #define arch_nmi_enter()        do { } while (0)
  62 #define arch_nmi_exit()         do { } while (0)
  63 #endif
  64
  65 #ifdef CONFIG_TINY_RCU
  66 static inline void rcu_nmi_enter(void) { }
  67 static inline void rcu_nmi_exit(void) { }
  68 #else
  69 extern void rcu_nmi_enter(void);
  70 extern void rcu_nmi_exit(void);
  71 #endif
  72
  73 /*
  74  * NMI vs Tracing
  75  * --------------
  76  *
  77  * We must not land in a tracer until (or after) we've changed preempt_count
  78  * such that in_nmi() becomes true. To that effect all NMI C entry points must
  79  * be marked 'notrace' and call nmi_enter() as soon as possible.
  80  */
  81
  82 /*
  83  * nmi_enter() can nest up to 15 times; see NMI_BITS.
  84  */
  85 #define nmi_enter()                                             \
  86         do {                                                    \
  87                 arch_nmi_enter();                               \
  88                 printk_nmi_enter();                             \
  89                 lockdep_off();                                  \
  90                 ftrace_nmi_enter();                             \
  91                 BUG_ON(in_nmi() == NMI_MASK);                   \
  92                 __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);       \
  93                 rcu_nmi_enter();                                \
  94                 lockdep_hardirq_enter();                        \
  95         } while (0)
  96
  97 #define nmi_exit()                                              \
  98         do {                                                    \
  99                 lockdep_hardirq_exit();                         \
 100                 rcu_nmi_exit();                                 \
 101                 BUG_ON(!in_nmi());                              \
 102                 __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);       \
 103                 ftrace_nmi_exit();                              \
 104                 lockdep_on();                                   \
 105                 printk_nmi_exit();                              \
 106                 arch_nmi_exit();                                \
 107         } while (0)
 108
 109 #endif /* LINUX_HARDIRQ_H */