blk-mq, elevator: Count requests per hctx to improve performance

[linux-2.6-microblaze.git] / include / linux / irqflags.h

diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h
index bd5c557..3ed4e87 100644 (file)
--- a/include/linux/irqflags.h
+++ b/include/linux/irqflags.h
@@ -49,17 +49,18 @@ struct irqtrace_events {
 DECLARE_PER_CPU(int, hardirqs_enabled);
 DECLARE_PER_CPU(int, hardirq_context);
 
-  extern void trace_hardirqs_on_prepare(void);
-  extern void trace_hardirqs_off_finish(void);
-  extern void trace_hardirqs_on(void);
-  extern void trace_hardirqs_off(void);
-# define lockdep_hardirq_context()     (this_cpu_read(hardirq_context))
+extern void trace_hardirqs_on_prepare(void);
+extern void trace_hardirqs_off_finish(void);
+extern void trace_hardirqs_on(void);
+extern void trace_hardirqs_off(void);
+
+# define lockdep_hardirq_context()     (raw_cpu_read(hardirq_context))
 # define lockdep_softirq_context(p)    ((p)->softirq_context)
 # define lockdep_hardirqs_enabled()    (this_cpu_read(hardirqs_enabled))
 # define lockdep_softirqs_enabled(p)   ((p)->softirqs_enabled)
 # define lockdep_hardirq_enter()                       \
 do {                                                   \
-       if (this_cpu_inc_return(hardirq_context) == 1)  \
+       if (__this_cpu_inc_return(hardirq_context) == 1)\
                current->hardirq_threaded = 0;          \
 } while (0)
 # define lockdep_hardirq_threaded()            \
@@ -68,7 +69,7 @@ do {                                          \
 } while (0)
 # define lockdep_hardirq_exit()                        \
 do {                                           \
-       this_cpu_dec(hardirq_context);          \
+       __this_cpu_dec(hardirq_context);        \
 } while (0)
 # define lockdep_softirq_enter()               \
 do {                                           \
@@ -120,17 +121,17 @@ do {                                              \
 #else
 # define trace_hardirqs_on_prepare()           do { } while (0)
 # define trace_hardirqs_off_finish()           do { } while (0)
-# define trace_hardirqs_on()           do { } while (0)
-# define trace_hardirqs_off()          do { } while (0)
-# define lockdep_hardirq_context()     0
-# define lockdep_softirq_context(p)    0
-# define lockdep_hardirqs_enabled()    0
-# define lockdep_softirqs_enabled(p)   0
-# define lockdep_hardirq_enter()       do { } while (0)
-# define lockdep_hardirq_threaded()    do { } while (0)
-# define lockdep_hardirq_exit()                do { } while (0)
-# define lockdep_softirq_enter()       do { } while (0)
-# define lockdep_softirq_exit()                do { } while (0)
+# define trace_hardirqs_on()                   do { } while (0)
+# define trace_hardirqs_off()                  do { } while (0)
+# define lockdep_hardirq_context()             0
+# define lockdep_softirq_context(p)            0
+# define lockdep_hardirqs_enabled()            0
+# define lockdep_softirqs_enabled(p)           0
+# define lockdep_hardirq_enter()               do { } while (0)
+# define lockdep_hardirq_threaded()            do { } while (0)
+# define lockdep_hardirq_exit()                        do { } while (0)
+# define lockdep_softirq_enter()               do { } while (0)
+# define lockdep_softirq_exit()                        do { } while (0)
 # define lockdep_hrtimer_enter(__hrtimer)      false
 # define lockdep_hrtimer_exit(__context)       do { } while (0)
 # define lockdep_posixtimer_enter()            do { } while (0)
@@ -181,26 +182,33 @@ do {                                              \
  * if !TRACE_IRQFLAGS.
  */
 #ifdef CONFIG_TRACE_IRQFLAGS
-#define local_irq_enable() \
-       do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
-#define local_irq_disable() \
-       do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0)
+
+#define local_irq_enable()                             \
+       do {                                            \
+               trace_hardirqs_on();                    \
+               raw_local_irq_enable();                 \
+       } while (0)
+
+#define local_irq_disable()                            \
+       do {                                            \
+               bool was_disabled = raw_irqs_disabled();\
+               raw_local_irq_disable();                \
+               if (!was_disabled)                      \
+                       trace_hardirqs_off();           \
+       } while (0)
+
 #define local_irq_save(flags)                          \
        do {                                            \
                raw_local_irq_save(flags);              \
-               trace_hardirqs_off();                   \
+               if (!raw_irqs_disabled_flags(flags))    \
+                       trace_hardirqs_off();           \
        } while (0)
 
-
 #define local_irq_restore(flags)                       \
        do {                                            \
-               if (raw_irqs_disabled_flags(flags)) {   \
-                       raw_local_irq_restore(flags);   \
-                       trace_hardirqs_off();           \
-               } else {                                \
+               if (!raw_irqs_disabled_flags(flags))    \
                        trace_hardirqs_on();            \
-                       raw_local_irq_restore(flags);   \
-               }                                       \
+               raw_local_irq_restore(flags);           \
        } while (0)
 
 #define safe_halt()                            \
@@ -214,10 +222,7 @@ do {                                               \
 
 #define local_irq_enable()     do { raw_local_irq_enable(); } while (0)
 #define local_irq_disable()    do { raw_local_irq_disable(); } while (0)
-#define local_irq_save(flags)                                  \
-       do {                                                    \
-               raw_local_irq_save(flags);                      \
-       } while (0)
+#define local_irq_save(flags)  do { raw_local_irq_save(flags); } while (0)
 #define local_irq_restore(flags) do { raw_local_irq_restore(flags); } while (0)
 #define safe_halt()            do { raw_safe_halt(); } while (0)