shm: make exit_shm work proportional to task activity

[linux-2.6-microblaze.git] / kernel / irq_work.c

diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index a82170e..e6bcbe7 100644 (file)
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -16,11 +16,12 @@
 #include <linux/tick.h>
 #include <linux/cpu.h>
 #include <linux/notifier.h>
+#include <linux/smp.h>
 #include <asm/processor.h>
 
 
-static DEFINE_PER_CPU(struct llist_head, irq_work_list);
-static DEFINE_PER_CPU(int, irq_work_raised);
+static DEFINE_PER_CPU(struct llist_head, raised_list);
+static DEFINE_PER_CPU(struct llist_head, lazy_list);
 
 /*
  * Claim the entry so that no one else will poke at it.
@@ -55,12 +56,34 @@ void __weak arch_irq_work_raise(void)
         */
 }
 
+#ifdef CONFIG_SMP
 /*
- * Enqueue the irq_work @entry unless it's already pending
+ * Enqueue the irq_work @work on @cpu unless it's already pending
  * somewhere.
  *
  * Can be re-enqueued while the callback is still in progress.
  */
+bool irq_work_queue_on(struct irq_work *work, int cpu)
+{
+       /* All work should have been flushed before going offline */
+       WARN_ON_ONCE(cpu_is_offline(cpu));
+
+       /* Arch remote IPI send/receive backend aren't NMI safe */
+       WARN_ON_ONCE(in_nmi());
+
+       /* Only queue if not already pending */
+       if (!irq_work_claim(work))
+               return false;
+
+       if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
+               arch_send_call_function_single_ipi(cpu);
+
+       return true;
+}
+EXPORT_SYMBOL_GPL(irq_work_queue_on);
+#endif
+
+/* Enqueue the irq work @work on the current CPU */
 bool irq_work_queue(struct irq_work *work)
 {
        /* Only queue if not already pending */
@@ -70,15 +93,13 @@ bool irq_work_queue(struct irq_work *work)
        /* Queue the entry and raise the IPI if needed. */
        preempt_disable();
 
-       llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
-
-       /*
-        * If the work is not "lazy" or the tick is stopped, raise the irq
-        * work interrupt (if supported by the arch), otherwise, just wait
-        * for the next tick.
-        */
-       if (!(work->flags & IRQ_WORK_LAZY) || tick_nohz_tick_stopped()) {
-               if (!this_cpu_cmpxchg(irq_work_raised, 0, 1))
+       /* If the work is "lazy", handle it from next tick if any */
+       if (work->flags & IRQ_WORK_LAZY) {
+               if (llist_add(&work->llnode, &__get_cpu_var(lazy_list)) &&
+                   tick_nohz_tick_stopped())
+                       arch_irq_work_raise();
+       } else {
+               if (llist_add(&work->llnode, &__get_cpu_var(raised_list)))
                        arch_irq_work_raise();
        }
 
@@ -90,10 +111,11 @@ EXPORT_SYMBOL_GPL(irq_work_queue);
 
 bool irq_work_needs_cpu(void)
 {
-       struct llist_head *this_list;
+       struct llist_head *raised, *lazy;
 
-       this_list = &__get_cpu_var(irq_work_list);
-       if (llist_empty(this_list))
+       raised = &__get_cpu_var(raised_list);
+       lazy = &__get_cpu_var(lazy_list);
+       if (llist_empty(raised) && llist_empty(lazy))
                return false;
 
        /* All work should have been flushed before going offline */
@@ -102,28 +124,18 @@ bool irq_work_needs_cpu(void)
        return true;
 }
 
-static void __irq_work_run(void)
+static void irq_work_run_list(struct llist_head *list)
 {
        unsigned long flags;
        struct irq_work *work;
-       struct llist_head *this_list;
        struct llist_node *llnode;
 
+       BUG_ON(!irqs_disabled());
 
-       /*
-        * Reset the "raised" state right before we check the list because
-        * an NMI may enqueue after we find the list empty from the runner.
-        */
-       __this_cpu_write(irq_work_raised, 0);
-       barrier();
-
-       this_list = &__get_cpu_var(irq_work_list);
-       if (llist_empty(this_list))
+       if (llist_empty(list))
                return;
 
-       BUG_ON(!irqs_disabled());
-
-       llnode = llist_del_all(this_list);
+       llnode = llist_del_all(list);
        while (llnode != NULL) {
                work = llist_entry(llnode, struct irq_work, llnode);
 
@@ -149,13 +161,13 @@ static void __irq_work_run(void)
 }
 
 /*
- * Run the irq_work entries on this cpu. Requires to be ran from hardirq
- * context with local IRQs disabled.
+ * hotplug calls this through:
+ *  hotplug_cfd() -> flush_smp_call_function_queue()
  */
 void irq_work_run(void)
 {
-       BUG_ON(!in_irq());
-       __irq_work_run();
+       irq_work_run_list(&__get_cpu_var(raised_list));
+       irq_work_run_list(&__get_cpu_var(lazy_list));
 }
 EXPORT_SYMBOL_GPL(irq_work_run);
 
@@ -171,35 +183,3 @@ void irq_work_sync(struct irq_work *work)
                cpu_relax();
 }
 EXPORT_SYMBOL_GPL(irq_work_sync);
-
-#ifdef CONFIG_HOTPLUG_CPU
-static int irq_work_cpu_notify(struct notifier_block *self,
-                              unsigned long action, void *hcpu)
-{
-       long cpu = (long)hcpu;
-
-       switch (action) {
-       case CPU_DYING:
-               /* Called from stop_machine */
-               if (WARN_ON_ONCE(cpu != smp_processor_id()))
-                       break;
-               __irq_work_run();
-               break;
-       default:
-               break;
-       }
-       return NOTIFY_OK;
-}
-
-static struct notifier_block cpu_notify;
-
-static __init int irq_work_init_cpu_notifier(void)
-{
-       cpu_notify.notifier_call = irq_work_cpu_notify;
-       cpu_notify.priority = 0;
-       register_cpu_notifier(&cpu_notify);
-       return 0;
-}
-device_initcall(irq_work_init_cpu_notifier);
-
-#endif /* CONFIG_HOTPLUG_CPU */