sched: Add rq::ttwu_pending

[linux-2.6-microblaze.git] / kernel / sched / core.c

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 6130ab1..b71ed5e 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -637,41 +637,25 @@ void wake_up_nohz_cpu(int cpu)
                wake_up_idle_cpu(cpu);
 }
 
-static inline bool got_nohz_idle_kick(void)
+static void nohz_csd_func(void *info)
 {
-       int cpu = smp_processor_id();
-
-       if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))
-               return false;
-
-       if (idle_cpu(cpu) && !need_resched())
-               return true;
+       struct rq *rq = info;
+       int cpu = cpu_of(rq);
+       unsigned int flags;
 
        /*
-        * We can't run Idle Load Balance on this CPU for this time so we
-        * cancel it and clear NOHZ_BALANCE_KICK
+        * Release the rq::nohz_csd.
         */
-       atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
-       return false;
-}
+       flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
+       WARN_ON(!(flags & NOHZ_KICK_MASK));
 
-static void nohz_csd_func(void *info)
-{
-       struct rq *rq = info;
-
-       if (got_nohz_idle_kick()) {
-               rq->idle_balance = 1;
+       rq->idle_balance = idle_cpu(cpu);
+       if (rq->idle_balance && !need_resched()) {
+               rq->nohz_idle_balance = flags;
                raise_softirq_irqoff(SCHED_SOFTIRQ);
        }
 }
 
-#else /* CONFIG_NO_HZ_COMMON */
-
-static inline bool got_nohz_idle_kick(void)
-{
-       return false;
-}
-
 #endif /* CONFIG_NO_HZ_COMMON */
 
 #ifdef CONFIG_NO_HZ_FULL
@@ -2291,13 +2275,21 @@ static int ttwu_remote(struct task_struct *p, int wake_flags)
 void sched_ttwu_pending(void)
 {
        struct rq *rq = this_rq();
-       struct llist_node *llist = llist_del_all(&rq->wake_list);
+       struct llist_node *llist;
        struct task_struct *p, *t;
        struct rq_flags rf;
 
+       llist = llist_del_all(&rq->wake_list);
        if (!llist)
                return;
 
+       /*
+        * rq::ttwu_pending racy indication of out-standing wakeups.
+        * Races such that false-negatives are possible, since they
+        * are shorter lived that false-positives would be.
+        */
+       WRITE_ONCE(rq->ttwu_pending, 0);
+
        rq_lock_irqsave(rq, &rf);
        update_rq_clock(rq);
 
@@ -2312,6 +2304,16 @@ static void wake_csd_func(void *info)
        sched_ttwu_pending();
 }
 
+void send_call_function_single_ipi(int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+
+       if (!set_nr_if_polling(rq->idle))
+               arch_send_call_function_single_ipi(cpu);
+       else
+               trace_sched_wake_idle_without_ipi(cpu);
+}
+
 /*
  * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
  * necessary. The wakee CPU on receipt of the IPI will queue the task
@@ -2324,6 +2326,7 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags
 
        p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
 
+       WRITE_ONCE(rq->ttwu_pending, 1);
        if (llist_add(&p->wake_entry, &rq->wake_list)) {
                if (!set_nr_if_polling(rq->idle))
                        smp_call_function_single_async(cpu, &rq->wake_csd);
@@ -2578,6 +2581,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
         *
         * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
         * __schedule().  See the comment for smp_mb__after_spinlock().
+        *
+        * A similar smb_rmb() lives in try_invoke_on_locked_down_task().
         */
        smp_rmb();
        if (p->on_rq && ttwu_remote(p, wake_flags))
@@ -2653,6 +2658,52 @@ out:
        return success;
 }
 
+/**
+ * try_invoke_on_locked_down_task - Invoke a function on task in fixed state
+ * @p: Process for which the function is to be invoked.
+ * @func: Function to invoke.
+ * @arg: Argument to function.
+ *
+ * If the specified task can be quickly locked into a definite state
+ * (either sleeping or on a given runqueue), arrange to keep it in that
+ * state while invoking @func(@arg).  This function can use ->on_rq and
+ * task_curr() to work out what the state is, if required.  Given that
+ * @func can be invoked with a runqueue lock held, it had better be quite
+ * lightweight.
+ *
+ * Returns:
+ *     @false if the task slipped out from under the locks.
+ *     @true if the task was locked onto a runqueue or is sleeping.
+ *             However, @func can override this by returning @false.
+ */
+bool try_invoke_on_locked_down_task(struct task_struct *p, bool (*func)(struct task_struct *t, void *arg), void *arg)
+{
+       bool ret = false;
+       struct rq_flags rf;
+       struct rq *rq;
+
+       lockdep_assert_irqs_enabled();
+       raw_spin_lock_irq(&p->pi_lock);
+       if (p->on_rq) {
+               rq = __task_rq_lock(p, &rf);
+               if (task_rq(p) == rq)
+                       ret = func(p, arg);
+               rq_unlock(rq, &rf);
+       } else {
+               switch (p->state) {
+               case TASK_RUNNING:
+               case TASK_WAKING:
+                       break;
+               default:
+                       smp_rmb(); // See smp_rmb() comment in try_to_wake_up().
+                       if (!p->on_rq)
+                               ret = func(p, arg);
+               }
+       }
+       raw_spin_unlock_irq(&p->pi_lock);
+       return ret;
+}
+
 /**
  * wake_up_process - Wake up a specific process
  * @p: The process to be woken up.
@@ -4663,7 +4714,7 @@ int idle_cpu(int cpu)
                return 0;
 
 #ifdef CONFIG_SMP
-       if (!llist_empty(&rq->wake_list))
+       if (rq->ttwu_pending)
                return 0;
 #endif