sched/core, workqueues: Distangle worker accounting from rq lock

[linux-2.6-microblaze.git] / kernel / sched / core.c

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 4778c48..6184a08 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1685,10 +1685,6 @@ static inline void ttwu_activate(struct rq *rq, struct task_struct *p, int en_fl
 {
        activate_task(rq, p, en_flags);
        p->on_rq = TASK_ON_RQ_QUEUED;
-
-       /* If a worker is waking up, notify the workqueue: */
-       if (p->flags & PF_WQ_WORKER)
-               wq_worker_waking_up(p, cpu_of(rq));
 }
 
 /*
@@ -2106,56 +2102,6 @@ out:
        return success;
 }
 
-/**
- * try_to_wake_up_local - try to wake up a local task with rq lock held
- * @p: the thread to be awakened
- * @rf: request-queue flags for pinning
- *
- * Put @p on the run-queue if it's not already there. The caller must
- * ensure that this_rq() is locked, @p is bound to this_rq() and not
- * the current task.
- */
-static void try_to_wake_up_local(struct task_struct *p, struct rq_flags *rf)
-{
-       struct rq *rq = task_rq(p);
-
-       if (WARN_ON_ONCE(rq != this_rq()) ||
-           WARN_ON_ONCE(p == current))
-               return;
-
-       lockdep_assert_held(&rq->lock);
-
-       if (!raw_spin_trylock(&p->pi_lock)) {
-               /*
-                * This is OK, because current is on_cpu, which avoids it being
-                * picked for load-balance and preemption/IRQs are still
-                * disabled avoiding further scheduler activity on it and we've
-                * not yet picked a replacement task.
-                */
-               rq_unlock(rq, rf);
-               raw_spin_lock(&p->pi_lock);
-               rq_relock(rq, rf);
-       }
-
-       if (!(p->state & TASK_NORMAL))
-               goto out;
-
-       trace_sched_waking(p);
-
-       if (!task_on_rq_queued(p)) {
-               if (p->in_iowait) {
-                       delayacct_blkio_end(p);
-                       atomic_dec(&rq->nr_iowait);
-               }
-               ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK);
-       }
-
-       ttwu_do_wakeup(rq, p, 0, rf);
-       ttwu_stat(p, smp_processor_id(), 0);
-out:
-       raw_spin_unlock(&p->pi_lock);
-}
-
 /**
  * wake_up_process - Wake up a specific process
  * @p: The process to be woken up.
@@ -3472,19 +3418,6 @@ static void __sched notrace __schedule(bool preempt)
                                atomic_inc(&rq->nr_iowait);
                                delayacct_blkio_start();
                        }
-
-                       /*
-                        * If a worker went to sleep, notify and ask workqueue
-                        * whether it wants to wake up a task to maintain
-                        * concurrency.
-                        */
-                       if (prev->flags & PF_WQ_WORKER) {
-                               struct task_struct *to_wakeup;
-
-                               to_wakeup = wq_worker_sleeping(prev);
-                               if (to_wakeup)
-                                       try_to_wake_up_local(to_wakeup, &rf);
-                       }
                }
                switch_count = &prev->nvcsw;
        }
@@ -3544,6 +3477,20 @@ static inline void sched_submit_work(struct task_struct *tsk)
 {
        if (!tsk->state || tsk_is_pi_blocked(tsk))
                return;
+
+       /*
+        * If a worker went to sleep, notify and ask workqueue whether
+        * it wants to wake up a task to maintain concurrency.
+        * As this function is called inside the schedule() context,
+        * we disable preemption to avoid it calling schedule() again
+        * in the possible wakeup of a kworker.
+        */
+       if (tsk->flags & PF_WQ_WORKER) {
+               preempt_disable();
+               wq_worker_sleeping(tsk);
+               preempt_enable_no_resched();
+       }
+
        /*
         * If we are going to sleep and we have plugged IO queued,
         * make sure to submit it to avoid deadlocks.
@@ -3552,6 +3499,12 @@ static inline void sched_submit_work(struct task_struct *tsk)
                blk_schedule_flush_plug(tsk);
 }
 
+static void sched_update_worker(struct task_struct *tsk)
+{
+       if (tsk->flags & PF_WQ_WORKER)
+               wq_worker_running(tsk);
+}
+
 asmlinkage __visible void __sched schedule(void)
 {
        struct task_struct *tsk = current;
@@ -3562,6 +3515,7 @@ asmlinkage __visible void __sched schedule(void)
                __schedule(false);
                sched_preempt_enable_no_resched();
        } while (need_resched());
+       sched_update_worker(tsk);
 }
 EXPORT_SYMBOL(schedule);