#include <linux/rculist_nulls.h>
#include <linux/cpu.h>
#include <linux/tracehook.h>
+#include <linux/freezer.h>
#include "../kernel/sched/sched.h"
#include "io-wq.h"
struct io_wq_work *cur_work;
spinlock_t lock;
- const struct cred *cur_creds;
- const struct cred *saved_creds;
-
struct completion ref_done;
- struct completion started;
struct rcu_head rcu;
};
struct io_wq_hash *hash;
refcount_t refs;
- struct completion started;
struct completion exited;
atomic_t worker_refs;
static enum cpuhp_state io_wq_online;
+struct io_cb_cancel_data {
+ work_cancel_fn *fn;
+ void *data;
+ int nr_running;
+ int nr_pending;
+ bool cancel_all;
+};
+
+static void io_wqe_cancel_pending_work(struct io_wqe *wqe,
+ struct io_cb_cancel_data *match);
+
static bool io_worker_get(struct io_worker *worker)
{
return refcount_inc_not_zero(&worker->ref);
worker->flags = 0;
preempt_enable();
- if (worker->saved_creds) {
- revert_creds(worker->saved_creds);
- worker->cur_creds = worker->saved_creds = NULL;
- }
-
raw_spin_lock_irq(&wqe->lock);
if (flags & IO_WORKER_F_FREE)
hlist_nulls_del_rcu(&worker->nulls_node);
kfree_rcu(worker, rcu);
if (atomic_dec_and_test(&wqe->wq->worker_refs))
complete(&wqe->wq->worker_done);
+ do_exit(0);
}
static inline bool io_wqe_run_queue(struct io_wqe *wqe)
io_wqe_wake_worker(wqe, acct);
}
-static void io_worker_start(struct io_worker *worker)
-{
- worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING);
- io_wqe_inc_running(worker);
- complete(&worker->started);
-}
-
/*
* Worker will start processing some work. Move it to the busy list, if
* it's currently on the freelist
worker->flags |= IO_WORKER_F_FREE;
hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
}
- if (worker->saved_creds) {
- revert_creds(worker->saved_creds);
- worker->cur_creds = worker->saved_creds = NULL;
- }
}
static inline unsigned int io_get_work_hash(struct io_wq_work *work)
}
}
-static void io_wq_switch_creds(struct io_worker *worker,
- struct io_wq_work *work)
-{
- const struct cred *old_creds = override_creds(work->creds);
-
- worker->cur_creds = work->creds;
- if (worker->saved_creds)
- put_cred(old_creds); /* creds set by previous switch */
- else
- worker->saved_creds = old_creds;
-}
-
static void io_assign_current_work(struct io_worker *worker,
struct io_wq_work *work)
{
unsigned int hash = io_get_work_hash(work);
next_hashed = wq_next_work(work);
- if (work->creds && worker->cur_creds != work->creds)
- io_wq_switch_creds(worker, work);
wq->do_work(work);
io_assign_current_work(worker, NULL);
struct io_worker *worker = data;
struct io_wqe *wqe = worker->wqe;
struct io_wq *wq = wqe->wq;
+ char buf[TASK_COMM_LEN];
- io_worker_start(worker);
+ worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING);
+ io_wqe_inc_running(worker);
+
+ sprintf(buf, "iou-wrk-%d", wq->task_pid);
+ set_task_comm(current, buf);
while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
set_current_state(TASK_INTERRUPTIBLE);
raw_spin_unlock_irq(&worker->wqe->lock);
}
-static int task_thread(void *data, int index)
-{
- struct io_worker *worker = data;
- struct io_wqe *wqe = worker->wqe;
- struct io_wqe_acct *acct = &wqe->acct[index];
- struct io_wq *wq = wqe->wq;
- char buf[TASK_COMM_LEN];
-
- sprintf(buf, "iou-wrk-%d", wq->task_pid);
- set_task_comm(current, buf);
-
- current->pf_io_worker = worker;
- worker->task = current;
-
- set_cpus_allowed_ptr(current, cpumask_of_node(wqe->node));
- current->flags |= PF_NO_SETAFFINITY;
-
- raw_spin_lock_irq(&wqe->lock);
- hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
- list_add_tail_rcu(&worker->all_list, &wqe->all_list);
- worker->flags |= IO_WORKER_F_FREE;
- if (index == IO_WQ_ACCT_BOUND)
- worker->flags |= IO_WORKER_F_BOUND;
- if (!acct->nr_workers && (worker->flags & IO_WORKER_F_BOUND))
- worker->flags |= IO_WORKER_F_FIXED;
- acct->nr_workers++;
- raw_spin_unlock_irq(&wqe->lock);
-
- io_wqe_worker(data);
- do_exit(0);
-}
-
-static int task_thread_bound(void *data)
-{
- return task_thread(data, IO_WQ_ACCT_BOUND);
-}
-
-static int task_thread_unbound(void *data)
-{
- return task_thread(data, IO_WQ_ACCT_UNBOUND);
-}
-
-pid_t io_wq_fork_thread(int (*fn)(void *), void *arg)
-{
- unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|
- CLONE_IO|SIGCHLD;
- struct kernel_clone_args args = {
- .flags = ((lower_32_bits(flags) | CLONE_VM |
- CLONE_UNTRACED) & ~CSIGNAL),
- .exit_signal = (lower_32_bits(flags) & CSIGNAL),
- .stack = (unsigned long)fn,
- .stack_size = (unsigned long)arg,
- };
-
- return kernel_clone(&args);
-}
-
static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index)
{
+ struct io_wqe_acct *acct = &wqe->acct[index];
struct io_worker *worker;
- pid_t pid;
+ struct task_struct *tsk;
__set_current_state(TASK_RUNNING);
worker->wqe = wqe;
spin_lock_init(&worker->lock);
init_completion(&worker->ref_done);
- init_completion(&worker->started);
atomic_inc(&wq->worker_refs);
- if (index == IO_WQ_ACCT_BOUND)
- pid = io_wq_fork_thread(task_thread_bound, worker);
- else
- pid = io_wq_fork_thread(task_thread_unbound, worker);
- if (pid < 0) {
+ tsk = create_io_thread(io_wqe_worker, worker, wqe->node);
+ if (IS_ERR(tsk)) {
if (atomic_dec_and_test(&wq->worker_refs))
complete(&wq->worker_done);
kfree(worker);
return false;
}
- wait_for_completion(&worker->started);
+
+ tsk->pf_io_worker = worker;
+ worker->task = tsk;
+ set_cpus_allowed_ptr(tsk, cpumask_of_node(wqe->node));
+ tsk->flags |= PF_NOFREEZE | PF_NO_SETAFFINITY;
+
+ raw_spin_lock_irq(&wqe->lock);
+ hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
+ list_add_tail_rcu(&worker->all_list, &wqe->all_list);
+ worker->flags |= IO_WORKER_F_FREE;
+ if (index == IO_WQ_ACCT_BOUND)
+ worker->flags |= IO_WORKER_F_BOUND;
+ if (!acct->nr_workers && (worker->flags & IO_WORKER_F_BOUND))
+ worker->flags |= IO_WORKER_F_FIXED;
+ acct->nr_workers++;
+ raw_spin_unlock_irq(&wqe->lock);
+ wake_up_new_task(tsk);
return true;
}
static bool io_wq_worker_wake(struct io_worker *worker, void *data)
{
+ set_notify_signal(worker->task);
wake_up_process(worker->task);
return false;
}
}
}
+static bool io_wq_work_match_all(struct io_wq_work *work, void *data)
+{
+ return true;
+}
+
+static void io_wq_cancel_pending(struct io_wq *wq)
+{
+ struct io_cb_cancel_data match = {
+ .fn = io_wq_work_match_all,
+ .cancel_all = true,
+ };
+ int node;
+
+ for_each_node(node)
+ io_wqe_cancel_pending_work(wq->wqes[node], &match);
+}
+
/*
* Manager thread. Tasked with creating new workers, if we need them.
*/
sprintf(buf, "iou-mgr-%d", wq->task_pid);
set_task_comm(current, buf);
- current->flags |= PF_IO_WORKER;
- wq->manager = get_task_struct(current);
-
- complete(&wq->started);
do {
set_current_state(TASK_INTERRUPTIBLE);
io_wq_check_workers(wq);
schedule_timeout(HZ);
+ try_to_freeze();
if (fatal_signal_pending(current))
set_bit(IO_WQ_BIT_EXIT, &wq->state);
} while (!test_bit(IO_WQ_BIT_EXIT, &wq->state));
/* we might not ever have created any workers */
if (atomic_read(&wq->worker_refs))
wait_for_completion(&wq->worker_done);
+
+ spin_lock_irq(&wq->hash->wait.lock);
+ for_each_node(node)
+ list_del_init(&wq->wqes[node]->wait.entry);
+ spin_unlock_irq(&wq->hash->wait.lock);
+
+ io_wq_cancel_pending(wq);
complete(&wq->exited);
do_exit(0);
}
static int io_wq_fork_manager(struct io_wq *wq)
{
- int ret;
+ struct task_struct *tsk;
if (wq->manager)
return 0;
reinit_completion(&wq->worker_done);
- current->flags |= PF_IO_WORKER;
- ret = io_wq_fork_thread(io_wq_manager, wq);
- current->flags &= ~PF_IO_WORKER;
- if (ret >= 0) {
- wait_for_completion(&wq->started);
+ tsk = create_io_thread(io_wq_manager, wq, NUMA_NO_NODE);
+ if (!IS_ERR(tsk)) {
+ wq->manager = get_task_struct(tsk);
+ wake_up_new_task(tsk);
return 0;
}
- return ret;
+ return PTR_ERR(tsk);
}
static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
unsigned long flags;
/* Can only happen if manager creation fails after exec */
- if (unlikely(io_wq_fork_manager(wqe->wq))) {
+ if (io_wq_fork_manager(wqe->wq) ||
+ test_bit(IO_WQ_BIT_EXIT, &wqe->wq->state)) {
work->flags |= IO_WQ_WORK_CANCEL;
wqe->wq->do_work(work);
return;
work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));
}
-struct io_cb_cancel_data {
- work_cancel_fn *fn;
- void *data;
- int nr_running;
- int nr_pending;
- bool cancel_all;
-};
-
static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
{
struct io_cb_cancel_data *match = data;
}
wq->task_pid = current->pid;
- init_completion(&wq->started);
init_completion(&wq->exited);
refcount_set(&wq->refs, 1);
if (!ret)
return wq;
- io_wq_put_hash(data->hash);
err:
+ io_wq_put_hash(data->hash);
cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
for_each_node(node)
kfree(wq->wqes[node]);
set_bit(IO_WQ_BIT_EXIT, &wq->state);
io_wq_destroy_manager(wq);
- spin_lock_irq(&wq->hash->wait.lock);
for_each_node(node) {
struct io_wqe *wqe = wq->wqes[node];
-
- list_del_init(&wqe->wait.entry);
+ WARN_ON_ONCE(!wq_list_empty(&wqe->work_list));
kfree(wqe);
}
- spin_unlock_irq(&wq->hash->wait.lock);
io_wq_put_hash(wq->hash);
kfree(wq->wqes);
kfree(wq);
projects / linux-2.6-microblaze.git / blobdiff