Running sqpoll cancellations via task_work_run() is a bad idea because
it depends on other task works to be run, but those may be locked in
currently running task_work_run() because of how it's (splicing the list
in batches).
Enqueue and run them through a separate callback head, namely
struct io_sq_data::park_task_work. As a nice bonus we now precisely
control where it's run, that's much safer than guessing where it can
happen as it was before.
Reported-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
unsigned long state;
struct completion exited;
unsigned long state;
struct completion exited;
+ struct callback_head *park_task_work;
};
#define IO_IOPOLL_BATCH 8
};
#define IO_IOPOLL_BATCH 8
cond_resched();
mutex_lock(&sqd->lock);
io_run_task_work();
cond_resched();
mutex_lock(&sqd->lock);
io_run_task_work();
+ io_run_task_work_head(&sqd->park_task_work);
timeout = jiffies + sqd->sq_thread_idle;
continue;
}
timeout = jiffies + sqd->sq_thread_idle;
continue;
}
}
finish_wait(&sqd->wait, &wait);
}
finish_wait(&sqd->wait, &wait);
+ io_run_task_work_head(&sqd->park_task_work);
timeout = jiffies + sqd->sq_thread_idle;
}
timeout = jiffies + sqd->sq_thread_idle;
}
mutex_unlock(&sqd->lock);
io_run_task_work();
mutex_unlock(&sqd->lock);
io_run_task_work();
+ io_run_task_work_head(&sqd->park_task_work);
complete(&sqd->exited);
do_exit(0);
}
complete(&sqd->exited);
do_exit(0);
}
if (task) {
init_completion(&work.completion);
init_task_work(&work.task_work, io_sqpoll_cancel_cb);
if (task) {
init_completion(&work.completion);
init_task_work(&work.task_work, io_sqpoll_cancel_cb);
- WARN_ON_ONCE(task_work_add(task, &work.task_work, TWA_SIGNAL));
+ io_task_work_add_head(&sqd->park_task_work, &work.task_work);
wake_up_process(task);
}
io_sq_thread_unpark(sqd);
wake_up_process(task);
}
io_sq_thread_unpark(sqd);