io_uring: use io-wq manager as backup task if task is exiting

If the original task is (or has) exited, then the task work will not get
queued properly. Allow for using the io-wq manager task to queue this
work for execution, and ensure that the io-wq manager notices and runs
this work if woken up (or exiting).

Reported-by: Dan Melnic <dmm@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index cc5cf22..4023c98 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -17,6 +17,7 @@
 #include <linux/kthread.h>
 #include <linux/rculist_nulls.h>
 #include <linux/fs_struct.h>
+#include <linux/task_work.h>
 
 #include "io-wq.h"
 
@@ -716,6 +717,9 @@ static int io_wq_manager(void *data)
        complete(&wq->done);
 
        while (!kthread_should_stop()) {
+               if (current->task_works)
+                       task_work_run();
+
                for_each_node(node) {
                        struct io_wqe *wqe = wq->wqes[node];
                        bool fork_worker[2] = { false, false };
@@ -738,6 +742,9 @@ static int io_wq_manager(void *data)
                schedule_timeout(HZ);
        }
 
+       if (current->task_works)
+               task_work_run();
+
        return 0;
 err:
        set_bit(IO_WQ_BIT_ERROR, &wq->state);
@@ -1124,3 +1131,8 @@ void io_wq_destroy(struct io_wq *wq)
        if (refcount_dec_and_test(&wq->use_refs))
                __io_wq_destroy(wq);
 }
+
+struct task_struct *io_wq_get_task(struct io_wq *wq)
+{
+       return wq->manager;
+}

diff --git a/fs/io-wq.h b/fs/io-wq.h
index 3ee7356..5ba12de 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -136,6 +136,8 @@ typedef bool (work_cancel_fn)(struct io_wq_work *, void *);
 enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
                                        void *data);
 
+struct task_struct *io_wq_get_task(struct io_wq *wq);
+
 #if defined(CONFIG_IO_WQ)
 extern void io_wq_worker_sleeping(struct task_struct *);
 extern void io_wq_worker_running(struct task_struct *);

diff --git a/fs/io_uring.c b/fs/io_uring.c
index b343525..2460c33 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -4120,6 +4120,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
                           __poll_t mask, task_work_func_t func)
 {
        struct task_struct *tsk;
+       int ret;
 
        /* for instances that support it check for an event match first: */
        if (mask && !(mask & poll->events))
@@ -4133,11 +4134,15 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
        req->result = mask;
        init_task_work(&req->task_work, func);
        /*
-        * If this fails, then the task is exiting. If that is the case, then
-        * the exit check will ultimately cancel these work items. Hence we
-        * don't need to check here and handle it specifically.
+        * If this fails, then the task is exiting. Punt to one of the io-wq
+        * threads to ensure the work gets run, we can't always rely on exit
+        * cancelation taking care of this.
         */
-       task_work_add(tsk, &req->task_work, true);
+       ret = task_work_add(tsk, &req->task_work, true);
+       if (unlikely(ret)) {
+               tsk = io_wq_get_task(req->ctx->io_wq);
+               task_work_add(tsk, &req->task_work, true);
+       }
        wake_up_process(tsk);
        return 1;
 }