irq_work: Optimize irq_work_single()

[linux-2.6-microblaze.git] / fs / io_uring.c

diff --git a/fs/io_uring.c b/fs/io_uring.c
index aeef02b..b42dfa0 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -790,7 +790,7 @@ static const struct io_op_def io_op_defs[] = {
                .unbound_nonreg_file    = 1,
                .pollin                 = 1,
                .async_size             = sizeof(struct io_async_rw),
-               .work_flags             = IO_WQ_WORK_BLKCG,
+               .work_flags             = IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM,
        },
        [IORING_OP_WRITE_FIXED] = {
                .needs_file             = 1,
@@ -798,7 +798,8 @@ static const struct io_op_def io_op_defs[] = {
                .unbound_nonreg_file    = 1,
                .pollout                = 1,
                .async_size             = sizeof(struct io_async_rw),
-               .work_flags             = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE,
+               .work_flags             = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE |
+                                               IO_WQ_WORK_MM,
        },
        [IORING_OP_POLL_ADD] = {
                .needs_file             = 1,
@@ -1983,7 +1984,8 @@ static int io_req_task_work_add(struct io_kiocb *req, bool twa_signal_ok)
 {
        struct task_struct *tsk = req->task;
        struct io_ring_ctx *ctx = req->ctx;
-       int ret, notify;
+       enum task_work_notify_mode notify;
+       int ret;
 
        if (tsk->flags & PF_EXITING)
                return -ESRCH;
@@ -1994,7 +1996,7 @@ static int io_req_task_work_add(struct io_kiocb *req, bool twa_signal_ok)
         * processing task_work. There's no reliable way to tell if TWA_RESUME
         * will do the job.
         */
-       notify = 0;
+       notify = TWA_NONE;
        if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok)
                notify = TWA_SIGNAL;
 
@@ -2063,7 +2065,7 @@ static void io_req_task_queue(struct io_kiocb *req)
 
                init_task_work(&req->task_work, io_req_task_cancel);
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
 }
@@ -2184,7 +2186,7 @@ static void io_free_req_deferred(struct io_kiocb *req)
                struct task_struct *tsk;
 
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
 }
@@ -3115,9 +3117,10 @@ static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
  * For files that don't have ->read_iter() and ->write_iter(), handle them
  * by looping over ->read() or ->write() manually.
  */
-static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
-                          struct iov_iter *iter)
+static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
 {
+       struct kiocb *kiocb = &req->rw.kiocb;
+       struct file *file = req->file;
        ssize_t ret = 0;
 
        /*
@@ -3137,11 +3140,8 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
                if (!iov_iter_is_bvec(iter)) {
                        iovec = iov_iter_iovec(iter);
                } else {
-                       /* fixed buffers import bvec */
-                       iovec.iov_base = kmap(iter->bvec->bv_page)
-                                               + iter->iov_offset;
-                       iovec.iov_len = min(iter->count,
-                                       iter->bvec->bv_len - iter->iov_offset);
+                       iovec.iov_base = u64_to_user_ptr(req->rw.addr);
+                       iovec.iov_len = req->rw.len;
                }
 
                if (rw == READ) {
@@ -3152,9 +3152,6 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
                                               iovec.iov_len, io_kiocb_ppos(kiocb));
                }
 
-               if (iov_iter_is_bvec(iter))
-                       kunmap(iter->bvec->bv_page);
-
                if (nr < 0) {
                        if (!ret)
                                ret = nr;
@@ -3163,6 +3160,8 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
                ret += nr;
                if (nr != iovec.iov_len)
                        break;
+               req->rw.len -= nr;
+               req->rw.addr += nr;
                iov_iter_advance(iter, nr);
        }
 
@@ -3298,7 +3297,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
                /* queue just for cancelation */
                init_task_work(&req->task_work, io_req_task_cancel);
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
        return 1;
@@ -3352,7 +3351,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
        if (req->file->f_op->read_iter)
                return call_read_iter(req->file, &req->rw.kiocb, iter);
        else if (req->file->f_op->read)
-               return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
+               return loop_rw_iter(READ, req, iter);
        else
                return -EINVAL;
 }
@@ -3543,7 +3542,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock,
        if (req->file->f_op->write_iter)
                ret2 = call_write_iter(req->file, kiocb, iter);
        else if (req->file->f_op->write)
-               ret2 = loop_rw_iter(WRITE, req->file, kiocb, iter);
+               ret2 = loop_rw_iter(WRITE, req, iter);
        else
                ret2 = -EINVAL;
 
@@ -4088,7 +4087,7 @@ static int io_madvise(struct io_kiocb *req, bool force_nonblock)
        if (force_nonblock)
                return -EAGAIN;
 
-       ret = do_madvise(ma->addr, ma->len, ma->advice);
+       ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
        if (ret < 0)
                req_set_fail_links(req);
        io_req_complete(req, ret);
@@ -4864,7 +4863,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
 
                WRITE_ONCE(poll->canceled, true);
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
        return 1;
@@ -6237,8 +6236,10 @@ punt:
        if (nxt) {
                req = nxt;
 
-               if (req->flags & REQ_F_FORCE_ASYNC)
+               if (req->flags & REQ_F_FORCE_ASYNC) {
+                       linked_timeout = NULL;
                        goto punt;
+               }
                goto again;
        }
 exit:
@@ -8682,19 +8683,11 @@ static void io_uring_del_task_file(struct file *file)
                fput(file);
 }
 
-static void __io_uring_attempt_task_drop(struct file *file)
-{
-       struct file *old = xa_load(&current->io_uring->xa, (unsigned long)file);
-
-       if (old == file)
-               io_uring_del_task_file(file);
-}
-
 /*
  * Drop task note for this file if we're the only ones that hold it after
  * pending fput()
  */
-static void io_uring_attempt_task_drop(struct file *file, bool exiting)
+static void io_uring_attempt_task_drop(struct file *file)
 {
        if (!current->io_uring)
                return;
@@ -8702,10 +8695,9 @@ static void io_uring_attempt_task_drop(struct file *file, bool exiting)
         * fput() is pending, will be 2 if the only other ref is our potential
         * task file note. If the task is exiting, drop regardless of count.
         */
-       if (!exiting && atomic_long_read(&file->f_count) != 2)
-               return;
-
-       __io_uring_attempt_task_drop(file);
+       if (fatal_signal_pending(current) || (current->flags & PF_EXITING) ||
+           atomic_long_read(&file->f_count) == 2)
+               io_uring_del_task_file(file);
 }
 
 void __io_uring_files_cancel(struct files_struct *files)
@@ -8763,16 +8755,7 @@ void __io_uring_task_cancel(void)
 
 static int io_uring_flush(struct file *file, void *data)
 {
-       struct io_ring_ctx *ctx = file->private_data;
-
-       /*
-        * If the task is going away, cancel work it may have pending
-        */
-       if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
-               data = NULL;
-
-       io_uring_cancel_task_requests(ctx, data);
-       io_uring_attempt_task_drop(file, !data);
+       io_uring_attempt_task_drop(file);
        return 0;
 }