irq_work: Optimize irq_work_single()

[linux-2.6-microblaze.git] / fs / io_uring.c

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 4532045..b42dfa0 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -790,7 +790,7 @@ static const struct io_op_def io_op_defs[] = {
                .unbound_nonreg_file    = 1,
                .pollin                 = 1,
                .async_size             = sizeof(struct io_async_rw),
-               .work_flags             = IO_WQ_WORK_BLKCG,
+               .work_flags             = IO_WQ_WORK_BLKCG | IO_WQ_WORK_MM,
        },
        [IORING_OP_WRITE_FIXED] = {
                .needs_file             = 1,
@@ -798,7 +798,8 @@ static const struct io_op_def io_op_defs[] = {
                .unbound_nonreg_file    = 1,
                .pollout                = 1,
                .async_size             = sizeof(struct io_async_rw),
-               .work_flags             = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE,
+               .work_flags             = IO_WQ_WORK_BLKCG | IO_WQ_WORK_FSIZE |
+                                               IO_WQ_WORK_MM,
        },
        [IORING_OP_POLL_ADD] = {
                .needs_file             = 1,
@@ -1983,7 +1984,8 @@ static int io_req_task_work_add(struct io_kiocb *req, bool twa_signal_ok)
 {
        struct task_struct *tsk = req->task;
        struct io_ring_ctx *ctx = req->ctx;
-       int ret, notify;
+       enum task_work_notify_mode notify;
+       int ret;
 
        if (tsk->flags & PF_EXITING)
                return -ESRCH;
@@ -1994,7 +1996,7 @@ static int io_req_task_work_add(struct io_kiocb *req, bool twa_signal_ok)
         * processing task_work. There's no reliable way to tell if TWA_RESUME
         * will do the job.
         */
-       notify = 0;
+       notify = TWA_NONE;
        if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok)
                notify = TWA_SIGNAL;
 
@@ -2063,7 +2065,7 @@ static void io_req_task_queue(struct io_kiocb *req)
 
                init_task_work(&req->task_work, io_req_task_cancel);
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
 }
@@ -2184,7 +2186,7 @@ static void io_free_req_deferred(struct io_kiocb *req)
                struct task_struct *tsk;
 
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
 }
@@ -3115,9 +3117,10 @@ static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
  * For files that don't have ->read_iter() and ->write_iter(), handle them
  * by looping over ->read() or ->write() manually.
  */
-static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
-                          struct iov_iter *iter)
+static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
 {
+       struct kiocb *kiocb = &req->rw.kiocb;
+       struct file *file = req->file;
        ssize_t ret = 0;
 
        /*
@@ -3137,11 +3140,8 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
                if (!iov_iter_is_bvec(iter)) {
                        iovec = iov_iter_iovec(iter);
                } else {
-                       /* fixed buffers import bvec */
-                       iovec.iov_base = kmap(iter->bvec->bv_page)
-                                               + iter->iov_offset;
-                       iovec.iov_len = min(iter->count,
-                                       iter->bvec->bv_len - iter->iov_offset);
+                       iovec.iov_base = u64_to_user_ptr(req->rw.addr);
+                       iovec.iov_len = req->rw.len;
                }
 
                if (rw == READ) {
@@ -3152,9 +3152,6 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
                                               iovec.iov_len, io_kiocb_ppos(kiocb));
                }
 
-               if (iov_iter_is_bvec(iter))
-                       kunmap(iter->bvec->bv_page);
-
                if (nr < 0) {
                        if (!ret)
                                ret = nr;
@@ -3163,6 +3160,8 @@ static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
                ret += nr;
                if (nr != iovec.iov_len)
                        break;
+               req->rw.len -= nr;
+               req->rw.addr += nr;
                iov_iter_advance(iter, nr);
        }
 
@@ -3298,7 +3297,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
                /* queue just for cancelation */
                init_task_work(&req->task_work, io_req_task_cancel);
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
        return 1;
@@ -3352,7 +3351,7 @@ static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
        if (req->file->f_op->read_iter)
                return call_read_iter(req->file, &req->rw.kiocb, iter);
        else if (req->file->f_op->read)
-               return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
+               return loop_rw_iter(READ, req, iter);
        else
                return -EINVAL;
 }
@@ -3543,7 +3542,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock,
        if (req->file->f_op->write_iter)
                ret2 = call_write_iter(req->file, kiocb, iter);
        else if (req->file->f_op->write)
-               ret2 = loop_rw_iter(WRITE, req->file, kiocb, iter);
+               ret2 = loop_rw_iter(WRITE, req, iter);
        else
                ret2 = -EINVAL;
 
@@ -4088,7 +4087,7 @@ static int io_madvise(struct io_kiocb *req, bool force_nonblock)
        if (force_nonblock)
                return -EAGAIN;
 
-       ret = do_madvise(ma->addr, ma->len, ma->advice);
+       ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
        if (ret < 0)
                req_set_fail_links(req);
        io_req_complete(req, ret);
@@ -4864,7 +4863,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
 
                WRITE_ONCE(poll->canceled, true);
                tsk = io_wq_get_task(req->ctx->io_wq);
-               task_work_add(tsk, &req->task_work, 0);
+               task_work_add(tsk, &req->task_work, TWA_NONE);
                wake_up_process(tsk);
        }
        return 1;