io_uring: refactor tctx task_work list splicing

[linux-2.6-microblaze.git] / fs / io_uring.c

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 1b8d0ad..51db0d8 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -109,6 +109,8 @@
 #define SQE_VALID_FLAGS        (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
                                IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
                                IOSQE_BUFFER_SELECT)
+#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
+                               REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS)
 
 #define IO_TCTX_REFS_CACHE_NR  (1U << 10)
 
@@ -718,6 +720,7 @@ enum {
        REQ_F_COMPLETE_INLINE_BIT,
        REQ_F_REISSUE_BIT,
        REQ_F_DONT_REISSUE_BIT,
+       REQ_F_CREDS_BIT,
        /* keep async read/write and isreg together and in order */
        REQ_F_ASYNC_READ_BIT,
        REQ_F_ASYNC_WRITE_BIT,
@@ -771,6 +774,8 @@ enum {
        REQ_F_ASYNC_WRITE       = BIT(REQ_F_ASYNC_WRITE_BIT),
        /* regular file */
        REQ_F_ISREG             = BIT(REQ_F_ISREG_BIT),
+       /* has creds assigned */
+       REQ_F_CREDS             = BIT(REQ_F_CREDS_BIT),
 };
 
 struct async_poll {
@@ -1236,8 +1241,10 @@ static void io_prep_async_work(struct io_kiocb *req)
        const struct io_op_def *def = &io_op_defs[req->opcode];
        struct io_ring_ctx *ctx = req->ctx;
 
-       if (!req->creds)
+       if (!(req->flags & REQ_F_CREDS)) {
+               req->flags |= REQ_F_CREDS;
                req->creds = get_current_cred();
+       }
 
        req->work.list.next = NULL;
        req->work.flags = 0;
@@ -1622,8 +1629,7 @@ static void io_req_complete_post(struct io_kiocb *req, long res,
 
 static inline bool io_req_needs_clean(struct io_kiocb *req)
 {
-       return req->flags & (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP |
-                               REQ_F_POLLED | REQ_F_INFLIGHT);
+       return req->flags & IO_REQ_CLEAN_FLAGS;
 }
 
 static void io_req_complete_state(struct io_kiocb *req, long res,
@@ -1747,10 +1753,6 @@ static void io_dismantle_req(struct io_kiocb *req)
                percpu_ref_put(req->fixed_rsrc_refs);
        if (req->async_data)
                kfree(req->async_data);
-       if (req->creds) {
-               put_cred(req->creds);
-               req->creds = NULL;
-       }
 }
 
 /* must to be called somewhat shortly after putting a request */
@@ -1886,48 +1888,39 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx)
        percpu_ref_put(&ctx->refs);
 }
 
-static bool __tctx_task_work(struct io_uring_task *tctx)
+static void tctx_task_work(struct callback_head *cb)
 {
        struct io_ring_ctx *ctx = NULL;
-       struct io_wq_work_list list;
-       struct io_wq_work_node *node;
+       struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
+                                                 task_work);
 
-       if (wq_list_empty(&tctx->task_list))
-               return false;
+       clear_bit(0, &tctx->task_state);
 
-       spin_lock_irq(&tctx->task_lock);
-       list = tctx->task_list;
-       INIT_WQ_LIST(&tctx->task_list);
-       spin_unlock_irq(&tctx->task_lock);
+       while (!wq_list_empty(&tctx->task_list)) {
+               struct io_wq_work_node *node;
 
-       node = list.first;
-       while (node) {
-               struct io_wq_work_node *next = node->next;
-               struct io_kiocb *req;
+               spin_lock_irq(&tctx->task_lock);
+               node = tctx->task_list.first;
+               INIT_WQ_LIST(&tctx->task_list);
+               spin_unlock_irq(&tctx->task_lock);
 
-               req = container_of(node, struct io_kiocb, io_task_work.node);
-               if (req->ctx != ctx) {
-                       ctx_flush_and_put(ctx);
-                       ctx = req->ctx;
-                       percpu_ref_get(&ctx->refs);
-               }
+               while (node) {
+                       struct io_wq_work_node *next = node->next;
+                       struct io_kiocb *req = container_of(node, struct io_kiocb,
+                                                           io_task_work.node);
 
-               req->task_work.func(&req->task_work);
-               node = next;
+                       if (req->ctx != ctx) {
+                               ctx_flush_and_put(ctx);
+                               ctx = req->ctx;
+                               percpu_ref_get(&ctx->refs);
+                       }
+                       req->task_work.func(&req->task_work);
+                       node = next;
+               }
+               cond_resched();
        }
 
        ctx_flush_and_put(ctx);
-       return list.first != NULL;
-}
-
-static void tctx_task_work(struct callback_head *cb)
-{
-       struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work);
-
-       clear_bit(0, &tctx->task_state);
-
-       while (__tctx_task_work(tctx))
-               cond_resched();
 }
 
 static int io_req_task_work_add(struct io_kiocb *req)
@@ -5991,13 +5984,12 @@ static int io_req_prep_async(struct io_kiocb *req)
 
 static u32 io_get_sequence(struct io_kiocb *req)
 {
-       struct io_kiocb *pos;
-       struct io_ring_ctx *ctx = req->ctx;
-       u32 nr_reqs = 0;
+       u32 seq = req->ctx->cached_sq_head;
 
-       io_for_each_link(pos, req)
-               nr_reqs++;
-       return ctx->cached_sq_head - nr_reqs;
+       /* need original cached_sq_head, but it was increased for each req */
+       io_for_each_link(req, req)
+               seq--;
+       return seq;
 }
 
 static bool io_drain_req(struct io_kiocb *req)
@@ -6079,7 +6071,6 @@ static void io_clean_op(struct io_kiocb *req)
                        kfree(req->sr_msg.kbuf);
                        break;
                }
-               req->flags &= ~REQ_F_BUFFER_SELECTED;
        }
 
        if (req->flags & REQ_F_NEED_CLEANUP) {
@@ -6091,8 +6082,8 @@ static void io_clean_op(struct io_kiocb *req)
                case IORING_OP_WRITE_FIXED:
                case IORING_OP_WRITE: {
                        struct io_async_rw *io = req->async_data;
-                       if (io->free_iovec)
-                               kfree(io->free_iovec);
+
+                       kfree(io->free_iovec);
                        break;
                        }
                case IORING_OP_RECVMSG:
@@ -6120,7 +6111,6 @@ static void io_clean_op(struct io_kiocb *req)
                        putname(req->unlink.filename);
                        break;
                }
-               req->flags &= ~REQ_F_NEED_CLEANUP;
        }
        if ((req->flags & REQ_F_POLLED) && req->apoll) {
                kfree(req->apoll->double_poll);
@@ -6131,8 +6121,11 @@ static void io_clean_op(struct io_kiocb *req)
                struct io_uring_task *tctx = req->task->io_uring;
 
                atomic_dec(&tctx->inflight_tracked);
-               req->flags &= ~REQ_F_INFLIGHT;
        }
+       if (req->flags & REQ_F_CREDS)
+               put_cred(req->creds);
+
+       req->flags &= ~IO_REQ_CLEAN_FLAGS;
 }
 
 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
@@ -6141,7 +6134,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
        const struct cred *creds = NULL;
        int ret;
 
-       if (req->creds && req->creds != current_cred())
+       if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
                creds = override_creds(req->creds);
 
        switch (req->opcode) {
@@ -6534,7 +6527,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
        atomic_set(&req->refs, 2);
        req->task = current;
        req->result = 0;
-       req->creds = NULL;
 
        /* enforce forwards compatibility on users */
        if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
@@ -6556,6 +6548,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
                if (!req->creds)
                        return -EINVAL;
                get_cred(req->creds);
+               req->flags |= REQ_F_CREDS;
        }
        state = &ctx->submit_state;