io_uring: optimise tctx node checks/alloc

First of all, w need to set tctx->sqpoll only when we add a new entry
into ->xa, so move it from the hot path. Also extract a hot path for
io_uring_add_task_file() as an inline helper.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 13a64b1..1c846d5 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -8846,10 +8846,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
        }
 }
 
-/*
- * Note that this task has used io_uring. We use it for cancelation purposes.
- */
-static int io_uring_add_task_file(struct io_ring_ctx *ctx)
+static int __io_uring_add_task_file(struct io_ring_ctx *ctx)
 {
        struct io_uring_task *tctx = current->io_uring;
        struct io_tctx_node *node;
@@ -8861,32 +8858,40 @@ static int io_uring_add_task_file(struct io_ring_ctx *ctx)
                        return ret;
                tctx = current->io_uring;
        }
-       if (tctx->last != ctx) {
-               void *old = xa_load(&tctx->xa, (unsigned long)ctx);
-
-               if (!old) {
-                       node = kmalloc(sizeof(*node), GFP_KERNEL);
-                       if (!node)
-                               return -ENOMEM;
-                       node->ctx = ctx;
-                       node->task = current;
-
-                       ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
-                                               node, GFP_KERNEL));
-                       if (ret) {
-                               kfree(node);
-                               return ret;
-                       }
+       if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
+               node = kmalloc(sizeof(*node), GFP_KERNEL);
+               if (!node)
+                       return -ENOMEM;
+               node->ctx = ctx;
+               node->task = current;
 
-                       mutex_lock(&ctx->uring_lock);
-                       list_add(&node->ctx_node, &ctx->tctx_list);
-                       mutex_unlock(&ctx->uring_lock);
+               ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
+                                       node, GFP_KERNEL));
+               if (ret) {
+                       kfree(node);
+                       return ret;
                }
-               tctx->last = ctx;
+
+               mutex_lock(&ctx->uring_lock);
+               list_add(&node->ctx_node, &ctx->tctx_list);
+               mutex_unlock(&ctx->uring_lock);
        }
+       tctx->last = ctx;
        return 0;
 }
 
+/*
+ * Note that this task has used io_uring. We use it for cancelation purposes.
+ */
+static inline int io_uring_add_task_file(struct io_ring_ctx *ctx)
+{
+       struct io_uring_task *tctx = current->io_uring;
+
+       if (likely(tctx && tctx->last == ctx))
+               return 0;
+       return __io_uring_add_task_file(ctx);
+}
+
 /*
  * Remove this io_uring_file -> task mapping.
  */