io_uring: remove io_identity

We are no longer grabbing state, so no need to maintain an IO identity
that we COW if there are changes.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 4104211..acc67ed 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -53,6 +53,9 @@ struct io_worker {
        struct io_wq_work *cur_work;
        spinlock_t lock;
 
+       const struct cred *cur_creds;
+       const struct cred *saved_creds;
+
        struct rcu_head rcu;
 };
 
@@ -171,6 +174,11 @@ static void io_worker_exit(struct io_worker *worker)
        worker->flags = 0;
        preempt_enable();
 
+       if (worker->saved_creds) {
+               revert_creds(worker->saved_creds);
+               worker->cur_creds = worker->saved_creds = NULL;
+       }
+
        raw_spin_lock_irq(&wqe->lock);
        hlist_nulls_del_rcu(&worker->nulls_node);
        list_del_rcu(&worker->all_list);
@@ -312,6 +320,10 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker)
                worker->flags |= IO_WORKER_F_FREE;
                hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
        }
+       if (worker->saved_creds) {
+               revert_creds(worker->saved_creds);
+               worker->cur_creds = worker->saved_creds = NULL;
+       }
 }
 
 static inline unsigned int io_get_work_hash(struct io_wq_work *work)
@@ -359,6 +371,18 @@ static void io_flush_signals(void)
        }
 }
 
+static void io_wq_switch_creds(struct io_worker *worker,
+                              struct io_wq_work *work)
+{
+       const struct cred *old_creds = override_creds(work->creds);
+
+       worker->cur_creds = work->creds;
+       if (worker->saved_creds)
+               put_cred(old_creds); /* creds set by previous switch */
+       else
+               worker->saved_creds = old_creds;
+}
+
 static void io_assign_current_work(struct io_worker *worker,
                                   struct io_wq_work *work)
 {
@@ -407,6 +431,8 @@ get_next:
                        unsigned int hash = io_get_work_hash(work);
 
                        next_hashed = wq_next_work(work);
+                       if (work->creds && worker->cur_creds != work->creds)
+                               io_wq_switch_creds(worker, work);
                        wq->do_work(work);
                        io_assign_current_work(worker, NULL);
 

diff --git a/fs/io-wq.h b/fs/io-wq.h
index bbe05dd..584f0bd 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -78,7 +78,7 @@ static inline void wq_list_del(struct io_wq_work_list *list,
 
 struct io_wq_work {
        struct io_wq_work_node list;
-       struct io_identity *identity;
+       const struct cred *creds;
        unsigned flags;
 };
 

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 6e88295..6d85103 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1094,7 +1094,7 @@ static bool io_match_task(struct io_kiocb *head,
                        continue;
                if (req->file && req->file->f_op == &io_uring_fops)
                        return true;
-               if (req->work.identity->files == files)
+               if (req->task->files == files)
                        return true;
        }
        return false;
@@ -1218,31 +1218,6 @@ static inline void req_set_fail_links(struct io_kiocb *req)
                req->flags |= REQ_F_FAIL_LINK;
 }
 
-/*
- * None of these are dereferenced, they are simply used to check if any of
- * them have changed. If we're under current and check they are still the
- * same, we're fine to grab references to them for actual out-of-line use.
- */
-static void io_init_identity(struct io_identity *id)
-{
-       id->files = current->files;
-       id->mm = current->mm;
-#ifdef CONFIG_BLK_CGROUP
-       rcu_read_lock();
-       id->blkcg_css = blkcg_css();
-       rcu_read_unlock();
-#endif
-       id->creds = current_cred();
-       id->nsproxy = current->nsproxy;
-       id->fs = current->fs;
-       id->fsize = rlimit(RLIMIT_FSIZE);
-#ifdef CONFIG_AUDIT
-       id->loginuid = current->loginuid;
-       id->sessionid = current->sessionid;
-#endif
-       refcount_set(&id->count, 1);
-}
-
 static inline void __io_req_init_async(struct io_kiocb *req)
 {
        memset(&req->work, 0, sizeof(req->work));
@@ -1255,17 +1230,10 @@ static inline void __io_req_init_async(struct io_kiocb *req)
  */
 static inline void io_req_init_async(struct io_kiocb *req)
 {
-       struct io_uring_task *tctx = current->io_uring;
-
        if (req->flags & REQ_F_WORK_INITIALIZED)
                return;
 
        __io_req_init_async(req);
-
-       /* Grab a ref if this isn't our static identity */
-       req->work.identity = tctx->identity;
-       if (tctx->identity != &tctx->__identity)
-               refcount_inc(&req->work.identity->count);
 }
 
 static void io_ring_ctx_ref_free(struct percpu_ref *ref)
@@ -1350,19 +1318,15 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
        return false;
 }
 
-static void io_put_identity(struct io_uring_task *tctx, struct io_kiocb *req)
-{
-       if (req->work.identity == &tctx->__identity)
-               return;
-       if (refcount_dec_and_test(&req->work.identity->count))
-               kfree(req->work.identity);
-}
-
 static void io_req_clean_work(struct io_kiocb *req)
 {
        if (!(req->flags & REQ_F_WORK_INITIALIZED))
                return;
 
+       if (req->work.creds) {
+               put_cred(req->work.creds);
+               req->work.creds = NULL;
+       }
        if (req->flags & REQ_F_INFLIGHT) {
                struct io_ring_ctx *ctx = req->ctx;
                struct io_uring_task *tctx = req->task->io_uring;
@@ -1377,7 +1341,6 @@ static void io_req_clean_work(struct io_kiocb *req)
        }
 
        req->flags &= ~REQ_F_WORK_INITIALIZED;
-       io_put_identity(req->task->io_uring, req);
 }
 
 static void io_req_track_inflight(struct io_kiocb *req)
@@ -1411,6 +1374,8 @@ static void io_prep_async_work(struct io_kiocb *req)
                if (def->unbound_nonreg_file)
                        req->work.flags |= IO_WQ_WORK_UNBOUND;
        }
+       if (!req->work.creds)
+               req->work.creds = get_current_cred();
 }
 
 static void io_prep_async_link(struct io_kiocb *req)
@@ -6376,9 +6341,9 @@ static void __io_queue_sqe(struct io_kiocb *req)
        const struct cred *old_creds = NULL;
        int ret;
 
-       if ((req->flags & REQ_F_WORK_INITIALIZED) &&
-           req->work.identity->creds != current_cred())
-               old_creds = override_creds(req->work.identity->creds);
+       if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
+           req->work.creds != current_cred())
+               old_creds = override_creds(req->work.creds);
 
        ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
 
@@ -6508,16 +6473,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
 
        id = READ_ONCE(sqe->personality);
        if (id) {
-               struct io_identity *iod;
-
-               iod = idr_find(&ctx->personality_idr, id);
-               if (unlikely(!iod))
-                       return -EINVAL;
-               refcount_inc(&iod->count);
-
                __io_req_init_async(req);
-               get_cred(iod->creds);
-               req->work.identity = iod;
+               req->work.creds = idr_find(&ctx->personality_idr, id);
+               if (unlikely(!req->work.creds))
+                       return -EINVAL;
+               get_cred(req->work.creds);
        }
 
        state = &ctx->submit_state;
@@ -7936,8 +7896,6 @@ static int io_uring_alloc_task_context(struct task_struct *task,
        tctx->last = NULL;
        atomic_set(&tctx->in_idle, 0);
        tctx->sqpoll = false;
-       io_init_identity(&tctx->__identity);
-       tctx->identity = &tctx->__identity;
        task->io_uring = tctx;
        spin_lock_init(&tctx->task_lock);
        INIT_WQ_LIST(&tctx->task_list);
@@ -7951,9 +7909,6 @@ void __io_uring_free(struct task_struct *tsk)
        struct io_uring_task *tctx = tsk->io_uring;
 
        WARN_ON_ONCE(!xa_empty(&tctx->xa));
-       WARN_ON_ONCE(refcount_read(&tctx->identity->count) != 1);
-       if (tctx->identity != &tctx->__identity)
-               kfree(tctx->identity);
        percpu_counter_destroy(&tctx->inflight);
        kfree(tctx);
        tsk->io_uring = NULL;
@@ -8593,13 +8548,11 @@ static int io_uring_fasync(int fd, struct file *file, int on)
 
 static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
 {
-       struct io_identity *iod;
+       const struct cred *creds;
 
-       iod = idr_remove(&ctx->personality_idr, id);
-       if (iod) {
-               put_cred(iod->creds);
-               if (refcount_dec_and_test(&iod->count))
-                       kfree(iod);
+       creds = idr_remove(&ctx->personality_idr, id);
+       if (creds) {
+               put_cred(creds);
                return 0;
        }
 
@@ -9300,8 +9253,7 @@ out_fput:
 #ifdef CONFIG_PROC_FS
 static int io_uring_show_cred(int id, void *p, void *data)
 {
-       struct io_identity *iod = p;
-       const struct cred *cred = iod->creds;
+       const struct cred *cred = p;
        struct seq_file *m = data;
        struct user_namespace *uns = seq_user_ns(m);
        struct group_info *gi;
@@ -9732,21 +9684,15 @@ out:
 
 static int io_register_personality(struct io_ring_ctx *ctx)
 {
-       struct io_identity *id;
+       const struct cred *creds;
        int ret;
 
-       id = kmalloc(sizeof(*id), GFP_KERNEL);
-       if (unlikely(!id))
-               return -ENOMEM;
-
-       io_init_identity(id);
-       id->creds = get_current_cred();
+       creds = get_current_cred();
 
-       ret = idr_alloc_cyclic(&ctx->personality_idr, id, 1, USHRT_MAX, GFP_KERNEL);
-       if (ret < 0) {
-               put_cred(id->creds);
-               kfree(id);
-       }
+       ret = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
+                               USHRT_MAX, GFP_KERNEL);
+       if (ret < 0)
+               put_cred(creds);
        return ret;
 }
 

diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h
index 0e95398..c48fcbd 100644 (file)
--- a/include/linux/io_uring.h
+++ b/include/linux/io_uring.h
@@ -5,23 +5,6 @@
 #include <linux/sched.h>
 #include <linux/xarray.h>
 
-struct io_identity {
-       struct files_struct             *files;
-       struct mm_struct                *mm;
-#ifdef CONFIG_BLK_CGROUP
-       struct cgroup_subsys_state      *blkcg_css;
-#endif
-       const struct cred               *creds;
-       struct nsproxy                  *nsproxy;
-       struct fs_struct                *fs;
-       unsigned long                   fsize;
-#ifdef CONFIG_AUDIT
-       kuid_t                          loginuid;
-       unsigned int                    sessionid;
-#endif
-       refcount_t                      count;
-};
-
 struct io_wq_work_node {
        struct io_wq_work_node *next;
 };
@@ -38,8 +21,6 @@ struct io_uring_task {
        struct file             *last;
        void                    *io_wq;
        struct percpu_counter   inflight;
-       struct io_identity      __identity;
-       struct io_identity      *identity;
        atomic_t                in_idle;
        bool                    sqpoll;