Merge tag 'thermal-6.1-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael...

[linux-2.6-microblaze.git] / io_uring / io_uring.h

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 2f73f83..ef77d2a 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -24,9 +24,11 @@ enum {
        IOU_STOP_MULTISHOT      = -ECANCELED,
 };
 
-struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx);
+struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
 bool io_req_cqe_overflow(struct io_kiocb *req);
-int io_run_task_work_sig(void);
+int io_run_task_work_sig(struct io_ring_ctx *ctx);
+int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
+int io_run_local_work(struct io_ring_ctx *ctx);
 void io_req_complete_failed(struct io_kiocb *req, s32 res);
 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
 void io_req_complete_post(struct io_kiocb *req);
@@ -91,7 +93,8 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx)
 
 void io_cq_unlock_post(struct io_ring_ctx *ctx);
 
-static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
+static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx,
+                                                      bool overflow)
 {
        if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
                struct io_uring_cqe *cqe = ctx->cqe_cached;
@@ -103,7 +106,12 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
                return cqe;
        }
 
-       return __io_get_cqe(ctx);
+       return __io_get_cqe(ctx, overflow);
+}
+
+static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
+{
+       return io_get_cqe_overflow(ctx, false);
 }
 
 static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx,
@@ -195,17 +203,24 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx)
        smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
 }
 
-static inline void io_cqring_wake(struct io_ring_ctx *ctx)
+/* requires smb_mb() prior, see wq_has_sleeper() */
+static inline void __io_cqring_wake(struct io_ring_ctx *ctx)
 {
        /*
         * wake_up_all() may seem excessive, but io_wake_function() and
         * io_should_wake() handle the termination of the loop and only
         * wake as many waiters as we need to.
         */
-       if (wq_has_sleeper(&ctx->cq_wait))
+       if (waitqueue_active(&ctx->cq_wait))
                wake_up_all(&ctx->cq_wait);
 }
 
+static inline void io_cqring_wake(struct io_ring_ctx *ctx)
+{
+       smp_mb();
+       __io_cqring_wake(ctx);
+}
+
 static inline bool io_sqring_full(struct io_ring_ctx *ctx)
 {
        struct io_rings *r = ctx->rings;
@@ -221,17 +236,50 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
        return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
 }
 
-static inline bool io_run_task_work(void)
+static inline int io_run_task_work(void)
 {
-       if (test_thread_flag(TIF_NOTIFY_SIGNAL)) {
+       if (task_work_pending(current)) {
+               if (test_thread_flag(TIF_NOTIFY_SIGNAL))
+                       clear_notify_signal();
                __set_current_state(TASK_RUNNING);
-               clear_notify_signal();
-               if (task_work_pending(current))
-                       task_work_run();
-               return true;
+               task_work_run();
+               return 1;
        }
 
-       return false;
+       return 0;
+}
+
+static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
+{
+       return test_thread_flag(TIF_NOTIFY_SIGNAL) ||
+               !wq_list_empty(&ctx->work_llist);
+}
+
+static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
+{
+       int ret = 0;
+       int ret2;
+
+       if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+               ret = io_run_local_work(ctx);
+
+       /* want to run this after in case more is added */
+       ret2 = io_run_task_work();
+
+       /* Try propagate error in favour of if tasks were run,
+        * but still make sure to run them if requested
+        */
+       if (ret >= 0)
+               ret += ret2;
+
+       return ret;
+}
+
+static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
+{
+       if (llist_empty(&ctx->work_llist))
+               return 0;
+       return __io_run_local_work(ctx, true);
 }
 
 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
@@ -301,4 +349,10 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
        return container_of(node, struct io_kiocb, comp_list);
 }
 
+static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx)
+{
+       return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) ||
+                     ctx->submitter_task == current);
+}
+
 #endif