io_uring: implement our own schedule timeout handling

In preparation for having two distinct timeouts and avoid waking the
task if we don't need to.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 9e2b8d4..c443bac 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2350,6 +2350,34 @@ static bool current_pending_io(void)
        return percpu_counter_read_positive(&tctx->inflight);
 }
 
+static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
+{
+       struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
+
+       WRITE_ONCE(iowq->hit_timeout, 1);
+       wake_up_process(iowq->wq.private);
+       return HRTIMER_NORESTART;
+}
+
+static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
+                                     clockid_t clock_id)
+{
+       iowq->hit_timeout = 0;
+       hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
+       iowq->t.function = io_cqring_timer_wakeup;
+       hrtimer_set_expires_range_ns(&iowq->t, iowq->timeout, 0);
+       hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
+
+       if (!READ_ONCE(iowq->hit_timeout))
+               schedule();
+
+       hrtimer_cancel(&iowq->t);
+       destroy_hrtimer_on_stack(&iowq->t);
+       __set_current_state(TASK_RUNNING);
+
+       return READ_ONCE(iowq->hit_timeout) ? -ETIME : 0;
+}
+
 static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
                                     struct io_wait_queue *iowq)
 {
@@ -2362,11 +2390,10 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
         */
        if (current_pending_io())
                current->in_iowait = 1;
-       if (iowq->timeout == KTIME_MAX)
+       if (iowq->timeout != KTIME_MAX)
+               ret = io_cqring_schedule_timeout(iowq, ctx->clockid);
+       else
                schedule();
-       else if (!schedule_hrtimeout_range_clock(&iowq->timeout, 0,
-                                                HRTIMER_MODE_ABS, ctx->clockid))
-               ret = -ETIME;
        current->in_iowait = 0;
        return ret;
 }

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 9935819..f95c1b0 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -40,7 +40,9 @@ struct io_wait_queue {
        struct io_ring_ctx *ctx;
        unsigned cq_tail;
        unsigned nr_timeouts;
+       int hit_timeout;
        ktime_t timeout;
+       struct hrtimer t;
 
 #ifdef CONFIG_NET_RX_BUSY_POLL
        ktime_t napi_busy_poll_dt;