io_uring: track link timeout's master explicitly

In preparation for converting singly linked lists for chaining requests,
make linked timeouts save requests that they're responsible for and not
count on doubly linked list for back referencing.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0441185..2d14a4f 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -446,6 +446,8 @@ struct io_timeout {
        u32                             off;
        u32                             target_seq;
        struct list_head                list;
+       /* head of the link, used by linked timeouts only */
+       struct io_kiocb                 *head;
 };
 
 struct io_timeout_rem {
@@ -1984,6 +1986,7 @@ static void io_kill_linked_timeout(struct io_kiocb *req)
                int ret;
 
                list_del_init(&link->link_list);
+               link->timeout.head = NULL;
                ret = hrtimer_try_to_cancel(&io->timer);
                if (ret != -1) {
                        io_cqring_fill_event(link, -ECANCELED);
@@ -6358,26 +6361,22 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
 {
        struct io_timeout_data *data = container_of(timer,
                                                struct io_timeout_data, timer);
-       struct io_kiocb *req = data->req;
+       struct io_kiocb *prev, *req = data->req;
        struct io_ring_ctx *ctx = req->ctx;
-       struct io_kiocb *prev = NULL;
        unsigned long flags;
 
        spin_lock_irqsave(&ctx->completion_lock, flags);
+       prev = req->timeout.head;
+       req->timeout.head = NULL;
 
        /*
         * We don't expect the list to be empty, that will only happen if we
         * race with the completion of the linked work.
         */
-       if (!list_empty(&req->link_list)) {
-               prev = list_entry(req->link_list.prev, struct io_kiocb,
-                                 link_list);
-               if (refcount_inc_not_zero(&prev->refs))
-                       list_del_init(&req->link_list);
-               else
-                       prev = NULL;
-       }
-
+       if (prev && refcount_inc_not_zero(&prev->refs))
+               list_del_init(&req->link_list);
+       else
+               prev = NULL;
        spin_unlock_irqrestore(&ctx->completion_lock, flags);
 
        if (prev) {
@@ -6396,7 +6395,7 @@ static void __io_queue_linked_timeout(struct io_kiocb *req)
         * If the list is now empty, then our linked request finished before
         * we got a chance to setup the timer
         */
-       if (!list_empty(&req->link_list)) {
+       if (req->timeout.head) {
                struct io_timeout_data *data = req->async_data;
 
                data->timer.function = io_link_timeout_fn;
@@ -6431,6 +6430,7 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
        if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
                return NULL;
 
+       nxt->timeout.head = req;
        nxt->flags |= REQ_F_LTIMEOUT_ACTIVE;
        req->flags |= REQ_F_LINK_TIMEOUT;
        return nxt;