Merge tag 'sysctl-6.8-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof...

[linux-2.6-microblaze.git] / fs / aio.c

diff --git a/fs/aio.c b/fs/aio.c
index ec8fdac..bb2ff48 100644 (file)
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -265,7 +265,7 @@ static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
                return ERR_CAST(inode);
 
        inode->i_mapping->a_ops = &aio_ctx_aops;
-       inode->i_mapping->private_data = ctx;
+       inode->i_mapping->i_private_data = ctx;
        inode->i_size = PAGE_SIZE * nr_pages;
 
        file = alloc_file_pseudo(inode, aio_mnt, "[aio]",
@@ -315,10 +315,10 @@ static void put_aio_ring_file(struct kioctx *ctx)
 
                /* Prevent further access to the kioctx from migratepages */
                i_mapping = aio_ring_file->f_mapping;
-               spin_lock(&i_mapping->private_lock);
-               i_mapping->private_data = NULL;
+               spin_lock(&i_mapping->i_private_lock);
+               i_mapping->i_private_data = NULL;
                ctx->aio_ring_file = NULL;
-               spin_unlock(&i_mapping->private_lock);
+               spin_unlock(&i_mapping->i_private_lock);
 
                fput(aio_ring_file);
        }
@@ -421,9 +421,9 @@ static int aio_migrate_folio(struct address_space *mapping, struct folio *dst,
 
        rc = 0;
 
-       /* mapping->private_lock here protects against the kioctx teardown.  */
-       spin_lock(&mapping->private_lock);
-       ctx = mapping->private_data;
+       /* mapping->i_private_lock here protects against the kioctx teardown.  */
+       spin_lock(&mapping->i_private_lock);
+       ctx = mapping->i_private_data;
        if (!ctx) {
                rc = -EINVAL;
                goto out;
@@ -475,7 +475,7 @@ static int aio_migrate_folio(struct address_space *mapping, struct folio *dst,
 out_unlock:
        mutex_unlock(&ctx->ring_lock);
 out:
-       spin_unlock(&mapping->private_lock);
+       spin_unlock(&mapping->i_private_lock);
        return rc;
 }
 #else
@@ -1105,6 +1105,11 @@ static inline void iocb_destroy(struct aio_kiocb *iocb)
        kmem_cache_free(kiocb_cachep, iocb);
 }
 
+struct aio_waiter {
+       struct wait_queue_entry w;
+       size_t                  min_nr;
+};
+
 /* aio_complete
  *     Called when the io request on the given iocb is complete.
  */
@@ -1113,7 +1118,7 @@ static void aio_complete(struct aio_kiocb *iocb)
        struct kioctx   *ctx = iocb->ki_ctx;
        struct aio_ring *ring;
        struct io_event *ev_page, *event;
-       unsigned tail, pos, head;
+       unsigned tail, pos, head, avail;
        unsigned long   flags;
 
        /*
@@ -1155,6 +1160,10 @@ static void aio_complete(struct aio_kiocb *iocb)
        ctx->completed_events++;
        if (ctx->completed_events > 1)
                refill_reqs_available(ctx, head, tail);
+
+       avail = tail > head
+               ? tail - head
+               : tail + ctx->nr_events - head;
        spin_unlock_irqrestore(&ctx->completion_lock, flags);
 
        pr_debug("added to ring %p at [%u]\n", iocb, tail);
@@ -1165,7 +1174,7 @@ static void aio_complete(struct aio_kiocb *iocb)
         * from IRQ context.
         */
        if (iocb->ki_eventfd)
-               eventfd_signal(iocb->ki_eventfd, 1);
+               eventfd_signal(iocb->ki_eventfd);
 
        /*
         * We have to order our ring_info tail store above and test
@@ -1175,8 +1184,18 @@ static void aio_complete(struct aio_kiocb *iocb)
         */
        smp_mb();
 
-       if (waitqueue_active(&ctx->wait))
-               wake_up(&ctx->wait);
+       if (waitqueue_active(&ctx->wait)) {
+               struct aio_waiter *curr, *next;
+               unsigned long flags;
+
+               spin_lock_irqsave(&ctx->wait.lock, flags);
+               list_for_each_entry_safe(curr, next, &ctx->wait.head, w.entry)
+                       if (avail >= curr->min_nr) {
+                               list_del_init_careful(&curr->w.entry);
+                               wake_up_process(curr->w.private);
+                       }
+               spin_unlock_irqrestore(&ctx->wait.lock, flags);
+       }
 }
 
 static inline void iocb_put(struct aio_kiocb *iocb)
@@ -1289,7 +1308,9 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr,
                        struct io_event __user *event,
                        ktime_t until)
 {
-       long ret = 0;
+       struct hrtimer_sleeper  t;
+       struct aio_waiter       w;
+       long ret = 0, ret2 = 0;
 
        /*
         * Note that aio_read_events() is being called as the conditional - i.e.
@@ -1305,12 +1326,38 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr,
         * the ringbuffer empty. So in practice we should be ok, but it's
         * something to be aware of when touching this code.
         */
-       if (until == 0)
-               aio_read_events(ctx, min_nr, nr, event, &ret);
-       else
-               wait_event_interruptible_hrtimeout(ctx->wait,
-                               aio_read_events(ctx, min_nr, nr, event, &ret),
-                               until);
+       aio_read_events(ctx, min_nr, nr, event, &ret);
+       if (until == 0 || ret < 0 || ret >= min_nr)
+               return ret;
+
+       hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+       if (until != KTIME_MAX) {
+               hrtimer_set_expires_range_ns(&t.timer, until, current->timer_slack_ns);
+               hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL);
+       }
+
+       init_wait(&w.w);
+
+       while (1) {
+               unsigned long nr_got = ret;
+
+               w.min_nr = min_nr - ret;
+
+               ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE);
+               if (!ret2 && !t.task)
+                       ret2 = -ETIME;
+
+               if (aio_read_events(ctx, min_nr, nr, event, &ret) || ret2)
+                       break;
+
+               if (nr_got == ret)
+                       schedule();
+       }
+
+       finish_wait(&ctx->wait, &w.w);
+       hrtimer_cancel(&t.timer);
+       destroy_hrtimer_on_stack(&t.timer);
+
        return ret;
 }
 
@@ -1497,7 +1544,7 @@ static ssize_t aio_setup_rw(int rw, const struct iocb *iocb,
        size_t len = iocb->aio_nbytes;
 
        if (!vectored) {
-               ssize_t ret = import_single_range(rw, buf, len, *iovec, iter);
+               ssize_t ret = import_ubuf(rw, buf, len, iter);
                *iovec = NULL;
                return ret;
        }