Merge tag 'io_uring-bio-cache.5-2021-08-30' of git://git.kernel.dk/linux-block

[linux-2.6-microblaze.git] / block / bio.c

diff --git a/block/bio.c b/block/bio.c
index ef88fa3..1319dd2 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -528,16 +528,11 @@ EXPORT_SYMBOL(bio_kmalloc);
 
 void zero_fill_bio(struct bio *bio)
 {
-       unsigned long flags;
        struct bio_vec bv;
        struct bvec_iter iter;
 
-       bio_for_each_segment(bv, bio, iter) {
-               char *data = bvec_kmap_irq(&bv, &flags);
-               memset(data, 0, bv.bv_len);
-               flush_dcache_page(bv.bv_page);
-               bvec_kunmap_irq(data, &flags);
-       }
+       bio_for_each_segment(bv, bio, iter)
+               memzero_bvec(&bv);
 }
 EXPORT_SYMBOL(zero_fill_bio);
 
@@ -1066,6 +1061,14 @@ static int bio_iov_bvec_set_append(struct bio *bio, struct iov_iter *iter)
        return 0;
 }
 
+static void bio_put_pages(struct page **pages, size_t size, size_t off)
+{
+       size_t i, nr = DIV_ROUND_UP(size + (off & ~PAGE_MASK), PAGE_SIZE);
+
+       for (i = 0; i < nr; i++)
+               put_page(pages[i]);
+}
+
 #define PAGE_PTRS_PER_BVEC     (sizeof(struct bio_vec) / sizeof(struct page *))
 
 /**
@@ -1110,8 +1113,10 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
                        if (same_page)
                                put_page(page);
                } else {
-                       if (WARN_ON_ONCE(bio_full(bio, len)))
-                                return -EINVAL;
+                       if (WARN_ON_ONCE(bio_full(bio, len))) {
+                               bio_put_pages(pages + i, left, offset);
+                               return -EINVAL;
+                       }
                        __bio_add_page(bio, page, len, offset);
                }
                offset = 0;
@@ -1156,6 +1161,7 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter)
                len = min_t(size_t, PAGE_SIZE - offset, left);
                if (bio_add_hw_page(q, bio, page, len, offset,
                                max_append_sectors, &same_page) != len) {
+                       bio_put_pages(pages + i, left, offset);
                        ret = -EINVAL;
                        break;
                }
@@ -1278,27 +1284,15 @@ EXPORT_SYMBOL(bio_advance);
 void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
                        struct bio *src, struct bvec_iter *src_iter)
 {
-       struct bio_vec src_bv, dst_bv;
-       void *src_p, *dst_p;
-       unsigned bytes;
-
        while (src_iter->bi_size && dst_iter->bi_size) {
-               src_bv = bio_iter_iovec(src, *src_iter);
-               dst_bv = bio_iter_iovec(dst, *dst_iter);
-
-               bytes = min(src_bv.bv_len, dst_bv.bv_len);
-
-               src_p = kmap_atomic(src_bv.bv_page);
-               dst_p = kmap_atomic(dst_bv.bv_page);
-
-               memcpy(dst_p + dst_bv.bv_offset,
-                      src_p + src_bv.bv_offset,
-                      bytes);
-
-               kunmap_atomic(dst_p);
-               kunmap_atomic(src_p);
-
-               flush_dcache_page(dst_bv.bv_page);
+               struct bio_vec src_bv = bio_iter_iovec(src, *src_iter);
+               struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter);
+               unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len);
+               void *src_buf;
+
+               src_buf = bvec_kmap_local(&src_bv);
+               memcpy_to_bvec(&dst_bv, src_buf);
+               kunmap_local(src_buf);
 
                bio_advance_iter_single(src, src_iter, bytes);
                bio_advance_iter_single(dst, dst_iter, bytes);