block: use singly linked list for bio cache

Pointless to maintain a head/tail for the list, as we never need to
access the tail. Entries are always LIFO for cache hotness reasons.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bio.c b/block/bio.c
index 15ab0d6..6fadc97 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -26,7 +26,7 @@
 #include "blk-rq-qos.h"
 
 struct bio_alloc_cache {
-       struct bio_list         free_list;
+       struct bio              *free_list;
        unsigned int            nr;
 };
 
@@ -630,7 +630,8 @@ static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
        unsigned int i = 0;
        struct bio *bio;
 
-       while ((bio = bio_list_pop(&cache->free_list)) != NULL) {
+       while ((bio = cache->free_list) != NULL) {
+               cache->free_list = bio->bi_next;
                cache->nr--;
                bio_free(bio);
                if (++i == nr)
@@ -689,7 +690,8 @@ void bio_put(struct bio *bio)
 
                bio_uninit(bio);
                cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
-               bio_list_add_head(&cache->free_list, bio);
+               bio->bi_next = cache->free_list;
+               cache->free_list = bio;
                if (++cache->nr > ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK)
                        bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK);
                put_cpu();
@@ -1704,8 +1706,9 @@ struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs,
                return bio_alloc_bioset(GFP_KERNEL, nr_vecs, bs);
 
        cache = per_cpu_ptr(bs->cache, get_cpu());
-       bio = bio_list_pop(&cache->free_list);
-       if (bio) {
+       if (cache->free_list) {
+               bio = cache->free_list;
+               cache->free_list = bio->bi_next;
                cache->nr--;
                put_cpu();
                bio_init(bio, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs);