Merge tag 'nvme-5.13-2021-05-13' of git://git.infradead.org/nvme into block-5.13

[linux-2.6-microblaze.git] / block / kyber-iosched.c

diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index c25c41d..81e3279 100644 (file)
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -13,6 +13,8 @@
 #include <linux/module.h>
 #include <linux/sbitmap.h>
 
+#include <trace/events/block.h>
+
 #include "blk.h"
 #include "blk-mq.h"
 #include "blk-mq-debugfs.h"
@@ -353,19 +355,9 @@ static void kyber_timer_fn(struct timer_list *t)
        }
 }
 
-static unsigned int kyber_sched_tags_shift(struct request_queue *q)
-{
-       /*
-        * All of the hardware queues have the same depth, so we can just grab
-        * the shift of the first one.
-        */
-       return q->queue_hw_ctx[0]->sched_tags->bitmap_tags->sb.shift;
-}
-
 static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
 {
        struct kyber_queue_data *kqd;
-       unsigned int shift;
        int ret = -ENOMEM;
        int i;
 
@@ -400,9 +392,6 @@ static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
                kqd->latency_targets[i] = kyber_latency_targets[i];
        }
 
-       shift = kyber_sched_tags_shift(q);
-       kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U;
-
        return kqd;
 
 err_buckets:
@@ -458,9 +447,19 @@ static void kyber_ctx_queue_init(struct kyber_ctx_queue *kcq)
                INIT_LIST_HEAD(&kcq->rq_list[i]);
 }
 
-static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
+static void kyber_depth_updated(struct blk_mq_hw_ctx *hctx)
 {
        struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;
+       struct blk_mq_tags *tags = hctx->sched_tags;
+       unsigned int shift = tags->bitmap_tags->sb.shift;
+
+       kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U;
+
+       sbitmap_queue_min_shallow_depth(tags->bitmap_tags, kqd->async_depth);
+}
+
+static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
+{
        struct kyber_hctx_data *khd;
        int i;
 
@@ -479,7 +478,8 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 
        for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
                if (sbitmap_init_node(&khd->kcq_map[i], hctx->nr_ctx,
-                                     ilog2(8), GFP_KERNEL, hctx->numa_node)) {
+                                     ilog2(8), GFP_KERNEL, hctx->numa_node,
+                                     false, false)) {
                        while (--i >= 0)
                                sbitmap_free(&khd->kcq_map[i]);
                        goto err_kcqs;
@@ -502,8 +502,7 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
        khd->batching = 0;
 
        hctx->sched_data = khd;
-       sbitmap_queue_min_shallow_depth(hctx->sched_tags->bitmap_tags,
-                                       kqd->async_depth);
+       kyber_depth_updated(hctx);
 
        return 0;
 
@@ -562,11 +561,12 @@ static void kyber_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
        }
 }
 
-static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio,
+static bool kyber_bio_merge(struct request_queue *q, struct bio *bio,
                unsigned int nr_segs)
 {
+       struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
+       struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, bio->bi_opf, ctx);
        struct kyber_hctx_data *khd = hctx->sched_data;
-       struct blk_mq_ctx *ctx = blk_mq_get_ctx(hctx->queue);
        struct kyber_ctx_queue *kcq = &khd->kcqs[ctx->index_hw[hctx->type]];
        unsigned int sched_domain = kyber_sched_domain(bio->bi_opf);
        struct list_head *rq_list = &kcq->rq_list[sched_domain];
@@ -602,7 +602,7 @@ static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx,
                        list_move_tail(&rq->queuelist, head);
                sbitmap_set_bit(&khd->kcq_map[sched_domain],
                                rq->mq_ctx->index_hw[hctx->type]);
-               blk_mq_sched_request_inserted(rq);
+               trace_block_rq_insert(rq);
                spin_unlock(&kcq->lock);
        }
 }
@@ -1022,6 +1022,7 @@ static struct elevator_type kyber_sched = {
                .completed_request = kyber_completed_request,
                .dispatch_request = kyber_dispatch_request,
                .has_work = kyber_has_work,
+               .depth_updated = kyber_depth_updated,
        },
 #ifdef CONFIG_BLK_DEBUG_FS
        .queue_debugfs_attrs = kyber_queue_debugfs_attrs,