blk-mq: split tag ->rqs[] into two

This is in preparation for having two sets of tags available. For
that we need a static index, and a dynamically assignable one.

Signed-off-by: Jens Axboe <axboe@fb.com>
Reviewed-by: Omar Sandoval <osandov@fb.com>

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index ced7527..9753747 100644 (file)
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -290,11 +290,11 @@ int blk_mq_reinit_tagset(struct blk_mq_tag_set *set)
                struct blk_mq_tags *tags = set->tags[i];
 
                for (j = 0; j < tags->nr_tags; j++) {
-                       if (!tags->rqs[j])
+                       if (!tags->static_rqs[j])
                                continue;
 
                        ret = set->ops->reinit_request(set->driver_data,
-                                               tags->rqs[j]);
+                                               tags->static_rqs[j]);
                        if (ret)
                                goto out;
                }

diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 923602d..41cd15f 100644 (file)
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -16,6 +16,7 @@ struct blk_mq_tags {
        struct sbitmap_queue breserved_tags;
 
        struct request **rqs;
+       struct request **static_rqs;
        struct list_head page_list;
 };
 

diff --git a/block/blk-mq.c b/block/blk-mq.c
index d40be64..89b8125 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -223,7 +223,7 @@ struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
 
        tag = blk_mq_get_tag(data);
        if (tag != BLK_MQ_TAG_FAIL) {
-               rq = data->hctx->tags->rqs[tag];
+               rq = data->hctx->tags->static_rqs[tag];
 
                if (blk_mq_tag_busy(data->hctx)) {
                        rq->rq_flags = RQF_MQ_INFLIGHT;
@@ -231,6 +231,7 @@ struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
                }
 
                rq->tag = tag;
+               data->hctx->tags->rqs[tag] = rq;
                blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
                return rq;
        }
@@ -1567,11 +1568,13 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                int i;
 
                for (i = 0; i < tags->nr_tags; i++) {
-                       if (!tags->rqs[i])
+                       struct request *rq = tags->static_rqs[i];
+
+                       if (!rq)
                                continue;
-                       set->ops->exit_request(set->driver_data, tags->rqs[i],
+                       set->ops->exit_request(set->driver_data, rq,
                                                hctx_idx, i);
-                       tags->rqs[i] = NULL;
+                       tags->static_rqs[i] = NULL;
                }
        }
 
@@ -1591,6 +1594,8 @@ void blk_mq_free_rq_map(struct blk_mq_tags *tags)
 {
        kfree(tags->rqs);
        tags->rqs = NULL;
+       kfree(tags->static_rqs);
+       tags->static_rqs = NULL;
 
        blk_mq_free_tags(tags);
 }
@@ -1616,6 +1621,15 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
                return NULL;
        }
 
+       tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
+                                GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
+                                set->numa_node);
+       if (!tags->static_rqs) {
+               kfree(tags->rqs);
+               blk_mq_free_tags(tags);
+               return NULL;
+       }
+
        return tags;
 }
 
@@ -1677,12 +1691,14 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
                to_do = min(entries_per_page, depth - i);
                left -= to_do * rq_size;
                for (j = 0; j < to_do; j++) {
-                       tags->rqs[i] = p;
+                       struct request *rq = p;
+
+                       tags->static_rqs[i] = rq;
                        if (set->ops->init_request) {
                                if (set->ops->init_request(set->driver_data,
-                                               tags->rqs[i], hctx_idx, i,
+                                               rq, hctx_idx, i,
                                                set->numa_node)) {
-                                       tags->rqs[i] = NULL;
+                                       tags->static_rqs[i] = NULL;
                                        goto fail;
                                }
                        }