Merge tag 'x86_urgent_for_v5.13_rc1' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-2.6-microblaze.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 4e3a70a..466676b 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -361,11 +361,12 @@ static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data)
 
        if (e) {
                /*
-                * Flush requests are special and go directly to the
+                * Flush/passthrough requests are special and go directly to the
                 * dispatch list. Don't include reserved tags in the
                 * limiting, as it isn't useful.
                 */
                if (!op_is_flush(data->cmd_flags) &&
+                   !blk_op_is_passthrough(data->cmd_flags) &&
                    e->type->ops.limit_depth &&
                    !(data->flags & BLK_MQ_REQ_RESERVED))
                        e->type->ops.limit_depth(data->cmd_flags, data);
@@ -1277,10 +1278,15 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
                                                  bool need_budget)
 {
        struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
+       int budget_token = -1;
 
-       if (need_budget && !blk_mq_get_dispatch_budget(rq->q)) {
-               blk_mq_put_driver_tag(rq);
-               return PREP_DISPATCH_NO_BUDGET;
+       if (need_budget) {
+               budget_token = blk_mq_get_dispatch_budget(rq->q);
+               if (budget_token < 0) {
+                       blk_mq_put_driver_tag(rq);
+                       return PREP_DISPATCH_NO_BUDGET;
+               }
+               blk_mq_set_rq_budget_token(rq, budget_token);
        }
 
        if (!blk_mq_get_driver_tag(rq)) {
@@ -1297,7 +1303,7 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
                         * together during handling partial dispatch
                         */
                        if (need_budget)
-                               blk_mq_put_dispatch_budget(rq->q);
+                               blk_mq_put_dispatch_budget(rq->q, budget_token);
                        return PREP_DISPATCH_NO_TAG;
                }
        }
@@ -1307,12 +1313,16 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
 
 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
 static void blk_mq_release_budgets(struct request_queue *q,
-               unsigned int nr_budgets)
+               struct list_head *list)
 {
-       int i;
+       struct request *rq;
 
-       for (i = 0; i < nr_budgets; i++)
-               blk_mq_put_dispatch_budget(q);
+       list_for_each_entry(rq, list, queuelist) {
+               int budget_token = blk_mq_get_rq_budget_token(rq);
+
+               if (budget_token >= 0)
+                       blk_mq_put_dispatch_budget(q, budget_token);
+       }
 }
 
 /*
@@ -1410,7 +1420,8 @@ out:
                        (hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
                bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
 
-               blk_mq_release_budgets(q, nr_budgets);
+               if (nr_budgets)
+                       blk_mq_release_budgets(q, list);
 
                spin_lock(&hctx->lock);
                list_splice_tail_init(list, &hctx->dispatch);
@@ -2010,6 +2021,7 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
 {
        struct request_queue *q = rq->q;
        bool run_queue = true;
+       int budget_token;
 
        /*
         * RCU or SRCU read lock is needed before checking quiesced flag.
@@ -2027,11 +2039,14 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
        if (q->elevator && !bypass_insert)
                goto insert;
 
-       if (!blk_mq_get_dispatch_budget(q))
+       budget_token = blk_mq_get_dispatch_budget(q);
+       if (budget_token < 0)
                goto insert;
 
+       blk_mq_set_rq_budget_token(rq, budget_token);
+
        if (!blk_mq_get_driver_tag(rq)) {
-               blk_mq_put_dispatch_budget(q);
+               blk_mq_put_dispatch_budget(q, budget_token);
                goto insert;
        }
 
@@ -2703,7 +2718,7 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
                goto free_cpumask;
 
        if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
-                               gfp, node))
+                               gfp, node, false, false))
                goto free_ctxs;
        hctx->nr_ctx = 0;