Merge tag 'platform-drivers-x86-v5.12-1' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-2.6-microblaze.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index f285a91..f21d922 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1646,6 +1646,42 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 }
 EXPORT_SYMBOL(blk_mq_run_hw_queue);
 
+/*
+ * Is the request queue handled by an IO scheduler that does not respect
+ * hardware queues when dispatching?
+ */
+static bool blk_mq_has_sqsched(struct request_queue *q)
+{
+       struct elevator_queue *e = q->elevator;
+
+       if (e && e->type->ops.dispatch_request &&
+           !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
+               return true;
+       return false;
+}
+
+/*
+ * Return prefered queue to dispatch from (if any) for non-mq aware IO
+ * scheduler.
+ */
+static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
+{
+       struct blk_mq_hw_ctx *hctx;
+
+       /*
+        * If the IO scheduler does not respect hardware queues when
+        * dispatching, we just don't bother with multiple HW queues and
+        * dispatch from hctx for the current CPU since running multiple queues
+        * just causes lock contention inside the scheduler and pointless cache
+        * bouncing.
+        */
+       hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT,
+                                    raw_smp_processor_id());
+       if (!blk_mq_hctx_stopped(hctx))
+               return hctx;
+       return NULL;
+}
+
 /**
  * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
  * @q: Pointer to the request queue to run.
@@ -1653,14 +1689,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queue);
  */
 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
 {
-       struct blk_mq_hw_ctx *hctx;
+       struct blk_mq_hw_ctx *hctx, *sq_hctx;
        int i;
 
+       sq_hctx = NULL;
+       if (blk_mq_has_sqsched(q))
+               sq_hctx = blk_mq_get_sq_hctx(q);
        queue_for_each_hw_ctx(q, hctx, i) {
                if (blk_mq_hctx_stopped(hctx))
                        continue;
-
-               blk_mq_run_hw_queue(hctx, async);
+               /*
+                * Dispatch from this hctx either if there's no hctx preferred
+                * by IO scheduler or if it has requests that bypass the
+                * scheduler.
+                */
+               if (!sq_hctx || sq_hctx == hctx ||
+                   !list_empty_careful(&hctx->dispatch))
+                       blk_mq_run_hw_queue(hctx, async);
        }
 }
 EXPORT_SYMBOL(blk_mq_run_hw_queues);
@@ -1672,14 +1717,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queues);
  */
 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
 {
-       struct blk_mq_hw_ctx *hctx;
+       struct blk_mq_hw_ctx *hctx, *sq_hctx;
        int i;
 
+       sq_hctx = NULL;
+       if (blk_mq_has_sqsched(q))
+               sq_hctx = blk_mq_get_sq_hctx(q);
        queue_for_each_hw_ctx(q, hctx, i) {
                if (blk_mq_hctx_stopped(hctx))
                        continue;
-
-               blk_mq_delay_run_hw_queue(hctx, msecs);
+               /*
+                * Dispatch from this hctx either if there's no hctx preferred
+                * by IO scheduler or if it has requests that bypass the
+                * scheduler.
+                */
+               if (!sq_hctx || sq_hctx == hctx ||
+                   !list_empty_careful(&hctx->dispatch))
+                       blk_mq_delay_run_hw_queue(hctx, msecs);
        }
 }
 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
@@ -2128,7 +2182,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
  */
 blk_qc_t blk_mq_submit_bio(struct bio *bio)
 {
-       struct request_queue *q = bio->bi_disk->queue;
+       struct request_queue *q = bio->bi_bdev->bd_disk->queue;
        const int is_sync = op_is_sync(bio->bi_opf);
        const int is_flush_fua = op_is_flush(bio->bi_opf);
        struct blk_mq_alloc_data data = {
@@ -2653,7 +2707,6 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
                goto free_hctx;
 
        atomic_set(&hctx->nr_active, 0);
-       atomic_set(&hctx->elevator_queued, 0);
        if (node == NUMA_NO_NODE)
                node = set->numa_node;
        hctx->numa_node = node;