perf cpumap: Add new map type for aggregation

[linux-2.6-microblaze.git] / block / blk-mq.c

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 55bcee5..b09ce00 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -95,7 +95,7 @@ static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
 }
 
 struct mq_inflight {
-       struct hd_struct *part;
+       struct block_device *part;
        unsigned int inflight[2];
 };
 
@@ -105,13 +105,15 @@ static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
 {
        struct mq_inflight *mi = priv;
 
-       if (rq->part == mi->part && blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
+       if ((!mi->part->bd_partno || rq->part == mi->part) &&
+           blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
                mi->inflight[rq_data_dir(rq)]++;
 
        return true;
 }
 
-unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part)
+unsigned int blk_mq_in_flight(struct request_queue *q,
+               struct block_device *part)
 {
        struct mq_inflight mi = { .part = part };
 
@@ -120,8 +122,8 @@ unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part)
        return mi.inflight[0] + mi.inflight[1];
 }
 
-void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part,
-                        unsigned int inflight[2])
+void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
+               unsigned int inflight[2])
 {
        struct mq_inflight mi = { .part = part };
 
@@ -671,9 +673,7 @@ bool blk_mq_complete_request_remote(struct request *rq)
                return false;
 
        if (blk_mq_complete_need_ipi(rq)) {
-               rq->csd.func = __blk_mq_complete_request_remote;
-               rq->csd.info = rq;
-               rq->csd.flags = 0;
+               INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
                smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd);
        } else {
                if (rq->q->nr_hw_queues > 1)
@@ -731,7 +731,7 @@ void blk_mq_start_request(struct request *rq)
 {
        struct request_queue *q = rq->q;
 
-       trace_block_rq_issue(q, rq);
+       trace_block_rq_issue(rq);
 
        if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
                rq->io_start_time_ns = ktime_get_ns();
@@ -758,7 +758,7 @@ static void __blk_mq_requeue_request(struct request *rq)
 
        blk_mq_put_driver_tag(rq);
 
-       trace_block_rq_requeue(q, rq);
+       trace_block_rq_requeue(rq);
        rq_qos_requeue(q, rq);
 
        if (blk_mq_request_started(rq)) {
@@ -1404,7 +1404,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
                        break;
                default:
                        errors++;
-                       blk_mq_end_request(rq, BLK_STS_IOERR);
+                       blk_mq_end_request(rq, ret);
                }
        } while (!list_empty(list));
 out:
@@ -1592,7 +1592,7 @@ select_cpu:
  * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue.
  * @hctx: Pointer to the hardware queue to run.
  * @async: If we want to run the queue asynchronously.
- * @msecs: Microseconds of delay to wait before running the queue.
+ * @msecs: Milliseconds of delay to wait before running the queue.
  *
  * If !@async, try to run the queue now. Else, run the queue asynchronously and
  * with a delay of @msecs.
@@ -1621,7 +1621,7 @@ static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
 /**
  * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
  * @hctx: Pointer to the hardware queue to run.
- * @msecs: Microseconds of delay to wait before running the queue.
+ * @msecs: Milliseconds of delay to wait before running the queue.
  *
  * Run a hardware queue asynchronously with a delay of @msecs.
  */
@@ -1685,7 +1685,7 @@ EXPORT_SYMBOL(blk_mq_run_hw_queues);
 /**
  * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
  * @q: Pointer to the request queue to run.
- * @msecs: Microseconds of delay to wait before running the queues.
+ * @msecs: Milliseconds of delay to wait before running the queues.
  */
 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
 {
@@ -1819,7 +1819,7 @@ static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
 
        lockdep_assert_held(&ctx->lock);
 
-       trace_block_rq_insert(hctx->queue, rq);
+       trace_block_rq_insert(rq);
 
        if (at_head)
                list_add(&rq->queuelist, &ctx->rq_lists[type]);
@@ -1876,7 +1876,7 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
         */
        list_for_each_entry(rq, list, queuelist) {
                BUG_ON(rq->mq_ctx != ctx);
-               trace_block_rq_insert(hctx->queue, rq);
+               trace_block_rq_insert(rq);
        }
 
        spin_lock(&ctx->lock);
@@ -2157,6 +2157,7 @@ blk_qc_t blk_mq_submit_bio(struct bio *bio)
        unsigned int nr_segs;
        blk_qc_t cookie;
        blk_status_t ret;
+       bool hipri;
 
        blk_queue_bounce(q, &bio);
        __blk_queue_split(&bio, &nr_segs);
@@ -2173,6 +2174,8 @@ blk_qc_t blk_mq_submit_bio(struct bio *bio)
 
        rq_qos_throttle(q, bio);
 
+       hipri = bio->bi_opf & REQ_HIPRI;
+
        data.cmd_flags = bio->bi_opf;
        rq = __blk_mq_alloc_request(&data);
        if (unlikely(!rq)) {
@@ -2182,7 +2185,7 @@ blk_qc_t blk_mq_submit_bio(struct bio *bio)
                goto queue_exit;
        }
 
-       trace_block_getrq(q, bio, bio->bi_opf);
+       trace_block_getrq(bio);
 
        rq_qos_track(q, rq, bio);
 
@@ -2265,6 +2268,8 @@ blk_qc_t blk_mq_submit_bio(struct bio *bio)
                blk_mq_sched_insert_request(rq, false, true, true);
        }
 
+       if (!hipri)
+               return BLK_QC_T_NONE;
        return cookie;
 queue_exit:
        blk_queue_exit(q);
@@ -3375,6 +3380,12 @@ static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
        return 0;
 }
 
+static int blk_mq_alloc_tag_set_tags(struct blk_mq_tag_set *set,
+                               int new_nr_hw_queues)
+{
+       return blk_mq_realloc_tag_set_tags(set, 0, new_nr_hw_queues);
+}
+
 /*
  * Alloc a tag set to be associated with one or more request queues.
  * May fail with EINVAL for various error conditions. May adjust the
@@ -3428,7 +3439,7 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
        if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
                set->nr_hw_queues = nr_cpu_ids;
 
-       if (blk_mq_realloc_tag_set_tags(set, 0, set->nr_hw_queues) < 0)
+       if (blk_mq_alloc_tag_set_tags(set, set->nr_hw_queues) < 0)
                return -ENOMEM;
 
        ret = -ENOMEM;
@@ -3863,9 +3874,10 @@ int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
         * the state. Like for the other success return cases, the
         * caller is responsible for checking if the IO completed. If
         * the IO isn't complete, we'll get called again and will go
-        * straight to the busy poll loop.
+        * straight to the busy poll loop. If specified not to spin,
+        * we also should not sleep.
         */
-       if (blk_mq_poll_hybrid(q, hctx, cookie))
+       if (spin && blk_mq_poll_hybrid(q, hctx, cookie))
                return 1;
 
        hctx->poll_considered++;