blk-iocost: use local[64]_t for percpu stat

blk-iocost has been reading percpu stat counters from remote cpus which on
some archs can lead to torn reads in really rare occassions. Use local[64]_t
for those counters.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-iocost.c b/block/blk-iocost.c
index d37b55d..e2266e7 100644 (file)
--- a/block/blk-iocost.c
+++ b/block/blk-iocost.c
@@ -179,6 +179,8 @@
 #include <linux/parser.h>
 #include <linux/sched/signal.h>
 #include <linux/blk-cgroup.h>
+#include <asm/local.h>
+#include <asm/local64.h>
 #include "blk-rq-qos.h"
 #include "blk-stat.h"
 #include "blk-wbt.h"
@@ -373,8 +375,8 @@ struct ioc_params {
 };
 
 struct ioc_missed {
-       u32                             nr_met;
-       u32                             nr_missed;
+       local_t                         nr_met;
+       local_t                         nr_missed;
        u32                             last_met;
        u32                             last_missed;
 };
@@ -382,7 +384,7 @@ struct ioc_missed {
 struct ioc_pcpu_stat {
        struct ioc_missed               missed[2];
 
-       u64                             rq_wait_ns;
+       local64_t                       rq_wait_ns;
        u64                             last_rq_wait_ns;
 };
 
@@ -1278,8 +1280,8 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p
                u64 this_rq_wait_ns;
 
                for (rw = READ; rw <= WRITE; rw++) {
-                       u32 this_met = READ_ONCE(stat->missed[rw].nr_met);
-                       u32 this_missed = READ_ONCE(stat->missed[rw].nr_missed);
+                       u32 this_met = local_read(&stat->missed[rw].nr_met);
+                       u32 this_missed = local_read(&stat->missed[rw].nr_missed);
 
                        nr_met[rw] += this_met - stat->missed[rw].last_met;
                        nr_missed[rw] += this_missed - stat->missed[rw].last_missed;
@@ -1287,7 +1289,7 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p
                        stat->missed[rw].last_missed = this_missed;
                }
 
-               this_rq_wait_ns = READ_ONCE(stat->rq_wait_ns);
+               this_rq_wait_ns = local64_read(&stat->rq_wait_ns);
                rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;
                stat->last_rq_wait_ns = this_rq_wait_ns;
        }
@@ -1908,6 +1910,7 @@ static void ioc_rqos_done_bio(struct rq_qos *rqos, struct bio *bio)
 static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
 {
        struct ioc *ioc = rqos_to_ioc(rqos);
+       struct ioc_pcpu_stat *ccs;
        u64 on_q_ns, rq_wait_ns, size_nsec;
        int pidx, rw;
 
@@ -1931,13 +1934,17 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
        rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
        size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);
 
+       ccs = get_cpu_ptr(ioc->pcpu_stat);
+
        if (on_q_ns <= size_nsec ||
            on_q_ns - size_nsec <= ioc->params.qos[pidx] * NSEC_PER_USEC)
-               this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);
+               local_inc(&ccs->missed[rw].nr_met);
        else
-               this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);
+               local_inc(&ccs->missed[rw].nr_missed);
+
+       local64_add(rq_wait_ns, &ccs->rq_wait_ns);
 
-       this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);
+       put_cpu_ptr(ccs);
 }
 
 static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)
@@ -1977,7 +1984,7 @@ static int blk_iocost_init(struct request_queue *q)
 {
        struct ioc *ioc;
        struct rq_qos *rqos;
-       int ret;
+       int i, cpu, ret;
 
        ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
        if (!ioc)
@@ -1989,6 +1996,16 @@ static int blk_iocost_init(struct request_queue *q)
                return -ENOMEM;
        }
 
+       for_each_possible_cpu(cpu) {
+               struct ioc_pcpu_stat *ccs = per_cpu_ptr(ioc->pcpu_stat, cpu);
+
+               for (i = 0; i < ARRAY_SIZE(ccs->missed); i++) {
+                       local_set(&ccs->missed[i].nr_met, 0);
+                       local_set(&ccs->missed[i].nr_missed, 0);
+               }
+               local64_set(&ccs->rq_wait_ns, 0);
+       }
+
        rqos = &ioc->rqos;
        rqos->id = RQ_QOS_COST;
        rqos->ops = &ioc_rqos_ops;