rcu/nocb: Remove obsolete nocb_q_count and nocb_q_count_lazy fields

[linux-2.6-microblaze.git] / block / blk-rq-qos.c

diff --git a/block/blk-rq-qos.c b/block/blk-rq-qos.c
index d169d71..3954c0d 100644 (file)
--- a/block/blk-rq-qos.c
+++ b/block/blk-rq-qos.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-2.0
+
 #include "blk-rq-qos.h"
 
 /*
@@ -200,6 +202,7 @@ static int rq_qos_wake_function(struct wait_queue_entry *curr,
                return -1;
 
        data->got_token = true;
+       smp_wmb();
        list_del_init(&curr->entry);
        wake_up_process(data->task);
        return 1;
@@ -207,9 +210,10 @@ static int rq_qos_wake_function(struct wait_queue_entry *curr,
 
 /**
  * rq_qos_wait - throttle on a rqw if we need to
- * @private_data - caller provided specific data
- * @acquire_inflight_cb - inc the rqw->inflight counter if we can
- * @cleanup_cb - the callback to cleanup in case we race with a waker
+ * @rqw: rqw to throttle on
+ * @private_data: caller provided specific data
+ * @acquire_inflight_cb: inc the rqw->inflight counter if we can
+ * @cleanup_cb: the callback to cleanup in case we race with a waker
  *
  * This provides a uniform place for the rq_qos users to do their throttling.
  * Since you can end up with a lot of things sleeping at once, this manages the
@@ -241,7 +245,9 @@ void rq_qos_wait(struct rq_wait *rqw, void *private_data,
                return;
 
        prepare_to_wait_exclusive(&rqw->wait, &data.wq, TASK_UNINTERRUPTIBLE);
+       has_sleeper = !wq_has_single_sleeper(&rqw->wait);
        do {
+               /* The memory barrier in set_task_state saves us here. */
                if (data.got_token)
                        break;
                if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) {
@@ -252,12 +258,14 @@ void rq_qos_wait(struct rq_wait *rqw, void *private_data,
                         * which means we now have two. Put our local token
                         * and wake anyone else potentially waiting for one.
                         */
+                       smp_rmb();
                        if (data.got_token)
                                cleanup_cb(rqw, private_data);
                        break;
                }
                io_schedule();
-               has_sleeper = false;
+               has_sleeper = true;
+               set_current_state(TASK_UNINTERRUPTIBLE);
        } while (1);
        finish_wait(&rqw->wait, &data.wq);
 }