srcu: Move heuristics fields from srcu_struct to srcu_usage

This commit moves the ->srcu_size_jiffies, ->srcu_n_lock_retries,
and ->srcu_n_exp_nodelay fields from the srcu_struct structure to the
srcu_usage structure to reduce the size of the former in order to improve
cache locality.

Suggested-by: Christoph Hellwig <hch@lst.de>
Tested-by: Sachin Sant <sachinp@linux.ibm.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Tested-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 372e35b..3023492 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -73,6 +73,9 @@ struct srcu_usage {
        unsigned long srcu_gp_seq_needed_exp;   /* Furthest future exp GP. */
        unsigned long srcu_gp_start;            /* Last GP start timestamp (jiffies) */
        unsigned long srcu_last_gp_end;         /* Last GP end timestamp (ns) */
+       unsigned long srcu_size_jiffies;        /* Current contention-measurement interval. */
+       unsigned long srcu_n_lock_retries;      /* Contention events in current interval. */
+       unsigned long srcu_n_exp_nodelay;       /* # expedited no-delays in current GP phase. */
 };
 
 /*
@@ -80,9 +83,6 @@ struct srcu_usage {
  */
 struct srcu_struct {
        unsigned int srcu_idx;                  /* Current rdr array element. */
-       unsigned long srcu_size_jiffies;        /* Current contention-measurement interval. */
-       unsigned long srcu_n_lock_retries;      /* Contention events in current interval. */
-       unsigned long srcu_n_exp_nodelay;       /* # expedited no-delays in current GP phase. */
        struct srcu_data __percpu *sda;         /* Per-CPU srcu_data array. */
        bool sda_is_static;                     /* May ->sda be passed to free_percpu()? */
        unsigned long srcu_barrier_seq;         /* srcu_barrier seq #. */

diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 340eb68..291fb52 100644 (file)
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -348,11 +348,11 @@ static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)
        if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)
                return;
        j = jiffies;
-       if (ssp->srcu_size_jiffies != j) {
-               ssp->srcu_size_jiffies = j;
-               ssp->srcu_n_lock_retries = 0;
+       if (ssp->srcu_sup->srcu_size_jiffies != j) {
+               ssp->srcu_sup->srcu_size_jiffies = j;
+               ssp->srcu_sup->srcu_n_lock_retries = 0;
        }
-       if (++ssp->srcu_n_lock_retries <= small_contention_lim)
+       if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)
                return;
        __srcu_transition_to_big(ssp);
 }
@@ -624,8 +624,8 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp)
                if (time_after(j, gpstart))
                        jbase += j - gpstart;
                if (!jbase) {
-                       WRITE_ONCE(ssp->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_n_exp_nodelay) + 1);
-                       if (READ_ONCE(ssp->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)
+                       WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay) + 1);
+                       if (READ_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)
                                jbase = 1;
                }
        }
@@ -783,7 +783,7 @@ static void srcu_gp_start(struct srcu_struct *ssp)
                                       rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq));
        spin_unlock_rcu_node(sdp);  /* Interrupts remain disabled. */
        WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);
-       WRITE_ONCE(ssp->srcu_n_exp_nodelay, 0);
+       WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);
        smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */
        rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);
        state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);
@@ -1625,7 +1625,7 @@ static void srcu_advance_state(struct srcu_struct *ssp)
                srcu_flip(ssp);
                spin_lock_irq_rcu_node(ssp->srcu_sup);
                rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);
-               ssp->srcu_n_exp_nodelay = 0;
+               ssp->srcu_sup->srcu_n_exp_nodelay = 0;
                spin_unlock_irq_rcu_node(ssp->srcu_sup);
        }
 
@@ -1640,7 +1640,7 @@ static void srcu_advance_state(struct srcu_struct *ssp)
                        mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
                        return; /* readers present, retry later. */
                }
-               ssp->srcu_n_exp_nodelay = 0;
+               ssp->srcu_sup->srcu_n_exp_nodelay = 0;
                srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */
        }
 }