SUNRPC: Use per-CPU counters to tally server RPC counts

 - Improves counting accuracy
 - Reduces cross-CPU memory traffic

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>

diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c
index 0b28a6c..1da0023 100644 (file)
--- a/fs/lockd/svc.c
+++ b/fs/lockd/svc.c
@@ -721,7 +721,7 @@ out_encode_err:
 /*
  * Define NLM program and procedures
  */
-static unsigned int nlmsvc_version1_count[17];
+static DEFINE_PER_CPU_ALIGNED(unsigned long, nlmsvc_version1_count[17]);
 static const struct svc_version        nlmsvc_version1 = {
        .vs_vers        = 1,
        .vs_nproc       = 17,
@@ -730,26 +730,31 @@ static const struct svc_version   nlmsvc_version1 = {
        .vs_dispatch    = nlmsvc_dispatch,
        .vs_xdrsize     = NLMSVC_XDRSIZE,
 };
-static unsigned int nlmsvc_version3_count[24];
+
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nlmsvc_version3_count[ARRAY_SIZE(nlmsvc_procedures)]);
 static const struct svc_version        nlmsvc_version3 = {
        .vs_vers        = 3,
-       .vs_nproc       = 24,
+       .vs_nproc       = ARRAY_SIZE(nlmsvc_procedures),
        .vs_proc        = nlmsvc_procedures,
        .vs_count       = nlmsvc_version3_count,
        .vs_dispatch    = nlmsvc_dispatch,
        .vs_xdrsize     = NLMSVC_XDRSIZE,
 };
+
 #ifdef CONFIG_LOCKD_V4
-static unsigned int nlmsvc_version4_count[24];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nlmsvc_version4_count[ARRAY_SIZE(nlmsvc_procedures4)]);
 static const struct svc_version        nlmsvc_version4 = {
        .vs_vers        = 4,
-       .vs_nproc       = 24,
+       .vs_nproc       = ARRAY_SIZE(nlmsvc_procedures4),
        .vs_proc        = nlmsvc_procedures4,
        .vs_count       = nlmsvc_version4_count,
        .vs_dispatch    = nlmsvc_dispatch,
        .vs_xdrsize     = NLMSVC_XDRSIZE,
 };
 #endif
+
 static const struct svc_version *nlmsvc_version[] = {
        [1] = &nlmsvc_version1,
        [3] = &nlmsvc_version3,

diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 13b2af5..321af81 100644 (file)
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -1069,7 +1069,8 @@ static const struct svc_procedure nfs4_callback_procedures1[] = {
        }
 };
 
-static unsigned int nfs4_callback_count1[ARRAY_SIZE(nfs4_callback_procedures1)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfs4_callback_count1[ARRAY_SIZE(nfs4_callback_procedures1)]);
 const struct svc_version nfs4_callback_version1 = {
        .vs_vers = 1,
        .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1),
@@ -1081,7 +1082,8 @@ const struct svc_version nfs4_callback_version1 = {
        .vs_need_cong_ctrl = true,
 };
 
-static unsigned int nfs4_callback_count4[ARRAY_SIZE(nfs4_callback_procedures1)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfs4_callback_count4[ARRAY_SIZE(nfs4_callback_procedures1)]);
 const struct svc_version nfs4_callback_version4 = {
        .vs_vers = 4,
        .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1),

diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c
index 1457f59..dea5588 100644 (file)
--- a/fs/nfsd/nfs2acl.c
+++ b/fs/nfsd/nfs2acl.c
@@ -377,10 +377,11 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = {
        },
 };
 
-static unsigned int nfsd_acl_count2[ARRAY_SIZE(nfsd_acl_procedures2)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_acl_count2[ARRAY_SIZE(nfsd_acl_procedures2)]);
 const struct svc_version nfsd_acl_version2 = {
        .vs_vers        = 2,
-       .vs_nproc       = 5,
+       .vs_nproc       = ARRAY_SIZE(nfsd_acl_procedures2),
        .vs_proc        = nfsd_acl_procedures2,
        .vs_count       = nfsd_acl_count2,
        .vs_dispatch    = nfsd_dispatch,

diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c
index 6471081..bf1c52c 100644 (file)
--- a/fs/nfsd/nfs3acl.c
+++ b/fs/nfsd/nfs3acl.c
@@ -266,10 +266,11 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = {
        },
 };
 
-static unsigned int nfsd_acl_count3[ARRAY_SIZE(nfsd_acl_procedures3)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_acl_count3[ARRAY_SIZE(nfsd_acl_procedures3)]);
 const struct svc_version nfsd_acl_version3 = {
        .vs_vers        = 3,
-       .vs_nproc       = 3,
+       .vs_nproc       = ARRAY_SIZE(nfsd_acl_procedures3),
        .vs_proc        = nfsd_acl_procedures3,
        .vs_count       = nfsd_acl_count3,
        .vs_dispatch    = nfsd_dispatch,

diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
index d01b29a..1c73921 100644 (file)
--- a/fs/nfsd/nfs3proc.c
+++ b/fs/nfsd/nfs3proc.c
@@ -1064,10 +1064,11 @@ static const struct svc_procedure nfsd_procedures3[22] = {
        },
 };
 
-static unsigned int nfsd_count3[ARRAY_SIZE(nfsd_procedures3)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_count3[ARRAY_SIZE(nfsd_procedures3)]);
 const struct svc_version nfsd_version3 = {
        .vs_vers        = 3,
-       .vs_nproc       = 22,
+       .vs_nproc       = ARRAY_SIZE(nfsd_procedures3),
        .vs_proc        = nfsd_procedures3,
        .vs_dispatch    = nfsd_dispatch,
        .vs_count       = nfsd_count3,

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index dbaf333..9435136 100644 (file)
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -3596,12 +3596,13 @@ static const struct svc_procedure nfsd_procedures4[2] = {
        },
 };
 
-static unsigned int nfsd_count3[ARRAY_SIZE(nfsd_procedures4)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_count4[ARRAY_SIZE(nfsd_procedures4)]);
 const struct svc_version nfsd_version4 = {
        .vs_vers                = 4,
-       .vs_nproc               = 2,
+       .vs_nproc               = ARRAY_SIZE(nfsd_procedures4),
        .vs_proc                = nfsd_procedures4,
-       .vs_count               = nfsd_count3,
+       .vs_count               = nfsd_count4,
        .vs_dispatch            = nfsd_dispatch,
        .vs_xdrsize             = NFS4_SVC_XDRSIZE,
        .vs_rpcb_optnl          = true,

diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c
index 9744443..9e6069e 100644 (file)
--- a/fs/nfsd/nfsproc.c
+++ b/fs/nfsd/nfsproc.c
@@ -838,11 +838,11 @@ static const struct svc_procedure nfsd_procedures2[18] = {
        },
 };
 
-
-static unsigned int nfsd_count2[ARRAY_SIZE(nfsd_procedures2)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_count2[ARRAY_SIZE(nfsd_procedures2)]);
 const struct svc_version nfsd_version2 = {
        .vs_vers        = 2,
-       .vs_nproc       = 18,
+       .vs_nproc       = ARRAY_SIZE(nfsd_procedures2),
        .vs_proc        = nfsd_procedures2,
        .vs_count       = nfsd_count2,
        .vs_dispatch    = nfsd_dispatch,

diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h
index 70ce419..84de6b7 100644 (file)
--- a/include/linux/lockd/lockd.h
+++ b/include/linux/lockd/lockd.h
@@ -196,9 +196,9 @@ struct nlm_block {
  * Global variables
  */
 extern const struct rpc_program        nlm_program;
-extern const struct svc_procedure nlmsvc_procedures[];
+extern const struct svc_procedure nlmsvc_procedures[24];
 #ifdef CONFIG_LOCKD_V4
-extern const struct svc_procedure nlmsvc_procedures4[];
+extern const struct svc_procedure nlmsvc_procedures4[24];
 #endif
 extern int                     nlmsvc_grace_period;
 extern unsigned long           nlmsvc_timeout;

diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 392d2d2..1b078c9 100644 (file)
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -377,7 +377,7 @@ struct svc_version {
        u32                     vs_vers;        /* version number */
        u32                     vs_nproc;       /* number of procedures */
        const struct svc_procedure *vs_proc;    /* per-procedure info */
-       unsigned int            *vs_count;      /* call counts */
+       unsigned long __percpu  *vs_count;      /* call counts */
        u32                     vs_xdrsize;     /* xdrsize needed for this version */
 
        /* Don't register with rpcbind */

diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
index 52908f9..65fc129 100644 (file)
--- a/net/sunrpc/stats.c
+++ b/net/sunrpc/stats.c
@@ -83,7 +83,8 @@ void svc_seq_show(struct seq_file *seq, const struct svc_stat *statp)
 {
        const struct svc_program *prog = statp->program;
        const struct svc_version *vers;
-       unsigned int i, j;
+       unsigned int i, j, k;
+       unsigned long count;
 
        seq_printf(seq,
                "net %u %u %u %u\n",
@@ -104,8 +105,12 @@ void svc_seq_show(struct seq_file *seq, const struct svc_stat *statp)
                if (!vers)
                        continue;
                seq_printf(seq, "proc%d %u", i, vers->vs_nproc);
-               for (j = 0; j < vers->vs_nproc; j++)
-                       seq_printf(seq, " %u", vers->vs_count[j]);
+               for (j = 0; j < vers->vs_nproc; j++) {
+                       count = 0;
+                       for_each_possible_cpu(k)
+                               count += per_cpu(vers->vs_count[j], k);
+                       seq_printf(seq, " %lu", count);
+               }
                seq_putc(seq, '\n');
        }
 }

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 7b208e0..5dc298c 100644 (file)
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -1208,7 +1208,7 @@ svc_generic_init_request(struct svc_rqst *rqstp,
        memset(rqstp->rq_resp, 0, procp->pc_ressize);
 
        /* Bump per-procedure stats counter */
-       versp->vs_count[rqstp->rq_proc]++;
+       this_cpu_inc(versp->vs_count[rqstp->rq_proc]);
 
        ret->dispatch = versp->vs_dispatch;
        return rpc_success;