1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_U64_STATS_SYNC_H
3 #define _LINUX_U64_STATS_SYNC_H
6 * Protect against 64-bit values tearing on 32-bit architectures. This is
7 * typically used for statistics read/update in different subsystems.
11 * - Use a seqcount on 32-bit SMP, only disable preemption for 32-bit UP.
12 * - The whole thing is a no-op on 64-bit architectures.
16 * 1) Write side must ensure mutual exclusion, or one seqcount update could
17 * be lost, thus blocking readers forever.
19 * 2) Write side must disable preemption, or a seqcount reader can preempt the
20 * writer and also spin forever.
22 * 3) Write side must use the _irqsave() variant if other writers, or a reader,
23 * can be invoked from an IRQ context.
25 * 4) If reader fetches several counters, there is no guarantee the whole values
26 * are consistent w.r.t. each other (remember point #2: seqcounts are not
27 * used for 64bit architectures).
29 * 5) Readers are allowed to sleep or be preempted/interrupted: they perform
32 * 6) Readers must use both u64_stats_fetch_{begin,retry}_irq() if the stats
33 * might be updated from a hardirq or softirq context (remember point #1:
34 * seqcounts are not used for UP kernels). 32-bit UP stat readers could read
35 * corrupted 64-bit values otherwise.
39 * Stats producer (writer) should use following template granted it already got
40 * an exclusive access to counters (a lock is already taken, or per cpu
41 * data is used [in a non preemptable context])
43 * spin_lock_bh(...) or other synchronization to get exclusive access
45 * u64_stats_update_begin(&stats->syncp);
46 * u64_stats_add(&stats->bytes64, len); // non atomic operation
47 * u64_stats_inc(&stats->packets64); // non atomic operation
48 * u64_stats_update_end(&stats->syncp);
50 * While a consumer (reader) should use following template to get consistent
51 * snapshot for each variable (but no guarantee on several ones)
53 * u64 tbytes, tpackets;
57 * start = u64_stats_fetch_begin(&stats->syncp);
58 * tbytes = u64_stats_read(&stats->bytes64); // non atomic operation
59 * tpackets = u64_stats_read(&stats->packets64); // non atomic operation
60 * } while (u64_stats_fetch_retry(&stats->syncp, start));
63 * Example of use in drivers/net/loopback.c, using per_cpu containers,
64 * in BH disabled context.
66 #include <linux/seqlock.h>
68 struct u64_stats_sync {
69 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
74 #if BITS_PER_LONG == 64
75 #include <asm/local64.h>
81 static inline u64 u64_stats_read(const u64_stats_t *p)
83 return local64_read(&p->v);
86 static inline void u64_stats_add(u64_stats_t *p, unsigned long val)
88 local64_add(val, &p->v);
91 static inline void u64_stats_inc(u64_stats_t *p)
102 static inline u64 u64_stats_read(const u64_stats_t *p)
107 static inline void u64_stats_add(u64_stats_t *p, unsigned long val)
112 static inline void u64_stats_inc(u64_stats_t *p)
118 static inline void u64_stats_init(struct u64_stats_sync *syncp)
120 #if BITS_PER_LONG == 32 && defined(CONFIG_SMP)
121 seqcount_init(&syncp->seq);
125 static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)
127 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
128 write_seqcount_begin(&syncp->seq);
132 static inline void u64_stats_update_end(struct u64_stats_sync *syncp)
134 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
135 write_seqcount_end(&syncp->seq);
139 static inline unsigned long
140 u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp)
142 unsigned long flags = 0;
144 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
145 local_irq_save(flags);
146 write_seqcount_begin(&syncp->seq);
152 u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,
155 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
156 write_seqcount_end(&syncp->seq);
157 local_irq_restore(flags);
161 static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
163 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
164 return read_seqcount_begin(&syncp->seq);
170 static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
172 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
175 return __u64_stats_fetch_begin(syncp);
178 static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
181 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
182 return read_seqcount_retry(&syncp->seq, start);
188 static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
191 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
194 return __u64_stats_fetch_retry(syncp, start);
198 * In case irq handlers can update u64 counters, readers can use following helpers
199 * - SMP 32bit arches use seqcount protection, irq safe.
200 * - UP 32bit must disable irqs.
201 * - 64bit have no problem atomically reading u64 values, irq safe.
203 static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp)
205 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
208 return __u64_stats_fetch_begin(syncp);
211 static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp,
214 #if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
217 return __u64_stats_fetch_retry(syncp, start);
220 #endif /* _LINUX_U64_STATS_SYNC_H */