Merge tag 'sched-core-2021-06-28' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-microblaze.git] / lib / atomic64.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Generic implementation of 64-bit atomics using spinlocks,
4  * useful on processors that don't have 64-bit atomic instructions.
5  *
6  * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7  */
8 #include <linux/types.h>
9 #include <linux/cache.h>
10 #include <linux/spinlock.h>
11 #include <linux/init.h>
12 #include <linux/export.h>
13 #include <linux/atomic.h>
14
15 /*
16  * We use a hashed array of spinlocks to provide exclusive access
17  * to each atomic64_t variable.  Since this is expected to used on
18  * systems with small numbers of CPUs (<= 4 or so), we use a
19  * relatively small array of 16 spinlocks to avoid wasting too much
20  * memory on the spinlock array.
21  */
22 #define NR_LOCKS        16
23
24 /*
25  * Ensure each lock is in a separate cacheline.
26  */
27 static union {
28         raw_spinlock_t lock;
29         char pad[L1_CACHE_BYTES];
30 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
31         [0 ... (NR_LOCKS - 1)] = {
32                 .lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
33         },
34 };
35
36 static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
37 {
38         unsigned long addr = (unsigned long) v;
39
40         addr >>= L1_CACHE_SHIFT;
41         addr ^= (addr >> 8) ^ (addr >> 16);
42         return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
43 }
44
45 s64 generic_atomic64_read(const atomic64_t *v)
46 {
47         unsigned long flags;
48         raw_spinlock_t *lock = lock_addr(v);
49         s64 val;
50
51         raw_spin_lock_irqsave(lock, flags);
52         val = v->counter;
53         raw_spin_unlock_irqrestore(lock, flags);
54         return val;
55 }
56 EXPORT_SYMBOL(generic_atomic64_read);
57
58 void generic_atomic64_set(atomic64_t *v, s64 i)
59 {
60         unsigned long flags;
61         raw_spinlock_t *lock = lock_addr(v);
62
63         raw_spin_lock_irqsave(lock, flags);
64         v->counter = i;
65         raw_spin_unlock_irqrestore(lock, flags);
66 }
67 EXPORT_SYMBOL(generic_atomic64_set);
68
69 #define ATOMIC64_OP(op, c_op)                                           \
70 void generic_atomic64_##op(s64 a, atomic64_t *v)                        \
71 {                                                                       \
72         unsigned long flags;                                            \
73         raw_spinlock_t *lock = lock_addr(v);                            \
74                                                                         \
75         raw_spin_lock_irqsave(lock, flags);                             \
76         v->counter c_op a;                                              \
77         raw_spin_unlock_irqrestore(lock, flags);                        \
78 }                                                                       \
79 EXPORT_SYMBOL(generic_atomic64_##op);
80
81 #define ATOMIC64_OP_RETURN(op, c_op)                                    \
82 s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v)                \
83 {                                                                       \
84         unsigned long flags;                                            \
85         raw_spinlock_t *lock = lock_addr(v);                            \
86         s64 val;                                                        \
87                                                                         \
88         raw_spin_lock_irqsave(lock, flags);                             \
89         val = (v->counter c_op a);                                      \
90         raw_spin_unlock_irqrestore(lock, flags);                        \
91         return val;                                                     \
92 }                                                                       \
93 EXPORT_SYMBOL(generic_atomic64_##op##_return);
94
95 #define ATOMIC64_FETCH_OP(op, c_op)                                     \
96 s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v)                   \
97 {                                                                       \
98         unsigned long flags;                                            \
99         raw_spinlock_t *lock = lock_addr(v);                            \
100         s64 val;                                                        \
101                                                                         \
102         raw_spin_lock_irqsave(lock, flags);                             \
103         val = v->counter;                                               \
104         v->counter c_op a;                                              \
105         raw_spin_unlock_irqrestore(lock, flags);                        \
106         return val;                                                     \
107 }                                                                       \
108 EXPORT_SYMBOL(generic_atomic64_fetch_##op);
109
110 #define ATOMIC64_OPS(op, c_op)                                          \
111         ATOMIC64_OP(op, c_op)                                           \
112         ATOMIC64_OP_RETURN(op, c_op)                                    \
113         ATOMIC64_FETCH_OP(op, c_op)
114
115 ATOMIC64_OPS(add, +=)
116 ATOMIC64_OPS(sub, -=)
117
118 #undef ATOMIC64_OPS
119 #define ATOMIC64_OPS(op, c_op)                                          \
120         ATOMIC64_OP(op, c_op)                                           \
121         ATOMIC64_OP_RETURN(op, c_op)                                    \
122         ATOMIC64_FETCH_OP(op, c_op)
123
124 ATOMIC64_OPS(and, &=)
125 ATOMIC64_OPS(or, |=)
126 ATOMIC64_OPS(xor, ^=)
127
128 #undef ATOMIC64_OPS
129 #undef ATOMIC64_FETCH_OP
130 #undef ATOMIC64_OP_RETURN
131 #undef ATOMIC64_OP
132
133 s64 generic_atomic64_dec_if_positive(atomic64_t *v)
134 {
135         unsigned long flags;
136         raw_spinlock_t *lock = lock_addr(v);
137         s64 val;
138
139         raw_spin_lock_irqsave(lock, flags);
140         val = v->counter - 1;
141         if (val >= 0)
142                 v->counter = val;
143         raw_spin_unlock_irqrestore(lock, flags);
144         return val;
145 }
146 EXPORT_SYMBOL(generic_atomic64_dec_if_positive);
147
148 s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
149 {
150         unsigned long flags;
151         raw_spinlock_t *lock = lock_addr(v);
152         s64 val;
153
154         raw_spin_lock_irqsave(lock, flags);
155         val = v->counter;
156         if (val == o)
157                 v->counter = n;
158         raw_spin_unlock_irqrestore(lock, flags);
159         return val;
160 }
161 EXPORT_SYMBOL(generic_atomic64_cmpxchg);
162
163 s64 generic_atomic64_xchg(atomic64_t *v, s64 new)
164 {
165         unsigned long flags;
166         raw_spinlock_t *lock = lock_addr(v);
167         s64 val;
168
169         raw_spin_lock_irqsave(lock, flags);
170         val = v->counter;
171         v->counter = new;
172         raw_spin_unlock_irqrestore(lock, flags);
173         return val;
174 }
175 EXPORT_SYMBOL(generic_atomic64_xchg);
176
177 s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
178 {
179         unsigned long flags;
180         raw_spinlock_t *lock = lock_addr(v);
181         s64 val;
182
183         raw_spin_lock_irqsave(lock, flags);
184         val = v->counter;
185         if (val != u)
186                 v->counter += a;
187         raw_spin_unlock_irqrestore(lock, flags);
188
189         return val;
190 }
191 EXPORT_SYMBOL(generic_atomic64_fetch_add_unless);