1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * Generic barrier definitions.
5 * It should be possible to use these on really simple architectures,
6 * but it serves more as a starting point for new ports.
8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
9 * Written by David Howells (dhowells@redhat.com)
11 #ifndef __ASM_GENERIC_BARRIER_H
12 #define __ASM_GENERIC_BARRIER_H
16 #include <asm/rwonce.h>
19 #define nop() asm volatile ("nop")
23 * Force strict CPU ordering. And yes, this is required on UP too when we're
26 * Fall back to compiler barriers if nothing better is provided.
30 #define mb() barrier()
42 #define dma_rmb() rmb()
46 #define dma_wmb() wmb()
50 #define __smp_mb() mb()
54 #define __smp_rmb() rmb()
58 #define __smp_wmb() wmb()
64 #define smp_mb() __smp_mb()
68 #define smp_rmb() __smp_rmb()
72 #define smp_wmb() __smp_wmb()
75 #else /* !CONFIG_SMP */
78 #define smp_mb() barrier()
82 #define smp_rmb() barrier()
86 #define smp_wmb() barrier()
89 #endif /* CONFIG_SMP */
91 #ifndef __smp_store_mb
92 #define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
95 #ifndef __smp_mb__before_atomic
96 #define __smp_mb__before_atomic() __smp_mb()
99 #ifndef __smp_mb__after_atomic
100 #define __smp_mb__after_atomic() __smp_mb()
103 #ifndef __smp_store_release
104 #define __smp_store_release(p, v) \
106 compiletime_assert_atomic_type(*p); \
112 #ifndef __smp_load_acquire
113 #define __smp_load_acquire(p) \
115 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
116 compiletime_assert_atomic_type(*p); \
125 #define smp_store_mb(var, value) __smp_store_mb(var, value)
128 #ifndef smp_mb__before_atomic
129 #define smp_mb__before_atomic() __smp_mb__before_atomic()
132 #ifndef smp_mb__after_atomic
133 #define smp_mb__after_atomic() __smp_mb__after_atomic()
136 #ifndef smp_store_release
137 #define smp_store_release(p, v) __smp_store_release(p, v)
140 #ifndef smp_load_acquire
141 #define smp_load_acquire(p) __smp_load_acquire(p)
144 #else /* !CONFIG_SMP */
147 #define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
150 #ifndef smp_mb__before_atomic
151 #define smp_mb__before_atomic() barrier()
154 #ifndef smp_mb__after_atomic
155 #define smp_mb__after_atomic() barrier()
158 #ifndef smp_store_release
159 #define smp_store_release(p, v) \
161 compiletime_assert_atomic_type(*p); \
167 #ifndef smp_load_acquire
168 #define smp_load_acquire(p) \
170 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
171 compiletime_assert_atomic_type(*p); \
177 #endif /* CONFIG_SMP */
179 /* Barriers for virtual machine guests when talking to an SMP host */
180 #define virt_mb() __smp_mb()
181 #define virt_rmb() __smp_rmb()
182 #define virt_wmb() __smp_wmb()
183 #define virt_store_mb(var, value) __smp_store_mb(var, value)
184 #define virt_mb__before_atomic() __smp_mb__before_atomic()
185 #define virt_mb__after_atomic() __smp_mb__after_atomic()
186 #define virt_store_release(p, v) __smp_store_release(p, v)
187 #define virt_load_acquire(p) __smp_load_acquire(p)
190 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
192 * A control dependency provides a LOAD->STORE order, the additional RMB
193 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
194 * aka. (load)-ACQUIRE.
196 * Architectures that do not do load speculation can have this be barrier().
198 #ifndef smp_acquire__after_ctrl_dep
199 #define smp_acquire__after_ctrl_dep() smp_rmb()
203 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
204 * @ptr: pointer to the variable to wait on
205 * @cond: boolean expression to wait for
207 * Equivalent to using READ_ONCE() on the condition variable.
209 * Due to C lacking lambda expressions we load the value of *ptr into a
210 * pre-named variable @VAL to be used in @cond.
212 #ifndef smp_cond_load_relaxed
213 #define smp_cond_load_relaxed(ptr, cond_expr) ({ \
214 typeof(ptr) __PTR = (ptr); \
215 __unqual_scalar_typeof(*ptr) VAL; \
217 VAL = READ_ONCE(*__PTR); \
227 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
228 * @ptr: pointer to the variable to wait on
229 * @cond: boolean expression to wait for
231 * Equivalent to using smp_load_acquire() on the condition variable but employs
232 * the control dependency of the wait to reduce the barrier on many platforms.
234 #ifndef smp_cond_load_acquire
235 #define smp_cond_load_acquire(ptr, cond_expr) ({ \
236 __unqual_scalar_typeof(*ptr) _val; \
237 _val = smp_cond_load_relaxed(ptr, cond_expr); \
238 smp_acquire__after_ctrl_dep(); \
239 (typeof(*ptr))_val; \
244 * pmem_wmb() ensures that all stores for which the modification
245 * are written to persistent storage by preceding instructions have
246 * updated persistent storage before any data access or data transfer
247 * caused by subsequent instructions is initiated.
250 #define pmem_wmb() wmb()
253 #endif /* !__ASSEMBLY__ */
254 #endif /* __ASM_GENERIC_BARRIER_H */