arch/arm64/include/asm/barrier.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Based on arch/arm/include/asm/barrier.h
   4  *
   5  * Copyright (C) 2012 ARM Ltd.
   6  */
   7 #ifndef __ASM_BARRIER_H
   8 #define __ASM_BARRIER_H
   9
  10 #ifndef __ASSEMBLY__
  11
  12 #include <linux/kasan-checks.h>
  13
  14 #define __nops(n)       ".rept  " #n "\nnop\n.endr\n"
  15 #define nops(n)         asm volatile(__nops(n))
  16
  17 #define sev()           asm volatile("sev" : : : "memory")
  18 #define wfe()           asm volatile("wfe" : : : "memory")
  19 #define wfet(val)       asm volatile("msr s0_3_c1_c0_0, %0"     \
  20                                      : : "r" (val) : "memory")
  21 #define wfi()           asm volatile("wfi" : : : "memory")
  22 #define wfit(val)       asm volatile("msr s0_3_c1_c0_1, %0"     \
  23                                      : : "r" (val) : "memory")
  24
  25 #define isb()           asm volatile("isb" : : : "memory")
  26 #define dmb(opt)        asm volatile("dmb " #opt : : : "memory")
  27 #define dsb(opt)        asm volatile("dsb " #opt : : : "memory")
  28
  29 #define psb_csync()     asm volatile("hint #17" : : : "memory")
  30 #define __tsb_csync()   asm volatile("hint #18" : : : "memory")
  31 #define csdb()          asm volatile("hint #20" : : : "memory")
  32
  33 /*
  34  * Data Gathering Hint:
  35  * This instruction prevents merging memory accesses with Normal-NC or
  36  * Device-GRE attributes before the hint instruction with any memory accesses
  37  * appearing after the hint instruction.
  38  */
  39 #define dgh()           asm volatile("hint #6" : : : "memory")
  40
  41 #ifdef CONFIG_ARM64_PSEUDO_NMI
  42 #define pmr_sync()                                              \
  43         do {                                                    \
  44                 extern struct static_key_false gic_pmr_sync;    \
  45                                                                 \
  46                 if (static_branch_unlikely(&gic_pmr_sync))      \
  47                         dsb(sy);                                \
  48         } while(0)
  49 #else
  50 #define pmr_sync()      do {} while (0)
  51 #endif
  52
  53 #define mb()            dsb(sy)
  54 #define rmb()           dsb(ld)
  55 #define wmb()           dsb(st)
  56
  57 #define dma_mb()        dmb(osh)
  58 #define dma_rmb()       dmb(oshld)
  59 #define dma_wmb()       dmb(oshst)
  60
  61 #define io_stop_wc()    dgh()
  62
  63 #define tsb_csync()                                                             \
  64         do {                                                                    \
  65                 /*                                                              \
  66                  * CPUs affected by Arm Erratum 2054223 or 2067961 needs        \
  67                  * another TSB to ensure the trace is flushed. The barriers     \
  68                  * don't have to be strictly back to back, as long as the       \
  69                  * CPU is in trace prohibited state.                            \
  70                  */                                                             \
  71                 if (cpus_have_final_cap(ARM64_WORKAROUND_TSB_FLUSH_FAILURE))    \
  72                         __tsb_csync();                                          \
  73                 __tsb_csync();                                                  \
  74         } while (0)
  75
  76 /*
  77  * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
  78  * and 0 otherwise.
  79  */
  80 #define array_index_mask_nospec array_index_mask_nospec
  81 static inline unsigned long array_index_mask_nospec(unsigned long idx,
  82                                                     unsigned long sz)
  83 {
  84         unsigned long mask;
  85
  86         asm volatile(
  87         "       cmp     %1, %2\n"
  88         "       sbc     %0, xzr, xzr\n"
  89         : "=r" (mask)
  90         : "r" (idx), "Ir" (sz)
  91         : "cc");
  92
  93         csdb();
  94         return mask;
  95 }
  96
  97 /*
  98  * Ensure that reads of the counter are treated the same as memory reads
  99  * for the purposes of ordering by subsequent memory barriers.
 100  *
 101  * This insanity brought to you by speculative system register reads,
 102  * out-of-order memory accesses, sequence locks and Thomas Gleixner.
 103  *
 104  * https://lore.kernel.org/r/alpine.DEB.2.21.1902081950260.1662@nanos.tec.linutronix.de/
 105  */
 106 #define arch_counter_enforce_ordering(val) do {                         \
 107         u64 tmp, _val = (val);                                          \
 108                                                                         \
 109         asm volatile(                                                   \
 110         "       eor     %0, %1, %1\n"                                   \
 111         "       add     %0, sp, %0\n"                                   \
 112         "       ldr     xzr, [%0]"                                      \
 113         : "=r" (tmp) : "r" (_val));                                     \
 114 } while (0)
 115
 116 #define __smp_mb()      dmb(ish)
 117 #define __smp_rmb()     dmb(ishld)
 118 #define __smp_wmb()     dmb(ishst)
 119
 120 #define __smp_store_release(p, v)                                       \
 121 do {                                                                    \
 122         typeof(p) __p = (p);                                            \
 123         union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u =  \
 124                 { .__val = (__force __unqual_scalar_typeof(*p)) (v) };  \
 125         compiletime_assert_atomic_type(*p);                             \
 126         kasan_check_write(__p, sizeof(*p));                             \
 127         switch (sizeof(*p)) {                                           \
 128         case 1:                                                         \
 129                 asm volatile ("stlrb %w1, %0"                           \
 130                                 : "=Q" (*__p)                           \
 131                                 : "r" (*(__u8 *)__u.__c)                \
 132                                 : "memory");                            \
 133                 break;                                                  \
 134         case 2:                                                         \
 135                 asm volatile ("stlrh %w1, %0"                           \
 136                                 : "=Q" (*__p)                           \
 137                                 : "r" (*(__u16 *)__u.__c)               \
 138                                 : "memory");                            \
 139                 break;                                                  \
 140         case 4:                                                         \
 141                 asm volatile ("stlr %w1, %0"                            \
 142                                 : "=Q" (*__p)                           \
 143                                 : "r" (*(__u32 *)__u.__c)               \
 144                                 : "memory");                            \
 145                 break;                                                  \
 146         case 8:                                                         \
 147                 asm volatile ("stlr %1, %0"                             \
 148                                 : "=Q" (*__p)                           \
 149                                 : "r" (*(__u64 *)__u.__c)               \
 150                                 : "memory");                            \
 151                 break;                                                  \
 152         }                                                               \
 153 } while (0)
 154
 155 #define __smp_load_acquire(p)                                           \
 156 ({                                                                      \
 157         union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u;   \
 158         typeof(p) __p = (p);                                            \
 159         compiletime_assert_atomic_type(*p);                             \
 160         kasan_check_read(__p, sizeof(*p));                              \
 161         switch (sizeof(*p)) {                                           \
 162         case 1:                                                         \
 163                 asm volatile ("ldarb %w0, %1"                           \
 164                         : "=r" (*(__u8 *)__u.__c)                       \
 165                         : "Q" (*__p) : "memory");                       \
 166                 break;                                                  \
 167         case 2:                                                         \
 168                 asm volatile ("ldarh %w0, %1"                           \
 169                         : "=r" (*(__u16 *)__u.__c)                      \
 170                         : "Q" (*__p) : "memory");                       \
 171                 break;                                                  \
 172         case 4:                                                         \
 173                 asm volatile ("ldar %w0, %1"                            \
 174                         : "=r" (*(__u32 *)__u.__c)                      \
 175                         : "Q" (*__p) : "memory");                       \
 176                 break;                                                  \
 177         case 8:                                                         \
 178                 asm volatile ("ldar %0, %1"                             \
 179                         : "=r" (*(__u64 *)__u.__c)                      \
 180                         : "Q" (*__p) : "memory");                       \
 181                 break;                                                  \
 182         }                                                               \
 183         (typeof(*p))__u.__val;                                          \
 184 })
 185
 186 #define smp_cond_load_relaxed(ptr, cond_expr)                           \
 187 ({                                                                      \
 188         typeof(ptr) __PTR = (ptr);                                      \
 189         __unqual_scalar_typeof(*ptr) VAL;                               \
 190         for (;;) {                                                      \
 191                 VAL = READ_ONCE(*__PTR);                                \
 192                 if (cond_expr)                                          \
 193                         break;                                          \
 194                 __cmpwait_relaxed(__PTR, VAL);                          \
 195         }                                                               \
 196         (typeof(*ptr))VAL;                                              \
 197 })
 198
 199 #define smp_cond_load_acquire(ptr, cond_expr)                           \
 200 ({                                                                      \
 201         typeof(ptr) __PTR = (ptr);                                      \
 202         __unqual_scalar_typeof(*ptr) VAL;                               \
 203         for (;;) {                                                      \
 204                 VAL = smp_load_acquire(__PTR);                          \
 205                 if (cond_expr)                                          \
 206                         break;                                          \
 207                 __cmpwait_relaxed(__PTR, VAL);                          \
 208         }                                                               \
 209         (typeof(*ptr))VAL;                                              \
 210 })
 211
 212 #include <asm-generic/barrier.h>
 213
 214 #endif  /* __ASSEMBLY__ */
 215
 216 #endif  /* __ASM_BARRIER_H */