arch/arm64/include/asm/cmpxchg.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Based on arch/arm/include/asm/cmpxchg.h
   4  *
   5  * Copyright (C) 2012 ARM Ltd.
   6  */
   7 #ifndef __ASM_CMPXCHG_H
   8 #define __ASM_CMPXCHG_H
   9
  10 #include <linux/build_bug.h>
  11 #include <linux/compiler.h>
  12
  13 #include <asm/barrier.h>
  14 #include <asm/lse.h>
  15
  16 /*
  17  * We need separate acquire parameters for ll/sc and lse, since the full
  18  * barrier case is generated as release+dmb for the former and
  19  * acquire+release for the latter.
  20  */
  21 #define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl)       \
  22 static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr)         \
  23 {                                                                               \
  24         u##sz ret;                                                              \
  25         unsigned long tmp;                                                      \
  26                                                                                 \
  27         asm volatile(ARM64_LSE_ATOMIC_INSN(                                     \
  28         /* LL/SC */                                                             \
  29         "       prfm    pstl1strm, %2\n"                                        \
  30         "1:     ld" #acq "xr" #sfx "\t%" #w "0, %2\n"                           \
  31         "       st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n"                      \
  32         "       cbnz    %w1, 1b\n"                                              \
  33         "       " #mb,                                                          \
  34         /* LSE atomics */                                                       \
  35         "       swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n"            \
  36                 __nops(3)                                                       \
  37         "       " #nop_lse)                                                     \
  38         : "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr)                        \
  39         : "r" (x)                                                               \
  40         : cl);                                                                  \
  41                                                                                 \
  42         return ret;                                                             \
  43 }
  44
  45 __XCHG_CASE(w, b,     ,  8,        ,    ,  ,  ,  ,         )
  46 __XCHG_CASE(w, h,     , 16,        ,    ,  ,  ,  ,         )
  47 __XCHG_CASE(w,  ,     , 32,        ,    ,  ,  ,  ,         )
  48 __XCHG_CASE( ,  ,     , 64,        ,    ,  ,  ,  ,         )
  49 __XCHG_CASE(w, b, acq_,  8,        ,    , a, a,  , "memory")
  50 __XCHG_CASE(w, h, acq_, 16,        ,    , a, a,  , "memory")
  51 __XCHG_CASE(w,  , acq_, 32,        ,    , a, a,  , "memory")
  52 __XCHG_CASE( ,  , acq_, 64,        ,    , a, a,  , "memory")
  53 __XCHG_CASE(w, b, rel_,  8,        ,    ,  ,  , l, "memory")
  54 __XCHG_CASE(w, h, rel_, 16,        ,    ,  ,  , l, "memory")
  55 __XCHG_CASE(w,  , rel_, 32,        ,    ,  ,  , l, "memory")
  56 __XCHG_CASE( ,  , rel_, 64,        ,    ,  ,  , l, "memory")
  57 __XCHG_CASE(w, b,  mb_,  8, dmb ish, nop,  , a, l, "memory")
  58 __XCHG_CASE(w, h,  mb_, 16, dmb ish, nop,  , a, l, "memory")
  59 __XCHG_CASE(w,  ,  mb_, 32, dmb ish, nop,  , a, l, "memory")
  60 __XCHG_CASE( ,  ,  mb_, 64, dmb ish, nop,  , a, l, "memory")
  61
  62 #undef __XCHG_CASE
  63
  64 #define __XCHG_GEN(sfx)                                                 \
  65 static __always_inline  unsigned long __xchg##sfx(unsigned long x,      \
  66                                         volatile void *ptr,             \
  67                                         int size)                       \
  68 {                                                                       \
  69         switch (size) {                                                 \
  70         case 1:                                                         \
  71                 return __xchg_case##sfx##_8(x, ptr);                    \
  72         case 2:                                                         \
  73                 return __xchg_case##sfx##_16(x, ptr);                   \
  74         case 4:                                                         \
  75                 return __xchg_case##sfx##_32(x, ptr);                   \
  76         case 8:                                                         \
  77                 return __xchg_case##sfx##_64(x, ptr);                   \
  78         default:                                                        \
  79                 BUILD_BUG();                                            \
  80         }                                                               \
  81                                                                         \
  82         unreachable();                                                  \
  83 }
  84
  85 __XCHG_GEN()
  86 __XCHG_GEN(_acq)
  87 __XCHG_GEN(_rel)
  88 __XCHG_GEN(_mb)
  89
  90 #undef __XCHG_GEN
  91
  92 #define __xchg_wrapper(sfx, ptr, x)                                     \
  93 ({                                                                      \
  94         __typeof__(*(ptr)) __ret;                                       \
  95         __ret = (__typeof__(*(ptr)))                                    \
  96                 __xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
  97         __ret;                                                          \
  98 })
  99
 100 /* xchg */
 101 #define arch_xchg_relaxed(...)  __xchg_wrapper(    , __VA_ARGS__)
 102 #define arch_xchg_acquire(...)  __xchg_wrapper(_acq, __VA_ARGS__)
 103 #define arch_xchg_release(...)  __xchg_wrapper(_rel, __VA_ARGS__)
 104 #define arch_xchg(...)          __xchg_wrapper( _mb, __VA_ARGS__)
 105
 106 #define __CMPXCHG_CASE(name, sz)                        \
 107 static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr,       \
 108                                               u##sz old,                \
 109                                               u##sz new)                \
 110 {                                                                       \
 111         return __lse_ll_sc_body(_cmpxchg_case_##name##sz,               \
 112                                 ptr, old, new);                         \
 113 }
 114
 115 __CMPXCHG_CASE(    ,  8)
 116 __CMPXCHG_CASE(    , 16)
 117 __CMPXCHG_CASE(    , 32)
 118 __CMPXCHG_CASE(    , 64)
 119 __CMPXCHG_CASE(acq_,  8)
 120 __CMPXCHG_CASE(acq_, 16)
 121 __CMPXCHG_CASE(acq_, 32)
 122 __CMPXCHG_CASE(acq_, 64)
 123 __CMPXCHG_CASE(rel_,  8)
 124 __CMPXCHG_CASE(rel_, 16)
 125 __CMPXCHG_CASE(rel_, 32)
 126 __CMPXCHG_CASE(rel_, 64)
 127 __CMPXCHG_CASE(mb_,  8)
 128 __CMPXCHG_CASE(mb_, 16)
 129 __CMPXCHG_CASE(mb_, 32)
 130 __CMPXCHG_CASE(mb_, 64)
 131
 132 #undef __CMPXCHG_CASE
 133
 134 #define __CMPXCHG_DBL(name)                                             \
 135 static inline long __cmpxchg_double##name(unsigned long old1,           \
 136                                          unsigned long old2,            \
 137                                          unsigned long new1,            \
 138                                          unsigned long new2,            \
 139                                          volatile void *ptr)            \
 140 {                                                                       \
 141         return __lse_ll_sc_body(_cmpxchg_double##name,                  \
 142                                 old1, old2, new1, new2, ptr);           \
 143 }
 144
 145 __CMPXCHG_DBL(   )
 146 __CMPXCHG_DBL(_mb)
 147
 148 #undef __CMPXCHG_DBL
 149
 150 #define __CMPXCHG_GEN(sfx)                                              \
 151 static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
 152                                            unsigned long old,           \
 153                                            unsigned long new,           \
 154                                            int size)                    \
 155 {                                                                       \
 156         switch (size) {                                                 \
 157         case 1:                                                         \
 158                 return __cmpxchg_case##sfx##_8(ptr, old, new);          \
 159         case 2:                                                         \
 160                 return __cmpxchg_case##sfx##_16(ptr, old, new);         \
 161         case 4:                                                         \
 162                 return __cmpxchg_case##sfx##_32(ptr, old, new);         \
 163         case 8:                                                         \
 164                 return __cmpxchg_case##sfx##_64(ptr, old, new);         \
 165         default:                                                        \
 166                 BUILD_BUG();                                            \
 167         }                                                               \
 168                                                                         \
 169         unreachable();                                                  \
 170 }
 171
 172 __CMPXCHG_GEN()
 173 __CMPXCHG_GEN(_acq)
 174 __CMPXCHG_GEN(_rel)
 175 __CMPXCHG_GEN(_mb)
 176
 177 #undef __CMPXCHG_GEN
 178
 179 #define __cmpxchg_wrapper(sfx, ptr, o, n)                               \
 180 ({                                                                      \
 181         __typeof__(*(ptr)) __ret;                                       \
 182         __ret = (__typeof__(*(ptr)))                                    \
 183                 __cmpxchg##sfx((ptr), (unsigned long)(o),               \
 184                                 (unsigned long)(n), sizeof(*(ptr)));    \
 185         __ret;                                                          \
 186 })
 187
 188 /* cmpxchg */
 189 #define arch_cmpxchg_relaxed(...)       __cmpxchg_wrapper(    , __VA_ARGS__)
 190 #define arch_cmpxchg_acquire(...)       __cmpxchg_wrapper(_acq, __VA_ARGS__)
 191 #define arch_cmpxchg_release(...)       __cmpxchg_wrapper(_rel, __VA_ARGS__)
 192 #define arch_cmpxchg(...)               __cmpxchg_wrapper( _mb, __VA_ARGS__)
 193 #define arch_cmpxchg_local              arch_cmpxchg_relaxed
 194
 195 /* cmpxchg64 */
 196 #define arch_cmpxchg64_relaxed          arch_cmpxchg_relaxed
 197 #define arch_cmpxchg64_acquire          arch_cmpxchg_acquire
 198 #define arch_cmpxchg64_release          arch_cmpxchg_release
 199 #define arch_cmpxchg64                  arch_cmpxchg
 200 #define arch_cmpxchg64_local            arch_cmpxchg_local
 201
 202 /* cmpxchg_double */
 203 #define system_has_cmpxchg_double()     1
 204
 205 #define __cmpxchg_double_check(ptr1, ptr2)                                      \
 206 ({                                                                              \
 207         if (sizeof(*(ptr1)) != 8)                                               \
 208                 BUILD_BUG();                                                    \
 209         VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1);      \
 210 })
 211
 212 #define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2)                         \
 213 ({                                                                              \
 214         int __ret;                                                              \
 215         __cmpxchg_double_check(ptr1, ptr2);                                     \
 216         __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2),  \
 217                                      (unsigned long)(n1), (unsigned long)(n2),  \
 218                                      ptr1);                                     \
 219         __ret;                                                                  \
 220 })
 221
 222 #define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2)                   \
 223 ({                                                                              \
 224         int __ret;                                                              \
 225         __cmpxchg_double_check(ptr1, ptr2);                                     \
 226         __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2),     \
 227                                   (unsigned long)(n1), (unsigned long)(n2),     \
 228                                   ptr1);                                        \
 229         __ret;                                                                  \
 230 })
 231
 232 #define __CMPWAIT_CASE(w, sfx, sz)                                      \
 233 static inline void __cmpwait_case_##sz(volatile void *ptr,              \
 234                                        unsigned long val)               \
 235 {                                                                       \
 236         unsigned long tmp;                                              \
 237                                                                         \
 238         asm volatile(                                                   \
 239         "       sevl\n"                                                 \
 240         "       wfe\n"                                                  \
 241         "       ldxr" #sfx "\t%" #w "[tmp], %[v]\n"                     \
 242         "       eor     %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n"     \
 243         "       cbnz    %" #w "[tmp], 1f\n"                             \
 244         "       wfe\n"                                                  \
 245         "1:"                                                            \
 246         : [tmp] "=&r" (tmp), [v] "+Q" (*(unsigned long *)ptr)           \
 247         : [val] "r" (val));                                             \
 248 }
 249
 250 __CMPWAIT_CASE(w, b, 8);
 251 __CMPWAIT_CASE(w, h, 16);
 252 __CMPWAIT_CASE(w,  , 32);
 253 __CMPWAIT_CASE( ,  , 64);
 254
 255 #undef __CMPWAIT_CASE
 256
 257 #define __CMPWAIT_GEN(sfx)                                              \
 258 static __always_inline void __cmpwait##sfx(volatile void *ptr,          \
 259                                   unsigned long val,                    \
 260                                   int size)                             \
 261 {                                                                       \
 262         switch (size) {                                                 \
 263         case 1:                                                         \
 264                 return __cmpwait_case##sfx##_8(ptr, (u8)val);           \
 265         case 2:                                                         \
 266                 return __cmpwait_case##sfx##_16(ptr, (u16)val);         \
 267         case 4:                                                         \
 268                 return __cmpwait_case##sfx##_32(ptr, val);              \
 269         case 8:                                                         \
 270                 return __cmpwait_case##sfx##_64(ptr, val);              \
 271         default:                                                        \
 272                 BUILD_BUG();                                            \
 273         }                                                               \
 274                                                                         \
 275         unreachable();                                                  \
 276 }
 277
 278 __CMPWAIT_GEN()
 279
 280 #undef __CMPWAIT_GEN
 281
 282 #define __cmpwait_relaxed(ptr, val) \
 283         __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))
 284
 285 #endif  /* __ASM_CMPXCHG_H */