arch/arm64/include/asm/atomic_ll_sc.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Based on arch/arm/include/asm/atomic.h
   4  *
   5  * Copyright (C) 1996 Russell King.
   6  * Copyright (C) 2002 Deep Blue Solutions Ltd.
   7  * Copyright (C) 2012 ARM Ltd.
   8  */
   9
  10 #ifndef __ASM_ATOMIC_LL_SC_H
  11 #define __ASM_ATOMIC_LL_SC_H
  12
  13 #include <linux/stringify.h>
  14
  15 #ifdef CONFIG_ARM64_LSE_ATOMICS
  16 #define __LL_SC_FALLBACK(asm_ops)                                       \
  17 "       b       3f\n"                                                   \
  18 "       .subsection     1\n"                                            \
  19 "3:\n"                                                                  \
  20 asm_ops "\n"                                                            \
  21 "       b       4f\n"                                                   \
  22 "       .previous\n"                                                    \
  23 "4:\n"
  24 #else
  25 #define __LL_SC_FALLBACK(asm_ops) asm_ops
  26 #endif
  27
  28 #ifndef CONFIG_CC_HAS_K_CONSTRAINT
  29 #define K
  30 #endif
  31
  32 /*
  33  * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
  34  * store exclusive to ensure that these are atomic.  We may loop
  35  * to ensure that the update happens.
  36  */
  37
  38 #define ATOMIC_OP(op, asm_op, constraint)                               \
  39 static inline void                                                      \
  40 __ll_sc_atomic_##op(int i, atomic_t *v)                                 \
  41 {                                                                       \
  42         unsigned long tmp;                                              \
  43         int result;                                                     \
  44                                                                         \
  45         asm volatile("// atomic_" #op "\n"                              \
  46         __LL_SC_FALLBACK(                                               \
  47 "       prfm    pstl1strm, %2\n"                                        \
  48 "1:     ldxr    %w0, %2\n"                                              \
  49 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  50 "       stxr    %w1, %w0, %2\n"                                         \
  51 "       cbnz    %w1, 1b\n")                                             \
  52         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  53         : __stringify(constraint) "r" (i));                             \
  54 }
  55
  56 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
  57 static inline int                                                       \
  58 __ll_sc_atomic_##op##_return##name(int i, atomic_t *v)                  \
  59 {                                                                       \
  60         unsigned long tmp;                                              \
  61         int result;                                                     \
  62                                                                         \
  63         asm volatile("// atomic_" #op "_return" #name "\n"              \
  64         __LL_SC_FALLBACK(                                               \
  65 "       prfm    pstl1strm, %2\n"                                        \
  66 "1:     ld" #acq "xr    %w0, %2\n"                                      \
  67 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  68 "       st" #rel "xr    %w1, %w0, %2\n"                                 \
  69 "       cbnz    %w1, 1b\n"                                              \
  70 "       " #mb )                                                         \
  71         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  72         : __stringify(constraint) "r" (i)                               \
  73         : cl);                                                          \
  74                                                                         \
  75         return result;                                                  \
  76 }
  77
  78 #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
  79 static inline int                                                       \
  80 __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)                     \
  81 {                                                                       \
  82         unsigned long tmp;                                              \
  83         int val, result;                                                \
  84                                                                         \
  85         asm volatile("// atomic_fetch_" #op #name "\n"                  \
  86         __LL_SC_FALLBACK(                                               \
  87 "       prfm    pstl1strm, %3\n"                                        \
  88 "1:     ld" #acq "xr    %w0, %3\n"                                      \
  89 "       " #asm_op "     %w1, %w0, %w4\n"                                \
  90 "       st" #rel "xr    %w2, %w1, %3\n"                                 \
  91 "       cbnz    %w2, 1b\n"                                              \
  92 "       " #mb )                                                         \
  93         : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)   \
  94         : __stringify(constraint) "r" (i)                               \
  95         : cl);                                                          \
  96                                                                         \
  97         return result;                                                  \
  98 }
  99
 100 #define ATOMIC_OPS(...)                                                 \
 101         ATOMIC_OP(__VA_ARGS__)                                          \
 102         ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\
 103         ATOMIC_OP_RETURN(_relaxed,        ,  ,  ,         , __VA_ARGS__)\
 104         ATOMIC_OP_RETURN(_acquire,        , a,  , "memory", __VA_ARGS__)\
 105         ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)\
 106         ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
 107         ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
 108         ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
 109         ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)
 110
 111 ATOMIC_OPS(add, add, I)
 112 ATOMIC_OPS(sub, sub, J)
 113
 114 #undef ATOMIC_OPS
 115 #define ATOMIC_OPS(...)                                                 \
 116         ATOMIC_OP(__VA_ARGS__)                                          \
 117         ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
 118         ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
 119         ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
 120         ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)
 121
 122 ATOMIC_OPS(and, and, K)
 123 ATOMIC_OPS(or, orr, K)
 124 ATOMIC_OPS(xor, eor, K)
 125 /*
 126  * GAS converts the mysterious and undocumented BIC (immediate) alias to
 127  * an AND (immediate) instruction with the immediate inverted. We don't
 128  * have a constraint for this, so fall back to register.
 129  */
 130 ATOMIC_OPS(andnot, bic, )
 131
 132 #undef ATOMIC_OPS
 133 #undef ATOMIC_FETCH_OP
 134 #undef ATOMIC_OP_RETURN
 135 #undef ATOMIC_OP
 136
 137 #define ATOMIC64_OP(op, asm_op, constraint)                             \
 138 static inline void                                                      \
 139 __ll_sc_atomic64_##op(s64 i, atomic64_t *v)                             \
 140 {                                                                       \
 141         s64 result;                                                     \
 142         unsigned long tmp;                                              \
 143                                                                         \
 144         asm volatile("// atomic64_" #op "\n"                            \
 145         __LL_SC_FALLBACK(                                               \
 146 "       prfm    pstl1strm, %2\n"                                        \
 147 "1:     ldxr    %0, %2\n"                                               \
 148 "       " #asm_op "     %0, %0, %3\n"                                   \
 149 "       stxr    %w1, %0, %2\n"                                          \
 150 "       cbnz    %w1, 1b")                                               \
 151         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 152         : __stringify(constraint) "r" (i));                             \
 153 }
 154
 155 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
 156 static inline long                                                      \
 157 __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v)              \
 158 {                                                                       \
 159         s64 result;                                                     \
 160         unsigned long tmp;                                              \
 161                                                                         \
 162         asm volatile("// atomic64_" #op "_return" #name "\n"            \
 163         __LL_SC_FALLBACK(                                               \
 164 "       prfm    pstl1strm, %2\n"                                        \
 165 "1:     ld" #acq "xr    %0, %2\n"                                       \
 166 "       " #asm_op "     %0, %0, %3\n"                                   \
 167 "       st" #rel "xr    %w1, %0, %2\n"                                  \
 168 "       cbnz    %w1, 1b\n"                                              \
 169 "       " #mb )                                                         \
 170         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 171         : __stringify(constraint) "r" (i)                               \
 172         : cl);                                                          \
 173                                                                         \
 174         return result;                                                  \
 175 }
 176
 177 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
 178 static inline long                                                      \
 179 __ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v)         \
 180 {                                                                       \
 181         s64 result, val;                                                \
 182         unsigned long tmp;                                              \
 183                                                                         \
 184         asm volatile("// atomic64_fetch_" #op #name "\n"                \
 185         __LL_SC_FALLBACK(                                               \
 186 "       prfm    pstl1strm, %3\n"                                        \
 187 "1:     ld" #acq "xr    %0, %3\n"                                       \
 188 "       " #asm_op "     %1, %0, %4\n"                                   \
 189 "       st" #rel "xr    %w2, %1, %3\n"                                  \
 190 "       cbnz    %w2, 1b\n"                                              \
 191 "       " #mb )                                                         \
 192         : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)   \
 193         : __stringify(constraint) "r" (i)                               \
 194         : cl);                                                          \
 195                                                                         \
 196         return result;                                                  \
 197 }
 198
 199 #define ATOMIC64_OPS(...)                                               \
 200         ATOMIC64_OP(__VA_ARGS__)                                        \
 201         ATOMIC64_OP_RETURN(, dmb ish,  , l, "memory", __VA_ARGS__)      \
 202         ATOMIC64_OP_RETURN(_relaxed,,  ,  ,         , __VA_ARGS__)      \
 203         ATOMIC64_OP_RETURN(_acquire,, a,  , "memory", __VA_ARGS__)      \
 204         ATOMIC64_OP_RETURN(_release,,  , l, "memory", __VA_ARGS__)      \
 205         ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)      \
 206         ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)      \
 207         ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)      \
 208         ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)
 209
 210 ATOMIC64_OPS(add, add, I)
 211 ATOMIC64_OPS(sub, sub, J)
 212
 213 #undef ATOMIC64_OPS
 214 #define ATOMIC64_OPS(...)                                               \
 215         ATOMIC64_OP(__VA_ARGS__)                                        \
 216         ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)      \
 217         ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)      \
 218         ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)      \
 219         ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)
 220
 221 ATOMIC64_OPS(and, and, L)
 222 ATOMIC64_OPS(or, orr, L)
 223 ATOMIC64_OPS(xor, eor, L)
 224 /*
 225  * GAS converts the mysterious and undocumented BIC (immediate) alias to
 226  * an AND (immediate) instruction with the immediate inverted. We don't
 227  * have a constraint for this, so fall back to register.
 228  */
 229 ATOMIC64_OPS(andnot, bic, )
 230
 231 #undef ATOMIC64_OPS
 232 #undef ATOMIC64_FETCH_OP
 233 #undef ATOMIC64_OP_RETURN
 234 #undef ATOMIC64_OP
 235
 236 static inline s64
 237 __ll_sc_atomic64_dec_if_positive(atomic64_t *v)
 238 {
 239         s64 result;
 240         unsigned long tmp;
 241
 242         asm volatile("// atomic64_dec_if_positive\n"
 243         __LL_SC_FALLBACK(
 244 "       prfm    pstl1strm, %2\n"
 245 "1:     ldxr    %0, %2\n"
 246 "       subs    %0, %0, #1\n"
 247 "       b.lt    2f\n"
 248 "       stlxr   %w1, %0, %2\n"
 249 "       cbnz    %w1, 1b\n"
 250 "       dmb     ish\n"
 251 "2:")
 252         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
 253         :
 254         : "cc", "memory");
 255
 256         return result;
 257 }
 258
 259 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)  \
 260 static inline u##sz                                                     \
 261 __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,                    \
 262                                          unsigned long old,             \
 263                                          u##sz new)                     \
 264 {                                                                       \
 265         unsigned long tmp;                                              \
 266         u##sz oldval;                                                   \
 267                                                                         \
 268         /*                                                              \
 269          * Sub-word sizes require explicit casting so that the compare  \
 270          * part of the cmpxchg doesn't end up interpreting non-zero     \
 271          * upper bits of the register containing "old".                 \
 272          */                                                             \
 273         if (sz < 32)                                                    \
 274                 old = (u##sz)old;                                       \
 275                                                                         \
 276         asm volatile(                                                   \
 277         __LL_SC_FALLBACK(                                               \
 278         "       prfm    pstl1strm, %[v]\n"                              \
 279         "1:     ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"          \
 280         "       eor     %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"  \
 281         "       cbnz    %" #w "[tmp], 2f\n"                             \
 282         "       st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n"    \
 283         "       cbnz    %w[tmp], 1b\n"                                  \
 284         "       " #mb "\n"                                              \
 285         "2:")                                                           \
 286         : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),                   \
 287           [v] "+Q" (*(u##sz *)ptr)                                      \
 288         : [old] __stringify(constraint) "r" (old), [new] "r" (new)      \
 289         : cl);                                                          \
 290                                                                         \
 291         return oldval;                                                  \
 292 }
 293
 294 /*
 295  * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
 296  * handle the 'K' constraint for the value 4294967295 - thus we use no
 297  * constraint for 32 bit operations.
 298  */
 299 __CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         , K)
 300 __CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         , K)
 301 __CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         , K)
 302 __CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         , L)
 303 __CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory", K)
 304 __CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory", K)
 305 __CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory", K)
 306 __CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory", L)
 307 __CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory", K)
 308 __CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory", K)
 309 __CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory", K)
 310 __CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory", L)
 311 __CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory", K)
 312 __CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory", K)
 313 __CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory", K)
 314 __CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)
 315
 316 #undef __CMPXCHG_CASE
 317
 318 #define __CMPXCHG_DBL(name, mb, rel, cl)                                \
 319 static inline long                                                      \
 320 __ll_sc__cmpxchg_double##name(unsigned long old1,                       \
 321                                       unsigned long old2,               \
 322                                       unsigned long new1,               \
 323                                       unsigned long new2,               \
 324                                       volatile void *ptr)               \
 325 {                                                                       \
 326         unsigned long tmp, ret;                                         \
 327                                                                         \
 328         asm volatile("// __cmpxchg_double" #name "\n"                   \
 329         __LL_SC_FALLBACK(                                               \
 330         "       prfm    pstl1strm, %2\n"                                \
 331         "1:     ldxp    %0, %1, %2\n"                                   \
 332         "       eor     %0, %0, %3\n"                                   \
 333         "       eor     %1, %1, %4\n"                                   \
 334         "       orr     %1, %0, %1\n"                                   \
 335         "       cbnz    %1, 2f\n"                                       \
 336         "       st" #rel "xp    %w0, %5, %6, %2\n"                      \
 337         "       cbnz    %w0, 1b\n"                                      \
 338         "       " #mb "\n"                                              \
 339         "2:")                                                           \
 340         : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)        \
 341         : "r" (old1), "r" (old2), "r" (new1), "r" (new2)                \
 342         : cl);                                                          \
 343                                                                         \
 344         return ret;                                                     \
 345 }
 346
 347 __CMPXCHG_DBL(   ,        ,  ,         )
 348 __CMPXCHG_DBL(_mb, dmb ish, l, "memory")
 349
 350 #undef __CMPXCHG_DBL
 351 #undef K
 352
 353 #endif  /* __ASM_ATOMIC_LL_SC_H */