arch/arc/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   4  */
   5
   6 #ifndef _ASM_ARC_ATOMIC_H
   7 #define _ASM_ARC_ATOMIC_H
   8
   9 #ifndef __ASSEMBLY__
  10
  11 #include <linux/types.h>
  12 #include <linux/compiler.h>
  13 #include <asm/cmpxchg.h>
  14 #include <asm/barrier.h>
  15 #include <asm/smp.h>
  16
  17 #define arch_atomic_read(v)  READ_ONCE((v)->counter)
  18
  19 #ifdef CONFIG_ARC_HAS_LLSC
  20
  21 #define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
  22
  23 #define ATOMIC_OP(op, c_op, asm_op)                                     \
  24 static inline void arch_atomic_##op(int i, atomic_t *v)                 \
  25 {                                                                       \
  26         unsigned int val;                                               \
  27                                                                         \
  28         __asm__ __volatile__(                                           \
  29         "1:     llock   %[val], [%[ctr]]                \n"             \
  30         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  31         "       scond   %[val], [%[ctr]]                \n"             \
  32         "       bnz     1b                              \n"             \
  33         : [val] "=&r"   (val) /* Early clobber to prevent reg reuse */  \
  34         : [ctr] "r"     (&v->counter), /* Not "m": llock only supports reg direct addr mode */  \
  35           [i]   "ir"    (i)                                             \
  36         : "cc");                                                        \
  37 }                                                                       \
  38
  39 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
  40 static inline int arch_atomic_##op##_return(int i, atomic_t *v)         \
  41 {                                                                       \
  42         unsigned int val;                                               \
  43                                                                         \
  44         /*                                                              \
  45          * Explicit full memory barrier needed before/after as          \
  46          * LLOCK/SCOND themselves don't provide any such semantics      \
  47          */                                                             \
  48         smp_mb();                                                       \
  49                                                                         \
  50         __asm__ __volatile__(                                           \
  51         "1:     llock   %[val], [%[ctr]]                \n"             \
  52         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  53         "       scond   %[val], [%[ctr]]                \n"             \
  54         "       bnz     1b                              \n"             \
  55         : [val] "=&r"   (val)                                           \
  56         : [ctr] "r"     (&v->counter),                                  \
  57           [i]   "ir"    (i)                                             \
  58         : "cc");                                                        \
  59                                                                         \
  60         smp_mb();                                                       \
  61                                                                         \
  62         return val;                                                     \
  63 }
  64
  65 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
  66 static inline int arch_atomic_fetch_##op(int i, atomic_t *v)            \
  67 {                                                                       \
  68         unsigned int val, orig;                                         \
  69                                                                         \
  70         /*                                                              \
  71          * Explicit full memory barrier needed before/after as          \
  72          * LLOCK/SCOND themselves don't provide any such semantics      \
  73          */                                                             \
  74         smp_mb();                                                       \
  75                                                                         \
  76         __asm__ __volatile__(                                           \
  77         "1:     llock   %[orig], [%[ctr]]               \n"             \
  78         "       " #asm_op " %[val], %[orig], %[i]       \n"             \
  79         "       scond   %[val], [%[ctr]]                \n"             \
  80         "       bnz     1b                              \n"             \
  81         : [val] "=&r"   (val),                                          \
  82           [orig] "=&r" (orig)                                           \
  83         : [ctr] "r"     (&v->counter),                                  \
  84           [i]   "ir"    (i)                                             \
  85         : "cc");                                                        \
  86                                                                         \
  87         smp_mb();                                                       \
  88                                                                         \
  89         return orig;                                                    \
  90 }
  91
  92 #else   /* !CONFIG_ARC_HAS_LLSC */
  93
  94 #ifndef CONFIG_SMP
  95
  96  /* violating atomic_xxx API locking protocol in UP for optimization sake */
  97 #define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
  98
  99 #else
 100
 101 static inline void arch_atomic_set(atomic_t *v, int i)
 102 {
 103         /*
 104          * Independent of hardware support, all of the atomic_xxx() APIs need
 105          * to follow the same locking rules to make sure that a "hardware"
 106          * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
 107          * sequence
 108          *
 109          * Thus atomic_set() despite being 1 insn (and seemingly atomic)
 110          * requires the locking.
 111          */
 112         unsigned long flags;
 113
 114         atomic_ops_lock(flags);
 115         WRITE_ONCE(v->counter, i);
 116         atomic_ops_unlock(flags);
 117 }
 118
 119 #define arch_atomic_set_release(v, i)   arch_atomic_set((v), (i))
 120
 121 #endif
 122
 123 /*
 124  * Non hardware assisted Atomic-R-M-W
 125  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 126  */
 127
 128 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 129 static inline void arch_atomic_##op(int i, atomic_t *v)                 \
 130 {                                                                       \
 131         unsigned long flags;                                            \
 132                                                                         \
 133         atomic_ops_lock(flags);                                         \
 134         v->counter c_op i;                                              \
 135         atomic_ops_unlock(flags);                                       \
 136 }
 137
 138 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 139 static inline int arch_atomic_##op##_return(int i, atomic_t *v)         \
 140 {                                                                       \
 141         unsigned long flags;                                            \
 142         unsigned long temp;                                             \
 143                                                                         \
 144         /*                                                              \
 145          * spin lock/unlock provides the needed smp_mb() before/after   \
 146          */                                                             \
 147         atomic_ops_lock(flags);                                         \
 148         temp = v->counter;                                              \
 149         temp c_op i;                                                    \
 150         v->counter = temp;                                              \
 151         atomic_ops_unlock(flags);                                       \
 152                                                                         \
 153         return temp;                                                    \
 154 }
 155
 156 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 157 static inline int arch_atomic_fetch_##op(int i, atomic_t *v)            \
 158 {                                                                       \
 159         unsigned long flags;                                            \
 160         unsigned long orig;                                             \
 161                                                                         \
 162         /*                                                              \
 163          * spin lock/unlock provides the needed smp_mb() before/after   \
 164          */                                                             \
 165         atomic_ops_lock(flags);                                         \
 166         orig = v->counter;                                              \
 167         v->counter c_op i;                                              \
 168         atomic_ops_unlock(flags);                                       \
 169                                                                         \
 170         return orig;                                                    \
 171 }
 172
 173 #endif /* !CONFIG_ARC_HAS_LLSC */
 174
 175 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 176         ATOMIC_OP(op, c_op, asm_op)                                     \
 177         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 178         ATOMIC_FETCH_OP(op, c_op, asm_op)
 179
 180 ATOMIC_OPS(add, +=, add)
 181 ATOMIC_OPS(sub, -=, sub)
 182
 183 #undef ATOMIC_OPS
 184 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 185         ATOMIC_OP(op, c_op, asm_op)                                     \
 186         ATOMIC_FETCH_OP(op, c_op, asm_op)
 187
 188 ATOMIC_OPS(and, &=, and)
 189 ATOMIC_OPS(andnot, &= ~, bic)
 190 ATOMIC_OPS(or, |=, or)
 191 ATOMIC_OPS(xor, ^=, xor)
 192
 193 #define arch_atomic_andnot              arch_atomic_andnot
 194 #define arch_atomic_fetch_andnot        arch_atomic_fetch_andnot
 195
 196 #undef ATOMIC_OPS
 197 #undef ATOMIC_FETCH_OP
 198 #undef ATOMIC_OP_RETURN
 199 #undef ATOMIC_OP
 200
 201 #ifdef CONFIG_GENERIC_ATOMIC64
 202
 203 #include <asm-generic/atomic64.h>
 204
 205 #else   /* Kconfig ensures this is only enabled with needed h/w assist */
 206
 207 /*
 208  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
 209  *  - The address HAS to be 64-bit aligned
 210  *  - There are 2 semantics involved here:
 211  *    = exclusive implies no interim update between load/store to same addr
 212  *    = both words are observed/updated together: this is guaranteed even
 213  *      for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
 214  *      is NOT required to use LLOCKD+SCONDD, STD suffices
 215  */
 216
 217 typedef struct {
 218         s64 __aligned(8) counter;
 219 } atomic64_t;
 220
 221 #define ATOMIC64_INIT(a) { (a) }
 222
 223 static inline s64 arch_atomic64_read(const atomic64_t *v)
 224 {
 225         s64 val;
 226
 227         __asm__ __volatile__(
 228         "       ldd   %0, [%1]  \n"
 229         : "=r"(val)
 230         : "r"(&v->counter));
 231
 232         return val;
 233 }
 234
 235 static inline void arch_atomic64_set(atomic64_t *v, s64 a)
 236 {
 237         /*
 238          * This could have been a simple assignment in "C" but would need
 239          * explicit volatile. Otherwise gcc optimizers could elide the store
 240          * which borked atomic64 self-test
 241          * In the inline asm version, memory clobber needed for exact same
 242          * reason, to tell gcc about the store.
 243          *
 244          * This however is not needed for sibling atomic64_add() etc since both
 245          * load/store are explicitly done in inline asm. As long as API is used
 246          * for each access, gcc has no way to optimize away any load/store
 247          */
 248         __asm__ __volatile__(
 249         "       std   %0, [%1]  \n"
 250         :
 251         : "r"(a), "r"(&v->counter)
 252         : "memory");
 253 }
 254
 255 #define ATOMIC64_OP(op, op1, op2)                                       \
 256 static inline void arch_atomic64_##op(s64 a, atomic64_t *v)             \
 257 {                                                                       \
 258         s64 val;                                                        \
 259                                                                         \
 260         __asm__ __volatile__(                                           \
 261         "1:                             \n"                             \
 262         "       llockd  %0, [%1]        \n"                             \
 263         "       " #op1 " %L0, %L0, %L2  \n"                             \
 264         "       " #op2 " %H0, %H0, %H2  \n"                             \
 265         "       scondd   %0, [%1]       \n"                             \
 266         "       bnz     1b              \n"                             \
 267         : "=&r"(val)                                                    \
 268         : "r"(&v->counter), "ir"(a)                                     \
 269         : "cc");                                                        \
 270 }                                                                       \
 271
 272 #define ATOMIC64_OP_RETURN(op, op1, op2)                                \
 273 static inline s64 arch_atomic64_##op##_return(s64 a, atomic64_t *v)     \
 274 {                                                                       \
 275         s64 val;                                                        \
 276                                                                         \
 277         smp_mb();                                                       \
 278                                                                         \
 279         __asm__ __volatile__(                                           \
 280         "1:                             \n"                             \
 281         "       llockd   %0, [%1]       \n"                             \
 282         "       " #op1 " %L0, %L0, %L2  \n"                             \
 283         "       " #op2 " %H0, %H0, %H2  \n"                             \
 284         "       scondd   %0, [%1]       \n"                             \
 285         "       bnz     1b              \n"                             \
 286         : [val] "=&r"(val)                                              \
 287         : "r"(&v->counter), "ir"(a)                                     \
 288         : "cc");        /* memory clobber comes from smp_mb() */        \
 289                                                                         \
 290         smp_mb();                                                       \
 291                                                                         \
 292         return val;                                                     \
 293 }
 294
 295 #define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
 296 static inline s64 arch_atomic64_fetch_##op(s64 a, atomic64_t *v)        \
 297 {                                                                       \
 298         s64 val, orig;                                                  \
 299                                                                         \
 300         smp_mb();                                                       \
 301                                                                         \
 302         __asm__ __volatile__(                                           \
 303         "1:                             \n"                             \
 304         "       llockd   %0, [%2]       \n"                             \
 305         "       " #op1 " %L1, %L0, %L3  \n"                             \
 306         "       " #op2 " %H1, %H0, %H3  \n"                             \
 307         "       scondd   %1, [%2]       \n"                             \
 308         "       bnz     1b              \n"                             \
 309         : "=&r"(orig), "=&r"(val)                                       \
 310         : "r"(&v->counter), "ir"(a)                                     \
 311         : "cc");        /* memory clobber comes from smp_mb() */        \
 312                                                                         \
 313         smp_mb();                                                       \
 314                                                                         \
 315         return orig;                                                    \
 316 }
 317
 318 #define ATOMIC64_OPS(op, op1, op2)                                      \
 319         ATOMIC64_OP(op, op1, op2)                                       \
 320         ATOMIC64_OP_RETURN(op, op1, op2)                                \
 321         ATOMIC64_FETCH_OP(op, op1, op2)
 322
 323 ATOMIC64_OPS(add, add.f, adc)
 324 ATOMIC64_OPS(sub, sub.f, sbc)
 325 ATOMIC64_OPS(and, and, and)
 326 ATOMIC64_OPS(andnot, bic, bic)
 327 ATOMIC64_OPS(or, or, or)
 328 ATOMIC64_OPS(xor, xor, xor)
 329
 330 #define arch_atomic64_andnot            arch_atomic64_andnot
 331 #define arch_atomic64_fetch_andnot      arch_atomic64_fetch_andnot
 332
 333 #undef ATOMIC64_OPS
 334 #undef ATOMIC64_FETCH_OP
 335 #undef ATOMIC64_OP_RETURN
 336 #undef ATOMIC64_OP
 337
 338 static inline s64
 339 arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)
 340 {
 341         s64 prev;
 342
 343         smp_mb();
 344
 345         __asm__ __volatile__(
 346         "1:     llockd  %0, [%1]        \n"
 347         "       brne    %L0, %L2, 2f    \n"
 348         "       brne    %H0, %H2, 2f    \n"
 349         "       scondd  %3, [%1]        \n"
 350         "       bnz     1b              \n"
 351         "2:                             \n"
 352         : "=&r"(prev)
 353         : "r"(ptr), "ir"(expected), "r"(new)
 354         : "cc");        /* memory clobber comes from smp_mb() */
 355
 356         smp_mb();
 357
 358         return prev;
 359 }
 360
 361 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
 362 {
 363         s64 prev;
 364
 365         smp_mb();
 366
 367         __asm__ __volatile__(
 368         "1:     llockd  %0, [%1]        \n"
 369         "       scondd  %2, [%1]        \n"
 370         "       bnz     1b              \n"
 371         "2:                             \n"
 372         : "=&r"(prev)
 373         : "r"(ptr), "r"(new)
 374         : "cc");        /* memory clobber comes from smp_mb() */
 375
 376         smp_mb();
 377
 378         return prev;
 379 }
 380
 381 /**
 382  * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
 383  * @v: pointer of type atomic64_t
 384  *
 385  * The function returns the old value of *v minus 1, even if
 386  * the atomic variable, v, was not decremented.
 387  */
 388
 389 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 390 {
 391         s64 val;
 392
 393         smp_mb();
 394
 395         __asm__ __volatile__(
 396         "1:     llockd  %0, [%1]        \n"
 397         "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 398         "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 399         "       brlt    %H0, 0, 2f      \n"
 400         "       scondd  %0, [%1]        \n"
 401         "       bnz     1b              \n"
 402         "2:                             \n"
 403         : "=&r"(val)
 404         : "r"(&v->counter)
 405         : "cc");        /* memory clobber comes from smp_mb() */
 406
 407         smp_mb();
 408
 409         return val;
 410 }
 411 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
 412
 413 /**
 414  * arch_atomic64_fetch_add_unless - add unless the number is a given value
 415  * @v: pointer of type atomic64_t
 416  * @a: the amount to add to v...
 417  * @u: ...unless v is equal to u.
 418  *
 419  * Atomically adds @a to @v, if it was not @u.
 420  * Returns the old value of @v
 421  */
 422 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 423 {
 424         s64 old, temp;
 425
 426         smp_mb();
 427
 428         __asm__ __volatile__(
 429         "1:     llockd  %0, [%2]        \n"
 430         "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 431         "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 432         "2:                             \n"
 433         "       add.f   %L1, %L0, %L3   \n"
 434         "       adc     %H1, %H0, %H3   \n"
 435         "       scondd  %1, [%2]        \n"
 436         "       bnz     1b              \n"
 437         "3:                             \n"
 438         : "=&r"(old), "=&r" (temp)
 439         : "r"(&v->counter), "r"(a), "r"(u)
 440         : "cc");        /* memory clobber comes from smp_mb() */
 441
 442         smp_mb();
 443
 444         return old;
 445 }
 446 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 447
 448 #endif  /* !CONFIG_GENERIC_ATOMIC64 */
 449
 450 #endif  /* !__ASSEMBLY__ */
 451
 452 #endif