arch/powerpc/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_ATOMIC_H_
   3 #define _ASM_POWERPC_ATOMIC_H_
   4
   5 /*
   6  * PowerPC atomic operations
   7  */
   8
   9 #ifdef __KERNEL__
  10 #include <linux/types.h>
  11 #include <asm/cmpxchg.h>
  12 #include <asm/barrier.h>
  13
  14 #define ATOMIC_INIT(i)          { (i) }
  15
  16 /*
  17  * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
  18  * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
  19  * on the platform without lwsync.
  20  */
  21 #define __atomic_acquire_fence()                                        \
  22         __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory")
  23
  24 #define __atomic_release_fence()                                        \
  25         __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory")
  26
  27 static __inline__ int atomic_read(const atomic_t *v)
  28 {
  29         int t;
  30
  31         __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
  32
  33         return t;
  34 }
  35
  36 static __inline__ void atomic_set(atomic_t *v, int i)
  37 {
  38         __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
  39 }
  40
  41 #define ATOMIC_OP(op, asm_op)                                           \
  42 static __inline__ void atomic_##op(int a, atomic_t *v)                  \
  43 {                                                                       \
  44         int t;                                                          \
  45                                                                         \
  46         __asm__ __volatile__(                                           \
  47 "1:     lwarx   %0,0,%3         # atomic_" #op "\n"                     \
  48         #asm_op " %0,%2,%0\n"                                           \
  49 "       stwcx.  %0,0,%3 \n"                                             \
  50 "       bne-    1b\n"                                                   \
  51         : "=&r" (t), "+m" (v->counter)                                  \
  52         : "r" (a), "r" (&v->counter)                                    \
  53         : "cc");                                                        \
  54 }                                                                       \
  55
  56 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op)                            \
  57 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v)      \
  58 {                                                                       \
  59         int t;                                                          \
  60                                                                         \
  61         __asm__ __volatile__(                                           \
  62 "1:     lwarx   %0,0,%3         # atomic_" #op "_return_relaxed\n"      \
  63         #asm_op " %0,%2,%0\n"                                           \
  64 "       stwcx.  %0,0,%3\n"                                              \
  65 "       bne-    1b\n"                                                   \
  66         : "=&r" (t), "+m" (v->counter)                                  \
  67         : "r" (a), "r" (&v->counter)                                    \
  68         : "cc");                                                        \
  69                                                                         \
  70         return t;                                                       \
  71 }
  72
  73 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op)                             \
  74 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v)       \
  75 {                                                                       \
  76         int res, t;                                                     \
  77                                                                         \
  78         __asm__ __volatile__(                                           \
  79 "1:     lwarx   %0,0,%4         # atomic_fetch_" #op "_relaxed\n"       \
  80         #asm_op " %1,%3,%0\n"                                           \
  81 "       stwcx.  %1,0,%4\n"                                              \
  82 "       bne-    1b\n"                                                   \
  83         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
  84         : "r" (a), "r" (&v->counter)                                    \
  85         : "cc");                                                        \
  86                                                                         \
  87         return res;                                                     \
  88 }
  89
  90 #define ATOMIC_OPS(op, asm_op)                                          \
  91         ATOMIC_OP(op, asm_op)                                           \
  92         ATOMIC_OP_RETURN_RELAXED(op, asm_op)                            \
  93         ATOMIC_FETCH_OP_RELAXED(op, asm_op)
  94
  95 ATOMIC_OPS(add, add)
  96 ATOMIC_OPS(sub, subf)
  97
  98 #define atomic_add_return_relaxed atomic_add_return_relaxed
  99 #define atomic_sub_return_relaxed atomic_sub_return_relaxed
 100
 101 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
 102 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
 103
 104 #undef ATOMIC_OPS
 105 #define ATOMIC_OPS(op, asm_op)                                          \
 106         ATOMIC_OP(op, asm_op)                                           \
 107         ATOMIC_FETCH_OP_RELAXED(op, asm_op)
 108
 109 ATOMIC_OPS(and, and)
 110 ATOMIC_OPS(or, or)
 111 ATOMIC_OPS(xor, xor)
 112
 113 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
 114 #define atomic_fetch_or_relaxed  atomic_fetch_or_relaxed
 115 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
 116
 117 #undef ATOMIC_OPS
 118 #undef ATOMIC_FETCH_OP_RELAXED
 119 #undef ATOMIC_OP_RETURN_RELAXED
 120 #undef ATOMIC_OP
 121
 122 static __inline__ void atomic_inc(atomic_t *v)
 123 {
 124         int t;
 125
 126         __asm__ __volatile__(
 127 "1:     lwarx   %0,0,%2         # atomic_inc\n\
 128         addic   %0,%0,1\n"
 129 "       stwcx.  %0,0,%2 \n\
 130         bne-    1b"
 131         : "=&r" (t), "+m" (v->counter)
 132         : "r" (&v->counter)
 133         : "cc", "xer");
 134 }
 135 #define atomic_inc atomic_inc
 136
 137 static __inline__ int atomic_inc_return_relaxed(atomic_t *v)
 138 {
 139         int t;
 140
 141         __asm__ __volatile__(
 142 "1:     lwarx   %0,0,%2         # atomic_inc_return_relaxed\n"
 143 "       addic   %0,%0,1\n"
 144 "       stwcx.  %0,0,%2\n"
 145 "       bne-    1b"
 146         : "=&r" (t), "+m" (v->counter)
 147         : "r" (&v->counter)
 148         : "cc", "xer");
 149
 150         return t;
 151 }
 152
 153 static __inline__ void atomic_dec(atomic_t *v)
 154 {
 155         int t;
 156
 157         __asm__ __volatile__(
 158 "1:     lwarx   %0,0,%2         # atomic_dec\n\
 159         addic   %0,%0,-1\n"
 160 "       stwcx.  %0,0,%2\n\
 161         bne-    1b"
 162         : "=&r" (t), "+m" (v->counter)
 163         : "r" (&v->counter)
 164         : "cc", "xer");
 165 }
 166 #define atomic_dec atomic_dec
 167
 168 static __inline__ int atomic_dec_return_relaxed(atomic_t *v)
 169 {
 170         int t;
 171
 172         __asm__ __volatile__(
 173 "1:     lwarx   %0,0,%2         # atomic_dec_return_relaxed\n"
 174 "       addic   %0,%0,-1\n"
 175 "       stwcx.  %0,0,%2\n"
 176 "       bne-    1b"
 177         : "=&r" (t), "+m" (v->counter)
 178         : "r" (&v->counter)
 179         : "cc", "xer");
 180
 181         return t;
 182 }
 183
 184 #define atomic_inc_return_relaxed atomic_inc_return_relaxed
 185 #define atomic_dec_return_relaxed atomic_dec_return_relaxed
 186
 187 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 188 #define atomic_cmpxchg_relaxed(v, o, n) \
 189         cmpxchg_relaxed(&((v)->counter), (o), (n))
 190 #define atomic_cmpxchg_acquire(v, o, n) \
 191         cmpxchg_acquire(&((v)->counter), (o), (n))
 192
 193 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
 194 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
 195
 196 /**
 197  * atomic_fetch_add_unless - add unless the number is a given value
 198  * @v: pointer of type atomic_t
 199  * @a: the amount to add to v...
 200  * @u: ...unless v is equal to u.
 201  *
 202  * Atomically adds @a to @v, so long as it was not @u.
 203  * Returns the old value of @v.
 204  */
 205 static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u)
 206 {
 207         int t;
 208
 209         __asm__ __volatile__ (
 210         PPC_ATOMIC_ENTRY_BARRIER
 211 "1:     lwarx   %0,0,%1         # atomic_fetch_add_unless\n\
 212         cmpw    0,%0,%3 \n\
 213         beq     2f \n\
 214         add     %0,%2,%0 \n"
 215 "       stwcx.  %0,0,%1 \n\
 216         bne-    1b \n"
 217         PPC_ATOMIC_EXIT_BARRIER
 218 "       subf    %0,%2,%0 \n\
 219 2:"
 220         : "=&r" (t)
 221         : "r" (&v->counter), "r" (a), "r" (u)
 222         : "cc", "memory");
 223
 224         return t;
 225 }
 226 #define atomic_fetch_add_unless atomic_fetch_add_unless
 227
 228 /**
 229  * atomic_inc_not_zero - increment unless the number is zero
 230  * @v: pointer of type atomic_t
 231  *
 232  * Atomically increments @v by 1, so long as @v is non-zero.
 233  * Returns non-zero if @v was non-zero, and zero otherwise.
 234  */
 235 static __inline__ int atomic_inc_not_zero(atomic_t *v)
 236 {
 237         int t1, t2;
 238
 239         __asm__ __volatile__ (
 240         PPC_ATOMIC_ENTRY_BARRIER
 241 "1:     lwarx   %0,0,%2         # atomic_inc_not_zero\n\
 242         cmpwi   0,%0,0\n\
 243         beq-    2f\n\
 244         addic   %1,%0,1\n"
 245 "       stwcx.  %1,0,%2\n\
 246         bne-    1b\n"
 247         PPC_ATOMIC_EXIT_BARRIER
 248         "\n\
 249 2:"
 250         : "=&r" (t1), "=&r" (t2)
 251         : "r" (&v->counter)
 252         : "cc", "xer", "memory");
 253
 254         return t1;
 255 }
 256 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v))
 257
 258 /*
 259  * Atomically test *v and decrement if it is greater than 0.
 260  * The function returns the old value of *v minus 1, even if
 261  * the atomic variable, v, was not decremented.
 262  */
 263 static __inline__ int atomic_dec_if_positive(atomic_t *v)
 264 {
 265         int t;
 266
 267         __asm__ __volatile__(
 268         PPC_ATOMIC_ENTRY_BARRIER
 269 "1:     lwarx   %0,0,%1         # atomic_dec_if_positive\n\
 270         cmpwi   %0,1\n\
 271         addi    %0,%0,-1\n\
 272         blt-    2f\n"
 273 "       stwcx.  %0,0,%1\n\
 274         bne-    1b"
 275         PPC_ATOMIC_EXIT_BARRIER
 276         "\n\
 277 2:"     : "=&b" (t)
 278         : "r" (&v->counter)
 279         : "cc", "memory");
 280
 281         return t;
 282 }
 283 #define atomic_dec_if_positive atomic_dec_if_positive
 284
 285 #ifdef __powerpc64__
 286
 287 #define ATOMIC64_INIT(i)        { (i) }
 288
 289 static __inline__ s64 atomic64_read(const atomic64_t *v)
 290 {
 291         s64 t;
 292
 293         __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
 294
 295         return t;
 296 }
 297
 298 static __inline__ void atomic64_set(atomic64_t *v, s64 i)
 299 {
 300         __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
 301 }
 302
 303 #define ATOMIC64_OP(op, asm_op)                                         \
 304 static __inline__ void atomic64_##op(s64 a, atomic64_t *v)              \
 305 {                                                                       \
 306         s64 t;                                                          \
 307                                                                         \
 308         __asm__ __volatile__(                                           \
 309 "1:     ldarx   %0,0,%3         # atomic64_" #op "\n"                   \
 310         #asm_op " %0,%2,%0\n"                                           \
 311 "       stdcx.  %0,0,%3 \n"                                             \
 312 "       bne-    1b\n"                                                   \
 313         : "=&r" (t), "+m" (v->counter)                                  \
 314         : "r" (a), "r" (&v->counter)                                    \
 315         : "cc");                                                        \
 316 }
 317
 318 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 319 static inline s64                                                       \
 320 atomic64_##op##_return_relaxed(s64 a, atomic64_t *v)                    \
 321 {                                                                       \
 322         s64 t;                                                          \
 323                                                                         \
 324         __asm__ __volatile__(                                           \
 325 "1:     ldarx   %0,0,%3         # atomic64_" #op "_return_relaxed\n"    \
 326         #asm_op " %0,%2,%0\n"                                           \
 327 "       stdcx.  %0,0,%3\n"                                              \
 328 "       bne-    1b\n"                                                   \
 329         : "=&r" (t), "+m" (v->counter)                                  \
 330         : "r" (a), "r" (&v->counter)                                    \
 331         : "cc");                                                        \
 332                                                                         \
 333         return t;                                                       \
 334 }
 335
 336 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op)                           \
 337 static inline s64                                                       \
 338 atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v)                     \
 339 {                                                                       \
 340         s64 res, t;                                                     \
 341                                                                         \
 342         __asm__ __volatile__(                                           \
 343 "1:     ldarx   %0,0,%4         # atomic64_fetch_" #op "_relaxed\n"     \
 344         #asm_op " %1,%3,%0\n"                                           \
 345 "       stdcx.  %1,0,%4\n"                                              \
 346 "       bne-    1b\n"                                                   \
 347         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
 348         : "r" (a), "r" (&v->counter)                                    \
 349         : "cc");                                                        \
 350                                                                         \
 351         return res;                                                     \
 352 }
 353
 354 #define ATOMIC64_OPS(op, asm_op)                                        \
 355         ATOMIC64_OP(op, asm_op)                                         \
 356         ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 357         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 358
 359 ATOMIC64_OPS(add, add)
 360 ATOMIC64_OPS(sub, subf)
 361
 362 #define atomic64_add_return_relaxed atomic64_add_return_relaxed
 363 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
 364
 365 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
 366 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
 367
 368 #undef ATOMIC64_OPS
 369 #define ATOMIC64_OPS(op, asm_op)                                        \
 370         ATOMIC64_OP(op, asm_op)                                         \
 371         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 372
 373 ATOMIC64_OPS(and, and)
 374 ATOMIC64_OPS(or, or)
 375 ATOMIC64_OPS(xor, xor)
 376
 377 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
 378 #define atomic64_fetch_or_relaxed  atomic64_fetch_or_relaxed
 379 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
 380
 381 #undef ATOPIC64_OPS
 382 #undef ATOMIC64_FETCH_OP_RELAXED
 383 #undef ATOMIC64_OP_RETURN_RELAXED
 384 #undef ATOMIC64_OP
 385
 386 static __inline__ void atomic64_inc(atomic64_t *v)
 387 {
 388         s64 t;
 389
 390         __asm__ __volatile__(
 391 "1:     ldarx   %0,0,%2         # atomic64_inc\n\
 392         addic   %0,%0,1\n\
 393         stdcx.  %0,0,%2 \n\
 394         bne-    1b"
 395         : "=&r" (t), "+m" (v->counter)
 396         : "r" (&v->counter)
 397         : "cc", "xer");
 398 }
 399 #define atomic64_inc atomic64_inc
 400
 401 static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v)
 402 {
 403         s64 t;
 404
 405         __asm__ __volatile__(
 406 "1:     ldarx   %0,0,%2         # atomic64_inc_return_relaxed\n"
 407 "       addic   %0,%0,1\n"
 408 "       stdcx.  %0,0,%2\n"
 409 "       bne-    1b"
 410         : "=&r" (t), "+m" (v->counter)
 411         : "r" (&v->counter)
 412         : "cc", "xer");
 413
 414         return t;
 415 }
 416
 417 static __inline__ void atomic64_dec(atomic64_t *v)
 418 {
 419         s64 t;
 420
 421         __asm__ __volatile__(
 422 "1:     ldarx   %0,0,%2         # atomic64_dec\n\
 423         addic   %0,%0,-1\n\
 424         stdcx.  %0,0,%2\n\
 425         bne-    1b"
 426         : "=&r" (t), "+m" (v->counter)
 427         : "r" (&v->counter)
 428         : "cc", "xer");
 429 }
 430 #define atomic64_dec atomic64_dec
 431
 432 static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v)
 433 {
 434         s64 t;
 435
 436         __asm__ __volatile__(
 437 "1:     ldarx   %0,0,%2         # atomic64_dec_return_relaxed\n"
 438 "       addic   %0,%0,-1\n"
 439 "       stdcx.  %0,0,%2\n"
 440 "       bne-    1b"
 441         : "=&r" (t), "+m" (v->counter)
 442         : "r" (&v->counter)
 443         : "cc", "xer");
 444
 445         return t;
 446 }
 447
 448 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
 449 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
 450
 451 /*
 452  * Atomically test *v and decrement if it is greater than 0.
 453  * The function returns the old value of *v minus 1.
 454  */
 455 static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v)
 456 {
 457         s64 t;
 458
 459         __asm__ __volatile__(
 460         PPC_ATOMIC_ENTRY_BARRIER
 461 "1:     ldarx   %0,0,%1         # atomic64_dec_if_positive\n\
 462         addic.  %0,%0,-1\n\
 463         blt-    2f\n\
 464         stdcx.  %0,0,%1\n\
 465         bne-    1b"
 466         PPC_ATOMIC_EXIT_BARRIER
 467         "\n\
 468 2:"     : "=&r" (t)
 469         : "r" (&v->counter)
 470         : "cc", "xer", "memory");
 471
 472         return t;
 473 }
 474 #define atomic64_dec_if_positive atomic64_dec_if_positive
 475
 476 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 477 #define atomic64_cmpxchg_relaxed(v, o, n) \
 478         cmpxchg_relaxed(&((v)->counter), (o), (n))
 479 #define atomic64_cmpxchg_acquire(v, o, n) \
 480         cmpxchg_acquire(&((v)->counter), (o), (n))
 481
 482 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 483 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
 484
 485 /**
 486  * atomic64_fetch_add_unless - add unless the number is a given value
 487  * @v: pointer of type atomic64_t
 488  * @a: the amount to add to v...
 489  * @u: ...unless v is equal to u.
 490  *
 491  * Atomically adds @a to @v, so long as it was not @u.
 492  * Returns the old value of @v.
 493  */
 494 static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 495 {
 496         s64 t;
 497
 498         __asm__ __volatile__ (
 499         PPC_ATOMIC_ENTRY_BARRIER
 500 "1:     ldarx   %0,0,%1         # atomic64_fetch_add_unless\n\
 501         cmpd    0,%0,%3 \n\
 502         beq     2f \n\
 503         add     %0,%2,%0 \n"
 504 "       stdcx.  %0,0,%1 \n\
 505         bne-    1b \n"
 506         PPC_ATOMIC_EXIT_BARRIER
 507 "       subf    %0,%2,%0 \n\
 508 2:"
 509         : "=&r" (t)
 510         : "r" (&v->counter), "r" (a), "r" (u)
 511         : "cc", "memory");
 512
 513         return t;
 514 }
 515 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
 516
 517 /**
 518  * atomic_inc64_not_zero - increment unless the number is zero
 519  * @v: pointer of type atomic64_t
 520  *
 521  * Atomically increments @v by 1, so long as @v is non-zero.
 522  * Returns non-zero if @v was non-zero, and zero otherwise.
 523  */
 524 static __inline__ int atomic64_inc_not_zero(atomic64_t *v)
 525 {
 526         s64 t1, t2;
 527
 528         __asm__ __volatile__ (
 529         PPC_ATOMIC_ENTRY_BARRIER
 530 "1:     ldarx   %0,0,%2         # atomic64_inc_not_zero\n\
 531         cmpdi   0,%0,0\n\
 532         beq-    2f\n\
 533         addic   %1,%0,1\n\
 534         stdcx.  %1,0,%2\n\
 535         bne-    1b\n"
 536         PPC_ATOMIC_EXIT_BARRIER
 537         "\n\
 538 2:"
 539         : "=&r" (t1), "=&r" (t2)
 540         : "r" (&v->counter)
 541         : "cc", "xer", "memory");
 542
 543         return t1 != 0;
 544 }
 545 #define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v))
 546
 547 #endif /* __powerpc64__ */
 548
 549 #endif /* __KERNEL__ */
 550 #endif /* _ASM_POWERPC_ATOMIC_H_ */