arch/powerpc/include/asm/cmpxchg.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_CMPXCHG_H_
   3 #define _ASM_POWERPC_CMPXCHG_H_
   4
   5 #ifdef __KERNEL__
   6 #include <linux/compiler.h>
   7 #include <asm/synch.h>
   8 #include <linux/bug.h>
   9
  10 #ifdef __BIG_ENDIAN
  11 #define BITOFF_CAL(size, off)   ((sizeof(u32) - size - off) * BITS_PER_BYTE)
  12 #else
  13 #define BITOFF_CAL(size, off)   (off * BITS_PER_BYTE)
  14 #endif
  15
  16 #define XCHG_GEN(type, sfx, cl)                         \
  17 static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
  18 {                                                               \
  19         unsigned int prev, prev_mask, tmp, bitoff, off;         \
  20                                                                 \
  21         off = (unsigned long)p % sizeof(u32);                   \
  22         bitoff = BITOFF_CAL(sizeof(type), off);                 \
  23         p -= off;                                               \
  24         val <<= bitoff;                                         \
  25         prev_mask = (u32)(type)-1 << bitoff;                    \
  26                                                                 \
  27         __asm__ __volatile__(                                   \
  28 "1:     lwarx   %0,0,%3\n"                                      \
  29 "       andc    %1,%0,%5\n"                                     \
  30 "       or      %1,%1,%4\n"                                     \
  31 "       stwcx.  %1,0,%3\n"                                      \
  32 "       bne-    1b\n"                                           \
  33         : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)            \
  34         : "r" (p), "r" (val), "r" (prev_mask)                   \
  35         : "cc", cl);                                            \
  36                                                                 \
  37         return prev >> bitoff;                                  \
  38 }
  39
  40 #define CMPXCHG_GEN(type, sfx, br, br2, cl)                     \
  41 static inline                                                   \
  42 u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new)   \
  43 {                                                               \
  44         unsigned int prev, prev_mask, tmp, bitoff, off;         \
  45                                                                 \
  46         off = (unsigned long)p % sizeof(u32);                   \
  47         bitoff = BITOFF_CAL(sizeof(type), off);                 \
  48         p -= off;                                               \
  49         old <<= bitoff;                                         \
  50         new <<= bitoff;                                         \
  51         prev_mask = (u32)(type)-1 << bitoff;                    \
  52                                                                 \
  53         __asm__ __volatile__(                                   \
  54         br                                                      \
  55 "1:     lwarx   %0,0,%3\n"                                      \
  56 "       and     %1,%0,%6\n"                                     \
  57 "       cmpw    0,%1,%4\n"                                      \
  58 "       bne-    2f\n"                                           \
  59 "       andc    %1,%0,%6\n"                                     \
  60 "       or      %1,%1,%5\n"                                     \
  61 "       stwcx.  %1,0,%3\n"                                      \
  62 "       bne-    1b\n"                                           \
  63         br2                                                     \
  64         "\n"                                                    \
  65 "2:"                                                            \
  66         : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)            \
  67         : "r" (p), "r" (old), "r" (new), "r" (prev_mask)        \
  68         : "cc", cl);                                            \
  69                                                                 \
  70         return prev >> bitoff;                                  \
  71 }
  72
  73 /*
  74  * Atomic exchange
  75  *
  76  * Changes the memory location '*p' to be val and returns
  77  * the previous value stored there.
  78  */
  79
  80 XCHG_GEN(u8, _local, "memory");
  81 XCHG_GEN(u8, _relaxed, "cc");
  82 XCHG_GEN(u16, _local, "memory");
  83 XCHG_GEN(u16, _relaxed, "cc");
  84
  85 static __always_inline unsigned long
  86 __xchg_u32_local(volatile void *p, unsigned long val)
  87 {
  88         unsigned long prev;
  89
  90         __asm__ __volatile__(
  91 "1:     lwarx   %0,0,%2 \n"
  92 "       stwcx.  %3,0,%2 \n\
  93         bne-    1b"
  94         : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
  95         : "r" (p), "r" (val)
  96         : "cc", "memory");
  97
  98         return prev;
  99 }
 100
 101 static __always_inline unsigned long
 102 __xchg_u32_relaxed(u32 *p, unsigned long val)
 103 {
 104         unsigned long prev;
 105
 106         __asm__ __volatile__(
 107 "1:     lwarx   %0,0,%2\n"
 108 "       stwcx.  %3,0,%2\n"
 109 "       bne-    1b"
 110         : "=&r" (prev), "+m" (*p)
 111         : "r" (p), "r" (val)
 112         : "cc");
 113
 114         return prev;
 115 }
 116
 117 #ifdef CONFIG_PPC64
 118 static __always_inline unsigned long
 119 __xchg_u64_local(volatile void *p, unsigned long val)
 120 {
 121         unsigned long prev;
 122
 123         __asm__ __volatile__(
 124 "1:     ldarx   %0,0,%2 \n"
 125 "       stdcx.  %3,0,%2 \n\
 126         bne-    1b"
 127         : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
 128         : "r" (p), "r" (val)
 129         : "cc", "memory");
 130
 131         return prev;
 132 }
 133
 134 static __always_inline unsigned long
 135 __xchg_u64_relaxed(u64 *p, unsigned long val)
 136 {
 137         unsigned long prev;
 138
 139         __asm__ __volatile__(
 140 "1:     ldarx   %0,0,%2\n"
 141 "       stdcx.  %3,0,%2\n"
 142 "       bne-    1b"
 143         : "=&r" (prev), "+m" (*p)
 144         : "r" (p), "r" (val)
 145         : "cc");
 146
 147         return prev;
 148 }
 149 #endif
 150
 151 static __always_inline unsigned long
 152 __xchg_local(void *ptr, unsigned long x, unsigned int size)
 153 {
 154         switch (size) {
 155         case 1:
 156                 return __xchg_u8_local(ptr, x);
 157         case 2:
 158                 return __xchg_u16_local(ptr, x);
 159         case 4:
 160                 return __xchg_u32_local(ptr, x);
 161 #ifdef CONFIG_PPC64
 162         case 8:
 163                 return __xchg_u64_local(ptr, x);
 164 #endif
 165         }
 166         BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
 167         return x;
 168 }
 169
 170 static __always_inline unsigned long
 171 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
 172 {
 173         switch (size) {
 174         case 1:
 175                 return __xchg_u8_relaxed(ptr, x);
 176         case 2:
 177                 return __xchg_u16_relaxed(ptr, x);
 178         case 4:
 179                 return __xchg_u32_relaxed(ptr, x);
 180 #ifdef CONFIG_PPC64
 181         case 8:
 182                 return __xchg_u64_relaxed(ptr, x);
 183 #endif
 184         }
 185         BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
 186         return x;
 187 }
 188 #define xchg_local(ptr,x)                                                    \
 189   ({                                                                         \
 190      __typeof__(*(ptr)) _x_ = (x);                                           \
 191      (__typeof__(*(ptr))) __xchg_local((ptr),                                \
 192                 (unsigned long)_x_, sizeof(*(ptr)));                         \
 193   })
 194
 195 #define xchg_relaxed(ptr, x)                                            \
 196 ({                                                                      \
 197         __typeof__(*(ptr)) _x_ = (x);                                   \
 198         (__typeof__(*(ptr))) __xchg_relaxed((ptr),                      \
 199                         (unsigned long)_x_, sizeof(*(ptr)));            \
 200 })
 201 /*
 202  * Compare and exchange - if *p == old, set it to new,
 203  * and return the old value of *p.
 204  */
 205
 206 CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
 207 CMPXCHG_GEN(u8, _local, , , "memory");
 208 CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
 209 CMPXCHG_GEN(u8, _relaxed, , , "cc");
 210 CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
 211 CMPXCHG_GEN(u16, _local, , , "memory");
 212 CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
 213 CMPXCHG_GEN(u16, _relaxed, , , "cc");
 214
 215 static __always_inline unsigned long
 216 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
 217 {
 218         unsigned int prev;
 219
 220         __asm__ __volatile__ (
 221         PPC_ATOMIC_ENTRY_BARRIER
 222 "1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 223         cmpw    0,%0,%3\n\
 224         bne-    2f\n"
 225 "       stwcx.  %4,0,%2\n\
 226         bne-    1b"
 227         PPC_ATOMIC_EXIT_BARRIER
 228         "\n\
 229 2:"
 230         : "=&r" (prev), "+m" (*p)
 231         : "r" (p), "r" (old), "r" (new)
 232         : "cc", "memory");
 233
 234         return prev;
 235 }
 236
 237 static __always_inline unsigned long
 238 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
 239                         unsigned long new)
 240 {
 241         unsigned int prev;
 242
 243         __asm__ __volatile__ (
 244 "1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 245         cmpw    0,%0,%3\n\
 246         bne-    2f\n"
 247 "       stwcx.  %4,0,%2\n\
 248         bne-    1b"
 249         "\n\
 250 2:"
 251         : "=&r" (prev), "+m" (*p)
 252         : "r" (p), "r" (old), "r" (new)
 253         : "cc", "memory");
 254
 255         return prev;
 256 }
 257
 258 static __always_inline unsigned long
 259 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
 260 {
 261         unsigned long prev;
 262
 263         __asm__ __volatile__ (
 264 "1:     lwarx   %0,0,%2         # __cmpxchg_u32_relaxed\n"
 265 "       cmpw    0,%0,%3\n"
 266 "       bne-    2f\n"
 267 "       stwcx.  %4,0,%2\n"
 268 "       bne-    1b\n"
 269 "2:"
 270         : "=&r" (prev), "+m" (*p)
 271         : "r" (p), "r" (old), "r" (new)
 272         : "cc");
 273
 274         return prev;
 275 }
 276
 277 /*
 278  * cmpxchg family don't have order guarantee if cmp part fails, therefore we
 279  * can avoid superfluous barriers if we use assembly code to implement
 280  * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
 281  * cmpxchg_release() because that will result in putting a barrier in the
 282  * middle of a ll/sc loop, which is probably a bad idea. For example, this
 283  * might cause the conditional store more likely to fail.
 284  */
 285 static __always_inline unsigned long
 286 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
 287 {
 288         unsigned long prev;
 289
 290         __asm__ __volatile__ (
 291 "1:     lwarx   %0,0,%2         # __cmpxchg_u32_acquire\n"
 292 "       cmpw    0,%0,%3\n"
 293 "       bne-    2f\n"
 294 "       stwcx.  %4,0,%2\n"
 295 "       bne-    1b\n"
 296         PPC_ACQUIRE_BARRIER
 297         "\n"
 298 "2:"
 299         : "=&r" (prev), "+m" (*p)
 300         : "r" (p), "r" (old), "r" (new)
 301         : "cc", "memory");
 302
 303         return prev;
 304 }
 305
 306 #ifdef CONFIG_PPC64
 307 static __always_inline unsigned long
 308 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
 309 {
 310         unsigned long prev;
 311
 312         __asm__ __volatile__ (
 313         PPC_ATOMIC_ENTRY_BARRIER
 314 "1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 315         cmpd    0,%0,%3\n\
 316         bne-    2f\n\
 317         stdcx.  %4,0,%2\n\
 318         bne-    1b"
 319         PPC_ATOMIC_EXIT_BARRIER
 320         "\n\
 321 2:"
 322         : "=&r" (prev), "+m" (*p)
 323         : "r" (p), "r" (old), "r" (new)
 324         : "cc", "memory");
 325
 326         return prev;
 327 }
 328
 329 static __always_inline unsigned long
 330 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
 331                         unsigned long new)
 332 {
 333         unsigned long prev;
 334
 335         __asm__ __volatile__ (
 336 "1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 337         cmpd    0,%0,%3\n\
 338         bne-    2f\n\
 339         stdcx.  %4,0,%2\n\
 340         bne-    1b"
 341         "\n\
 342 2:"
 343         : "=&r" (prev), "+m" (*p)
 344         : "r" (p), "r" (old), "r" (new)
 345         : "cc", "memory");
 346
 347         return prev;
 348 }
 349
 350 static __always_inline unsigned long
 351 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
 352 {
 353         unsigned long prev;
 354
 355         __asm__ __volatile__ (
 356 "1:     ldarx   %0,0,%2         # __cmpxchg_u64_relaxed\n"
 357 "       cmpd    0,%0,%3\n"
 358 "       bne-    2f\n"
 359 "       stdcx.  %4,0,%2\n"
 360 "       bne-    1b\n"
 361 "2:"
 362         : "=&r" (prev), "+m" (*p)
 363         : "r" (p), "r" (old), "r" (new)
 364         : "cc");
 365
 366         return prev;
 367 }
 368
 369 static __always_inline unsigned long
 370 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
 371 {
 372         unsigned long prev;
 373
 374         __asm__ __volatile__ (
 375 "1:     ldarx   %0,0,%2         # __cmpxchg_u64_acquire\n"
 376 "       cmpd    0,%0,%3\n"
 377 "       bne-    2f\n"
 378 "       stdcx.  %4,0,%2\n"
 379 "       bne-    1b\n"
 380         PPC_ACQUIRE_BARRIER
 381         "\n"
 382 "2:"
 383         : "=&r" (prev), "+m" (*p)
 384         : "r" (p), "r" (old), "r" (new)
 385         : "cc", "memory");
 386
 387         return prev;
 388 }
 389 #endif
 390
 391 static __always_inline unsigned long
 392 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
 393           unsigned int size)
 394 {
 395         switch (size) {
 396         case 1:
 397                 return __cmpxchg_u8(ptr, old, new);
 398         case 2:
 399                 return __cmpxchg_u16(ptr, old, new);
 400         case 4:
 401                 return __cmpxchg_u32(ptr, old, new);
 402 #ifdef CONFIG_PPC64
 403         case 8:
 404                 return __cmpxchg_u64(ptr, old, new);
 405 #endif
 406         }
 407         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
 408         return old;
 409 }
 410
 411 static __always_inline unsigned long
 412 __cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
 413           unsigned int size)
 414 {
 415         switch (size) {
 416         case 1:
 417                 return __cmpxchg_u8_local(ptr, old, new);
 418         case 2:
 419                 return __cmpxchg_u16_local(ptr, old, new);
 420         case 4:
 421                 return __cmpxchg_u32_local(ptr, old, new);
 422 #ifdef CONFIG_PPC64
 423         case 8:
 424                 return __cmpxchg_u64_local(ptr, old, new);
 425 #endif
 426         }
 427         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
 428         return old;
 429 }
 430
 431 static __always_inline unsigned long
 432 __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
 433                   unsigned int size)
 434 {
 435         switch (size) {
 436         case 1:
 437                 return __cmpxchg_u8_relaxed(ptr, old, new);
 438         case 2:
 439                 return __cmpxchg_u16_relaxed(ptr, old, new);
 440         case 4:
 441                 return __cmpxchg_u32_relaxed(ptr, old, new);
 442 #ifdef CONFIG_PPC64
 443         case 8:
 444                 return __cmpxchg_u64_relaxed(ptr, old, new);
 445 #endif
 446         }
 447         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
 448         return old;
 449 }
 450
 451 static __always_inline unsigned long
 452 __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
 453                   unsigned int size)
 454 {
 455         switch (size) {
 456         case 1:
 457                 return __cmpxchg_u8_acquire(ptr, old, new);
 458         case 2:
 459                 return __cmpxchg_u16_acquire(ptr, old, new);
 460         case 4:
 461                 return __cmpxchg_u32_acquire(ptr, old, new);
 462 #ifdef CONFIG_PPC64
 463         case 8:
 464                 return __cmpxchg_u64_acquire(ptr, old, new);
 465 #endif
 466         }
 467         BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
 468         return old;
 469 }
 470 #define cmpxchg(ptr, o, n)                                               \
 471   ({                                                                     \
 472      __typeof__(*(ptr)) _o_ = (o);                                       \
 473      __typeof__(*(ptr)) _n_ = (n);                                       \
 474      (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,           \
 475                                     (unsigned long)_n_, sizeof(*(ptr))); \
 476   })
 477
 478
 479 #define cmpxchg_local(ptr, o, n)                                         \
 480   ({                                                                     \
 481      __typeof__(*(ptr)) _o_ = (o);                                       \
 482      __typeof__(*(ptr)) _n_ = (n);                                       \
 483      (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
 484                                     (unsigned long)_n_, sizeof(*(ptr))); \
 485   })
 486
 487 #define cmpxchg_relaxed(ptr, o, n)                                      \
 488 ({                                                                      \
 489         __typeof__(*(ptr)) _o_ = (o);                                   \
 490         __typeof__(*(ptr)) _n_ = (n);                                   \
 491         (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),                   \
 492                         (unsigned long)_o_, (unsigned long)_n_,         \
 493                         sizeof(*(ptr)));                                \
 494 })
 495
 496 #define cmpxchg_acquire(ptr, o, n)                                      \
 497 ({                                                                      \
 498         __typeof__(*(ptr)) _o_ = (o);                                   \
 499         __typeof__(*(ptr)) _n_ = (n);                                   \
 500         (__typeof__(*(ptr))) __cmpxchg_acquire((ptr),                   \
 501                         (unsigned long)_o_, (unsigned long)_n_,         \
 502                         sizeof(*(ptr)));                                \
 503 })
 504 #ifdef CONFIG_PPC64
 505 #define cmpxchg64(ptr, o, n)                                            \
 506   ({                                                                    \
 507         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 508         cmpxchg((ptr), (o), (n));                                       \
 509   })
 510 #define cmpxchg64_local(ptr, o, n)                                      \
 511   ({                                                                    \
 512         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 513         cmpxchg_local((ptr), (o), (n));                                 \
 514   })
 515 #define cmpxchg64_relaxed(ptr, o, n)                                    \
 516 ({                                                                      \
 517         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 518         cmpxchg_relaxed((ptr), (o), (n));                               \
 519 })
 520 #define cmpxchg64_acquire(ptr, o, n)                                    \
 521 ({                                                                      \
 522         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 523         cmpxchg_acquire((ptr), (o), (n));                               \
 524 })
 525 #else
 526 #include <asm-generic/cmpxchg-local.h>
 527 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
 528 #endif
 529
 530 #endif /* __KERNEL__ */
 531 #endif /* _ASM_POWERPC_CMPXCHG_H_ */