arch/riscv/include/asm/cmpxchg.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2014 Regents of the University of California
   4  */
   5
   6 #ifndef _ASM_RISCV_CMPXCHG_H
   7 #define _ASM_RISCV_CMPXCHG_H
   8
   9 #include <linux/bug.h>
  10
  11 #include <asm/barrier.h>
  12 #include <asm/fence.h>
  13
  14 #define __xchg_relaxed(ptr, new, size)                                  \
  15 ({                                                                      \
  16         __typeof__(ptr) __ptr = (ptr);                                  \
  17         __typeof__(new) __new = (new);                                  \
  18         __typeof__(*(ptr)) __ret;                                       \
  19         switch (size) {                                                 \
  20         case 4:                                                         \
  21                 __asm__ __volatile__ (                                  \
  22                         "       amoswap.w %0, %2, %1\n"                 \
  23                         : "=r" (__ret), "+A" (*__ptr)                   \
  24                         : "r" (__new)                                   \
  25                         : "memory");                                    \
  26                 break;                                                  \
  27         case 8:                                                         \
  28                 __asm__ __volatile__ (                                  \
  29                         "       amoswap.d %0, %2, %1\n"                 \
  30                         : "=r" (__ret), "+A" (*__ptr)                   \
  31                         : "r" (__new)                                   \
  32                         : "memory");                                    \
  33                 break;                                                  \
  34         default:                                                        \
  35                 BUILD_BUG();                                            \
  36         }                                                               \
  37         __ret;                                                          \
  38 })
  39
  40 #define arch_xchg_relaxed(ptr, x)                                       \
  41 ({                                                                      \
  42         __typeof__(*(ptr)) _x_ = (x);                                   \
  43         (__typeof__(*(ptr))) __xchg_relaxed((ptr),                      \
  44                                             _x_, sizeof(*(ptr)));       \
  45 })
  46
  47 #define __xchg_acquire(ptr, new, size)                                  \
  48 ({                                                                      \
  49         __typeof__(ptr) __ptr = (ptr);                                  \
  50         __typeof__(new) __new = (new);                                  \
  51         __typeof__(*(ptr)) __ret;                                       \
  52         switch (size) {                                                 \
  53         case 4:                                                         \
  54                 __asm__ __volatile__ (                                  \
  55                         "       amoswap.w %0, %2, %1\n"                 \
  56                         RISCV_ACQUIRE_BARRIER                           \
  57                         : "=r" (__ret), "+A" (*__ptr)                   \
  58                         : "r" (__new)                                   \
  59                         : "memory");                                    \
  60                 break;                                                  \
  61         case 8:                                                         \
  62                 __asm__ __volatile__ (                                  \
  63                         "       amoswap.d %0, %2, %1\n"                 \
  64                         RISCV_ACQUIRE_BARRIER                           \
  65                         : "=r" (__ret), "+A" (*__ptr)                   \
  66                         : "r" (__new)                                   \
  67                         : "memory");                                    \
  68                 break;                                                  \
  69         default:                                                        \
  70                 BUILD_BUG();                                            \
  71         }                                                               \
  72         __ret;                                                          \
  73 })
  74
  75 #define arch_xchg_acquire(ptr, x)                                       \
  76 ({                                                                      \
  77         __typeof__(*(ptr)) _x_ = (x);                                   \
  78         (__typeof__(*(ptr))) __xchg_acquire((ptr),                      \
  79                                             _x_, sizeof(*(ptr)));       \
  80 })
  81
  82 #define __xchg_release(ptr, new, size)                                  \
  83 ({                                                                      \
  84         __typeof__(ptr) __ptr = (ptr);                                  \
  85         __typeof__(new) __new = (new);                                  \
  86         __typeof__(*(ptr)) __ret;                                       \
  87         switch (size) {                                                 \
  88         case 4:                                                         \
  89                 __asm__ __volatile__ (                                  \
  90                         RISCV_RELEASE_BARRIER                           \
  91                         "       amoswap.w %0, %2, %1\n"                 \
  92                         : "=r" (__ret), "+A" (*__ptr)                   \
  93                         : "r" (__new)                                   \
  94                         : "memory");                                    \
  95                 break;                                                  \
  96         case 8:                                                         \
  97                 __asm__ __volatile__ (                                  \
  98                         RISCV_RELEASE_BARRIER                           \
  99                         "       amoswap.d %0, %2, %1\n"                 \
 100                         : "=r" (__ret), "+A" (*__ptr)                   \
 101                         : "r" (__new)                                   \
 102                         : "memory");                                    \
 103                 break;                                                  \
 104         default:                                                        \
 105                 BUILD_BUG();                                            \
 106         }                                                               \
 107         __ret;                                                          \
 108 })
 109
 110 #define arch_xchg_release(ptr, x)                                       \
 111 ({                                                                      \
 112         __typeof__(*(ptr)) _x_ = (x);                                   \
 113         (__typeof__(*(ptr))) __xchg_release((ptr),                      \
 114                                             _x_, sizeof(*(ptr)));       \
 115 })
 116
 117 #define __arch_xchg(ptr, new, size)                                     \
 118 ({                                                                      \
 119         __typeof__(ptr) __ptr = (ptr);                                  \
 120         __typeof__(new) __new = (new);                                  \
 121         __typeof__(*(ptr)) __ret;                                       \
 122         switch (size) {                                                 \
 123         case 4:                                                         \
 124                 __asm__ __volatile__ (                                  \
 125                         "       amoswap.w.aqrl %0, %2, %1\n"            \
 126                         : "=r" (__ret), "+A" (*__ptr)                   \
 127                         : "r" (__new)                                   \
 128                         : "memory");                                    \
 129                 break;                                                  \
 130         case 8:                                                         \
 131                 __asm__ __volatile__ (                                  \
 132                         "       amoswap.d.aqrl %0, %2, %1\n"            \
 133                         : "=r" (__ret), "+A" (*__ptr)                   \
 134                         : "r" (__new)                                   \
 135                         : "memory");                                    \
 136                 break;                                                  \
 137         default:                                                        \
 138                 BUILD_BUG();                                            \
 139         }                                                               \
 140         __ret;                                                          \
 141 })
 142
 143 #define arch_xchg(ptr, x)                                               \
 144 ({                                                                      \
 145         __typeof__(*(ptr)) _x_ = (x);                                   \
 146         (__typeof__(*(ptr))) __arch_xchg((ptr), _x_, sizeof(*(ptr)));   \
 147 })
 148
 149 #define xchg32(ptr, x)                                                  \
 150 ({                                                                      \
 151         BUILD_BUG_ON(sizeof(*(ptr)) != 4);                              \
 152         arch_xchg((ptr), (x));                                          \
 153 })
 154
 155 #define xchg64(ptr, x)                                                  \
 156 ({                                                                      \
 157         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 158         arch_xchg((ptr), (x));                                          \
 159 })
 160
 161 /*
 162  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 163  * store NEW in MEM.  Return the initial value in MEM.  Success is
 164  * indicated by comparing RETURN with OLD.
 165  */
 166 #define __cmpxchg_relaxed(ptr, old, new, size)                          \
 167 ({                                                                      \
 168         __typeof__(ptr) __ptr = (ptr);                                  \
 169         __typeof__(*(ptr)) __old = (old);                               \
 170         __typeof__(*(ptr)) __new = (new);                               \
 171         __typeof__(*(ptr)) __ret;                                       \
 172         register unsigned int __rc;                                     \
 173         switch (size) {                                                 \
 174         case 4:                                                         \
 175                 __asm__ __volatile__ (                                  \
 176                         "0:     lr.w %0, %2\n"                          \
 177                         "       bne  %0, %z3, 1f\n"                     \
 178                         "       sc.w %1, %z4, %2\n"                     \
 179                         "       bnez %1, 0b\n"                          \
 180                         "1:\n"                                          \
 181                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 182                         : "rJ" ((long)__old), "rJ" (__new)              \
 183                         : "memory");                                    \
 184                 break;                                                  \
 185         case 8:                                                         \
 186                 __asm__ __volatile__ (                                  \
 187                         "0:     lr.d %0, %2\n"                          \
 188                         "       bne %0, %z3, 1f\n"                      \
 189                         "       sc.d %1, %z4, %2\n"                     \
 190                         "       bnez %1, 0b\n"                          \
 191                         "1:\n"                                          \
 192                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 193                         : "rJ" (__old), "rJ" (__new)                    \
 194                         : "memory");                                    \
 195                 break;                                                  \
 196         default:                                                        \
 197                 BUILD_BUG();                                            \
 198         }                                                               \
 199         __ret;                                                          \
 200 })
 201
 202 #define arch_cmpxchg_relaxed(ptr, o, n)                                 \
 203 ({                                                                      \
 204         __typeof__(*(ptr)) _o_ = (o);                                   \
 205         __typeof__(*(ptr)) _n_ = (n);                                   \
 206         (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),                   \
 207                                         _o_, _n_, sizeof(*(ptr)));      \
 208 })
 209
 210 #define __cmpxchg_acquire(ptr, old, new, size)                          \
 211 ({                                                                      \
 212         __typeof__(ptr) __ptr = (ptr);                                  \
 213         __typeof__(*(ptr)) __old = (old);                               \
 214         __typeof__(*(ptr)) __new = (new);                               \
 215         __typeof__(*(ptr)) __ret;                                       \
 216         register unsigned int __rc;                                     \
 217         switch (size) {                                                 \
 218         case 4:                                                         \
 219                 __asm__ __volatile__ (                                  \
 220                         "0:     lr.w %0, %2\n"                          \
 221                         "       bne  %0, %z3, 1f\n"                     \
 222                         "       sc.w %1, %z4, %2\n"                     \
 223                         "       bnez %1, 0b\n"                          \
 224                         RISCV_ACQUIRE_BARRIER                           \
 225                         "1:\n"                                          \
 226                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 227                         : "rJ" ((long)__old), "rJ" (__new)              \
 228                         : "memory");                                    \
 229                 break;                                                  \
 230         case 8:                                                         \
 231                 __asm__ __volatile__ (                                  \
 232                         "0:     lr.d %0, %2\n"                          \
 233                         "       bne %0, %z3, 1f\n"                      \
 234                         "       sc.d %1, %z4, %2\n"                     \
 235                         "       bnez %1, 0b\n"                          \
 236                         RISCV_ACQUIRE_BARRIER                           \
 237                         "1:\n"                                          \
 238                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 239                         : "rJ" (__old), "rJ" (__new)                    \
 240                         : "memory");                                    \
 241                 break;                                                  \
 242         default:                                                        \
 243                 BUILD_BUG();                                            \
 244         }                                                               \
 245         __ret;                                                          \
 246 })
 247
 248 #define arch_cmpxchg_acquire(ptr, o, n)                                 \
 249 ({                                                                      \
 250         __typeof__(*(ptr)) _o_ = (o);                                   \
 251         __typeof__(*(ptr)) _n_ = (n);                                   \
 252         (__typeof__(*(ptr))) __cmpxchg_acquire((ptr),                   \
 253                                         _o_, _n_, sizeof(*(ptr)));      \
 254 })
 255
 256 #define __cmpxchg_release(ptr, old, new, size)                          \
 257 ({                                                                      \
 258         __typeof__(ptr) __ptr = (ptr);                                  \
 259         __typeof__(*(ptr)) __old = (old);                               \
 260         __typeof__(*(ptr)) __new = (new);                               \
 261         __typeof__(*(ptr)) __ret;                                       \
 262         register unsigned int __rc;                                     \
 263         switch (size) {                                                 \
 264         case 4:                                                         \
 265                 __asm__ __volatile__ (                                  \
 266                         RISCV_RELEASE_BARRIER                           \
 267                         "0:     lr.w %0, %2\n"                          \
 268                         "       bne  %0, %z3, 1f\n"                     \
 269                         "       sc.w %1, %z4, %2\n"                     \
 270                         "       bnez %1, 0b\n"                          \
 271                         "1:\n"                                          \
 272                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 273                         : "rJ" ((long)__old), "rJ" (__new)              \
 274                         : "memory");                                    \
 275                 break;                                                  \
 276         case 8:                                                         \
 277                 __asm__ __volatile__ (                                  \
 278                         RISCV_RELEASE_BARRIER                           \
 279                         "0:     lr.d %0, %2\n"                          \
 280                         "       bne %0, %z3, 1f\n"                      \
 281                         "       sc.d %1, %z4, %2\n"                     \
 282                         "       bnez %1, 0b\n"                          \
 283                         "1:\n"                                          \
 284                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 285                         : "rJ" (__old), "rJ" (__new)                    \
 286                         : "memory");                                    \
 287                 break;                                                  \
 288         default:                                                        \
 289                 BUILD_BUG();                                            \
 290         }                                                               \
 291         __ret;                                                          \
 292 })
 293
 294 #define arch_cmpxchg_release(ptr, o, n)                                 \
 295 ({                                                                      \
 296         __typeof__(*(ptr)) _o_ = (o);                                   \
 297         __typeof__(*(ptr)) _n_ = (n);                                   \
 298         (__typeof__(*(ptr))) __cmpxchg_release((ptr),                   \
 299                                         _o_, _n_, sizeof(*(ptr)));      \
 300 })
 301
 302 #define __cmpxchg(ptr, old, new, size)                                  \
 303 ({                                                                      \
 304         __typeof__(ptr) __ptr = (ptr);                                  \
 305         __typeof__(*(ptr)) __old = (old);                               \
 306         __typeof__(*(ptr)) __new = (new);                               \
 307         __typeof__(*(ptr)) __ret;                                       \
 308         register unsigned int __rc;                                     \
 309         switch (size) {                                                 \
 310         case 4:                                                         \
 311                 __asm__ __volatile__ (                                  \
 312                         "0:     lr.w %0, %2\n"                          \
 313                         "       bne  %0, %z3, 1f\n"                     \
 314                         "       sc.w.rl %1, %z4, %2\n"                  \
 315                         "       bnez %1, 0b\n"                          \
 316                         RISCV_FULL_BARRIER                              \
 317                         "1:\n"                                          \
 318                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 319                         : "rJ" ((long)__old), "rJ" (__new)              \
 320                         : "memory");                                    \
 321                 break;                                                  \
 322         case 8:                                                         \
 323                 __asm__ __volatile__ (                                  \
 324                         "0:     lr.d %0, %2\n"                          \
 325                         "       bne %0, %z3, 1f\n"                      \
 326                         "       sc.d.rl %1, %z4, %2\n"                  \
 327                         "       bnez %1, 0b\n"                          \
 328                         RISCV_FULL_BARRIER                              \
 329                         "1:\n"                                          \
 330                         : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
 331                         : "rJ" (__old), "rJ" (__new)                    \
 332                         : "memory");                                    \
 333                 break;                                                  \
 334         default:                                                        \
 335                 BUILD_BUG();                                            \
 336         }                                                               \
 337         __ret;                                                          \
 338 })
 339
 340 #define arch_cmpxchg(ptr, o, n)                                         \
 341 ({                                                                      \
 342         __typeof__(*(ptr)) _o_ = (o);                                   \
 343         __typeof__(*(ptr)) _n_ = (n);                                   \
 344         (__typeof__(*(ptr))) __cmpxchg((ptr),                           \
 345                                        _o_, _n_, sizeof(*(ptr)));       \
 346 })
 347
 348 #define arch_cmpxchg_local(ptr, o, n)                                   \
 349         (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
 350
 351 #define arch_cmpxchg64(ptr, o, n)                                       \
 352 ({                                                                      \
 353         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 354         arch_cmpxchg((ptr), (o), (n));                                  \
 355 })
 356
 357 #define arch_cmpxchg64_local(ptr, o, n)                                 \
 358 ({                                                                      \
 359         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 360         arch_cmpxchg_relaxed((ptr), (o), (n));                          \
 361 })
 362
 363 #endif /* _ASM_RISCV_CMPXCHG_H */