arch/loongarch/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Atomic operations.
   4  *
   5  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   6  */
   7 #ifndef _ASM_ATOMIC_H
   8 #define _ASM_ATOMIC_H
   9
  10 #include <linux/types.h>
  11 #include <asm/barrier.h>
  12 #include <asm/cmpxchg.h>
  13 #include <asm/compiler.h>
  14
  15 #if __SIZEOF_LONG__ == 4
  16 #define __LL            "ll.w   "
  17 #define __SC            "sc.w   "
  18 #define __AMADD         "amadd.w        "
  19 #define __AMAND_DB      "amand_db.w     "
  20 #define __AMOR_DB       "amor_db.w      "
  21 #define __AMXOR_DB      "amxor_db.w     "
  22 #elif __SIZEOF_LONG__ == 8
  23 #define __LL            "ll.d   "
  24 #define __SC            "sc.d   "
  25 #define __AMADD         "amadd.d        "
  26 #define __AMAND_DB      "amand_db.d     "
  27 #define __AMOR_DB       "amor_db.d      "
  28 #define __AMXOR_DB      "amxor_db.d     "
  29 #endif
  30
  31 #define ATOMIC_INIT(i)    { (i) }
  32
  33 /*
  34  * arch_atomic_read - read atomic variable
  35  * @v: pointer of type atomic_t
  36  *
  37  * Atomically reads the value of @v.
  38  */
  39 #define arch_atomic_read(v)     READ_ONCE((v)->counter)
  40
  41 /*
  42  * arch_atomic_set - set atomic variable
  43  * @v: pointer of type atomic_t
  44  * @i: required value
  45  *
  46  * Atomically sets the value of @v to @i.
  47  */
  48 #define arch_atomic_set(v, i)   WRITE_ONCE((v)->counter, (i))
  49
  50 #define ATOMIC_OP(op, I, asm_op)                                        \
  51 static inline void arch_atomic_##op(int i, atomic_t *v)                 \
  52 {                                                                       \
  53         __asm__ __volatile__(                                           \
  54         "am"#asm_op"_db.w" " $zero, %1, %0      \n"                     \
  55         : "+ZB" (v->counter)                                            \
  56         : "r" (I)                                                       \
  57         : "memory");                                                    \
  58 }
  59
  60 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op)                           \
  61 static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \
  62 {                                                                       \
  63         int result;                                                     \
  64                                                                         \
  65         __asm__ __volatile__(                                           \
  66         "am"#asm_op"_db.w" " %1, %2, %0         \n"                     \
  67         : "+ZB" (v->counter), "=&r" (result)                            \
  68         : "r" (I)                                                       \
  69         : "memory");                                                    \
  70                                                                         \
  71         return result c_op I;                                           \
  72 }
  73
  74 #define ATOMIC_FETCH_OP(op, I, asm_op)                                  \
  75 static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)  \
  76 {                                                                       \
  77         int result;                                                     \
  78                                                                         \
  79         __asm__ __volatile__(                                           \
  80         "am"#asm_op"_db.w" " %1, %2, %0         \n"                     \
  81         : "+ZB" (v->counter), "=&r" (result)                            \
  82         : "r" (I)                                                       \
  83         : "memory");                                                    \
  84                                                                         \
  85         return result;                                                  \
  86 }
  87
  88 #define ATOMIC_OPS(op, I, asm_op, c_op)                                 \
  89         ATOMIC_OP(op, I, asm_op)                                        \
  90         ATOMIC_OP_RETURN(op, I, asm_op, c_op)                           \
  91         ATOMIC_FETCH_OP(op, I, asm_op)
  92
  93 ATOMIC_OPS(add, i, add, +)
  94 ATOMIC_OPS(sub, -i, add, +)
  95
  96 #define arch_atomic_add_return_relaxed  arch_atomic_add_return_relaxed
  97 #define arch_atomic_sub_return_relaxed  arch_atomic_sub_return_relaxed
  98 #define arch_atomic_fetch_add_relaxed   arch_atomic_fetch_add_relaxed
  99 #define arch_atomic_fetch_sub_relaxed   arch_atomic_fetch_sub_relaxed
 100
 101 #undef ATOMIC_OPS
 102
 103 #define ATOMIC_OPS(op, I, asm_op)                                       \
 104         ATOMIC_OP(op, I, asm_op)                                        \
 105         ATOMIC_FETCH_OP(op, I, asm_op)
 106
 107 ATOMIC_OPS(and, i, and)
 108 ATOMIC_OPS(or, i, or)
 109 ATOMIC_OPS(xor, i, xor)
 110
 111 #define arch_atomic_fetch_and_relaxed   arch_atomic_fetch_and_relaxed
 112 #define arch_atomic_fetch_or_relaxed    arch_atomic_fetch_or_relaxed
 113 #define arch_atomic_fetch_xor_relaxed   arch_atomic_fetch_xor_relaxed
 114
 115 #undef ATOMIC_OPS
 116 #undef ATOMIC_FETCH_OP
 117 #undef ATOMIC_OP_RETURN
 118 #undef ATOMIC_OP
 119
 120 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
 121 {
 122        int prev, rc;
 123
 124         __asm__ __volatile__ (
 125                 "0:     ll.w    %[p],  %[c]\n"
 126                 "       beq     %[p],  %[u], 1f\n"
 127                 "       add.w   %[rc], %[p], %[a]\n"
 128                 "       sc.w    %[rc], %[c]\n"
 129                 "       beqz    %[rc], 0b\n"
 130                 "       b       2f\n"
 131                 "1:\n"
 132                 __WEAK_LLSC_MB
 133                 "2:\n"
 134                 : [p]"=&r" (prev), [rc]"=&r" (rc),
 135                   [c]"=ZB" (v->counter)
 136                 : [a]"r" (a), [u]"r" (u)
 137                 : "memory");
 138
 139         return prev;
 140 }
 141 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
 142
 143 /*
 144  * arch_atomic_sub_if_positive - conditionally subtract integer from atomic variable
 145  * @i: integer value to subtract
 146  * @v: pointer of type atomic_t
 147  *
 148  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 149  * The function returns the old value of @v minus @i.
 150  */
 151 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
 152 {
 153         int result;
 154         int temp;
 155
 156         if (__builtin_constant_p(i)) {
 157                 __asm__ __volatile__(
 158                 "1:     ll.w    %1, %2          # atomic_sub_if_positive\n"
 159                 "       addi.w  %0, %1, %3                              \n"
 160                 "       or      %1, %0, $zero                           \n"
 161                 "       blt     %0, $zero, 2f                           \n"
 162                 "       sc.w    %1, %2                                  \n"
 163                 "       beq     $zero, %1, 1b                           \n"
 164                 "2:                                                     \n"
 165                 __WEAK_LLSC_MB
 166                 : "=&r" (result), "=&r" (temp),
 167                   "+" GCC_OFF_SMALL_ASM() (v->counter)
 168                 : "I" (-i));
 169         } else {
 170                 __asm__ __volatile__(
 171                 "1:     ll.w    %1, %2          # atomic_sub_if_positive\n"
 172                 "       sub.w   %0, %1, %3                              \n"
 173                 "       or      %1, %0, $zero                           \n"
 174                 "       blt     %0, $zero, 2f                           \n"
 175                 "       sc.w    %1, %2                                  \n"
 176                 "       beq     $zero, %1, 1b                           \n"
 177                 "2:                                                     \n"
 178                 __WEAK_LLSC_MB
 179                 : "=&r" (result), "=&r" (temp),
 180                   "+" GCC_OFF_SMALL_ASM() (v->counter)
 181                 : "r" (i));
 182         }
 183
 184         return result;
 185 }
 186
 187 #define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
 188 #define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
 189
 190 /*
 191  * arch_atomic_dec_if_positive - decrement by 1 if old value positive
 192  * @v: pointer of type atomic_t
 193  */
 194 #define arch_atomic_dec_if_positive(v)  arch_atomic_sub_if_positive(1, v)
 195
 196 #ifdef CONFIG_64BIT
 197
 198 #define ATOMIC64_INIT(i)    { (i) }
 199
 200 /*
 201  * arch_atomic64_read - read atomic variable
 202  * @v: pointer of type atomic64_t
 203  *
 204  */
 205 #define arch_atomic64_read(v)   READ_ONCE((v)->counter)
 206
 207 /*
 208  * arch_atomic64_set - set atomic variable
 209  * @v: pointer of type atomic64_t
 210  * @i: required value
 211  */
 212 #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
 213
 214 #define ATOMIC64_OP(op, I, asm_op)                                      \
 215 static inline void arch_atomic64_##op(long i, atomic64_t *v)            \
 216 {                                                                       \
 217         __asm__ __volatile__(                                           \
 218         "am"#asm_op"_db.d " " $zero, %1, %0     \n"                     \
 219         : "+ZB" (v->counter)                                            \
 220         : "r" (I)                                                       \
 221         : "memory");                                                    \
 222 }
 223
 224 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op)                                 \
 225 static inline long arch_atomic64_##op##_return_relaxed(long i, atomic64_t *v)   \
 226 {                                                                               \
 227         long result;                                                            \
 228         __asm__ __volatile__(                                                   \
 229         "am"#asm_op"_db.d " " %1, %2, %0                \n"                     \
 230         : "+ZB" (v->counter), "=&r" (result)                                    \
 231         : "r" (I)                                                               \
 232         : "memory");                                                            \
 233                                                                                 \
 234         return result c_op I;                                                   \
 235 }
 236
 237 #define ATOMIC64_FETCH_OP(op, I, asm_op)                                        \
 238 static inline long arch_atomic64_fetch_##op##_relaxed(long i, atomic64_t *v)    \
 239 {                                                                               \
 240         long result;                                                            \
 241                                                                                 \
 242         __asm__ __volatile__(                                                   \
 243         "am"#asm_op"_db.d " " %1, %2, %0                \n"                     \
 244         : "+ZB" (v->counter), "=&r" (result)                                    \
 245         : "r" (I)                                                               \
 246         : "memory");                                                            \
 247                                                                                 \
 248         return result;                                                          \
 249 }
 250
 251 #define ATOMIC64_OPS(op, I, asm_op, c_op)                                     \
 252         ATOMIC64_OP(op, I, asm_op)                                            \
 253         ATOMIC64_OP_RETURN(op, I, asm_op, c_op)                               \
 254         ATOMIC64_FETCH_OP(op, I, asm_op)
 255
 256 ATOMIC64_OPS(add, i, add, +)
 257 ATOMIC64_OPS(sub, -i, add, +)
 258
 259 #define arch_atomic64_add_return_relaxed        arch_atomic64_add_return_relaxed
 260 #define arch_atomic64_sub_return_relaxed        arch_atomic64_sub_return_relaxed
 261 #define arch_atomic64_fetch_add_relaxed         arch_atomic64_fetch_add_relaxed
 262 #define arch_atomic64_fetch_sub_relaxed         arch_atomic64_fetch_sub_relaxed
 263
 264 #undef ATOMIC64_OPS
 265
 266 #define ATOMIC64_OPS(op, I, asm_op)                                           \
 267         ATOMIC64_OP(op, I, asm_op)                                            \
 268         ATOMIC64_FETCH_OP(op, I, asm_op)
 269
 270 ATOMIC64_OPS(and, i, and)
 271 ATOMIC64_OPS(or, i, or)
 272 ATOMIC64_OPS(xor, i, xor)
 273
 274 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
 275 #define arch_atomic64_fetch_or_relaxed  arch_atomic64_fetch_or_relaxed
 276 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
 277
 278 #undef ATOMIC64_OPS
 279 #undef ATOMIC64_FETCH_OP
 280 #undef ATOMIC64_OP_RETURN
 281 #undef ATOMIC64_OP
 282
 283 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
 284 {
 285        long prev, rc;
 286
 287         __asm__ __volatile__ (
 288                 "0:     ll.d    %[p],  %[c]\n"
 289                 "       beq     %[p],  %[u], 1f\n"
 290                 "       add.d   %[rc], %[p], %[a]\n"
 291                 "       sc.d    %[rc], %[c]\n"
 292                 "       beqz    %[rc], 0b\n"
 293                 "       b       2f\n"
 294                 "1:\n"
 295                 __WEAK_LLSC_MB
 296                 "2:\n"
 297                 : [p]"=&r" (prev), [rc]"=&r" (rc),
 298                   [c] "=ZB" (v->counter)
 299                 : [a]"r" (a), [u]"r" (u)
 300                 : "memory");
 301
 302         return prev;
 303 }
 304 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 305
 306 /*
 307  * arch_atomic64_sub_if_positive - conditionally subtract integer from atomic variable
 308  * @i: integer value to subtract
 309  * @v: pointer of type atomic64_t
 310  *
 311  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 312  * The function returns the old value of @v minus @i.
 313  */
 314 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
 315 {
 316         long result;
 317         long temp;
 318
 319         if (__builtin_constant_p(i)) {
 320                 __asm__ __volatile__(
 321                 "1:     ll.d    %1, %2  # atomic64_sub_if_positive      \n"
 322                 "       addi.d  %0, %1, %3                              \n"
 323                 "       or      %1, %0, $zero                           \n"
 324                 "       blt     %0, $zero, 2f                           \n"
 325                 "       sc.d    %1, %2                                  \n"
 326                 "       beq     %1, $zero, 1b                           \n"
 327                 "2:                                                     \n"
 328                 __WEAK_LLSC_MB
 329                 : "=&r" (result), "=&r" (temp),
 330                   "+" GCC_OFF_SMALL_ASM() (v->counter)
 331                 : "I" (-i));
 332         } else {
 333                 __asm__ __volatile__(
 334                 "1:     ll.d    %1, %2  # atomic64_sub_if_positive      \n"
 335                 "       sub.d   %0, %1, %3                              \n"
 336                 "       or      %1, %0, $zero                           \n"
 337                 "       blt     %0, $zero, 2f                           \n"
 338                 "       sc.d    %1, %2                                  \n"
 339                 "       beq     %1, $zero, 1b                           \n"
 340                 "2:                                                     \n"
 341                 __WEAK_LLSC_MB
 342                 : "=&r" (result), "=&r" (temp),
 343                   "+" GCC_OFF_SMALL_ASM() (v->counter)
 344                 : "r" (i));
 345         }
 346
 347         return result;
 348 }
 349
 350 #define arch_atomic64_cmpxchg(v, o, n) \
 351         ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
 352 #define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
 353
 354 /*
 355  * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
 356  * @v: pointer of type atomic64_t
 357  */
 358 #define arch_atomic64_dec_if_positive(v)        arch_atomic64_sub_if_positive(1, v)
 359
 360 #endif /* CONFIG_64BIT */
 361
 362 #endif /* _ASM_ATOMIC_H */