arch/s390/include/asm/atomic_ops.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Low level function for atomic operations
   4  *
   5  * Copyright IBM Corp. 1999, 2016
   6  */
   7
   8 #ifndef __ARCH_S390_ATOMIC_OPS__
   9 #define __ARCH_S390_ATOMIC_OPS__
  10
  11 #include <linux/limits.h>
  12 #include <asm/march.h>
  13
  14 static __always_inline int __atomic_read(const atomic_t *v)
  15 {
  16         int c;
  17
  18         asm volatile(
  19                 "       l       %[c],%[counter]\n"
  20                 : [c] "=d" (c) : [counter] "R" (v->counter));
  21         return c;
  22 }
  23
  24 static __always_inline void __atomic_set(atomic_t *v, int i)
  25 {
  26         if (__builtin_constant_p(i) && i >= S16_MIN && i <= S16_MAX) {
  27                 asm volatile(
  28                         "       mvhi    %[counter], %[i]\n"
  29                         : [counter] "=Q" (v->counter) : [i] "K" (i));
  30         } else {
  31                 asm volatile(
  32                         "       st      %[i],%[counter]\n"
  33                         : [counter] "=R" (v->counter) : [i] "d" (i));
  34         }
  35 }
  36
  37 static __always_inline s64 __atomic64_read(const atomic64_t *v)
  38 {
  39         s64 c;
  40
  41         asm volatile(
  42                 "       lg      %[c],%[counter]\n"
  43                 : [c] "=d" (c) : [counter] "RT" (v->counter));
  44         return c;
  45 }
  46
  47 static __always_inline void __atomic64_set(atomic64_t *v, s64 i)
  48 {
  49         if (__builtin_constant_p(i) && i >= S16_MIN && i <= S16_MAX) {
  50                 asm volatile(
  51                         "       mvghi   %[counter], %[i]\n"
  52                         : [counter] "=Q" (v->counter) : [i] "K" (i));
  53         } else {
  54                 asm volatile(
  55                         "       stg     %[i],%[counter]\n"
  56                         : [counter] "=RT" (v->counter) : [i] "d" (i));
  57         }
  58 }
  59
  60 #ifdef MARCH_HAS_Z196_FEATURES
  61
  62 #define __ATOMIC_OP(op_name, op_type, op_string, op_barrier)            \
  63 static __always_inline op_type op_name(op_type val, op_type *ptr)       \
  64 {                                                                       \
  65         op_type old;                                                    \
  66                                                                         \
  67         asm volatile(                                                   \
  68                 op_string "     %[old],%[val],%[ptr]\n"                 \
  69                 op_barrier                                              \
  70                 : [old] "=d" (old), [ptr] "+QS" (*ptr)                  \
  71                 : [val] "d" (val) : "cc", "memory");                    \
  72         return old;                                                     \
  73 }                                                                       \
  74
  75 #define __ATOMIC_OPS(op_name, op_type, op_string)                       \
  76         __ATOMIC_OP(op_name, op_type, op_string, "\n")                  \
  77         __ATOMIC_OP(op_name##_barrier, op_type, op_string, "bcr 14,0\n")
  78
  79 __ATOMIC_OPS(__atomic_add, int, "laa")
  80 __ATOMIC_OPS(__atomic_and, int, "lan")
  81 __ATOMIC_OPS(__atomic_or,  int, "lao")
  82 __ATOMIC_OPS(__atomic_xor, int, "lax")
  83
  84 __ATOMIC_OPS(__atomic64_add, long, "laag")
  85 __ATOMIC_OPS(__atomic64_and, long, "lang")
  86 __ATOMIC_OPS(__atomic64_or,  long, "laog")
  87 __ATOMIC_OPS(__atomic64_xor, long, "laxg")
  88
  89 #undef __ATOMIC_OPS
  90 #undef __ATOMIC_OP
  91
  92 #define __ATOMIC_CONST_OP(op_name, op_type, op_string, op_barrier)      \
  93 static __always_inline void op_name(op_type val, op_type *ptr)          \
  94 {                                                                       \
  95         asm volatile(                                                   \
  96                 op_string "     %[ptr],%[val]\n"                        \
  97                 op_barrier                                              \
  98                 : [ptr] "+QS" (*ptr) : [val] "i" (val) : "cc", "memory");\
  99 }
 100
 101 #define __ATOMIC_CONST_OPS(op_name, op_type, op_string)                 \
 102         __ATOMIC_CONST_OP(op_name, op_type, op_string, "\n")            \
 103         __ATOMIC_CONST_OP(op_name##_barrier, op_type, op_string, "bcr 14,0\n")
 104
 105 __ATOMIC_CONST_OPS(__atomic_add_const, int, "asi")
 106 __ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi")
 107
 108 #undef __ATOMIC_CONST_OPS
 109 #undef __ATOMIC_CONST_OP
 110
 111 #else /* MARCH_HAS_Z196_FEATURES */
 112
 113 #define __ATOMIC_OP(op_name, op_string)                                 \
 114 static __always_inline int op_name(int val, int *ptr)                   \
 115 {                                                                       \
 116         int old, new;                                                   \
 117                                                                         \
 118         asm volatile(                                                   \
 119                 "0:     lr      %[new],%[old]\n"                        \
 120                 op_string "     %[new],%[val]\n"                        \
 121                 "       cs      %[old],%[new],%[ptr]\n"                 \
 122                 "       jl      0b"                                     \
 123                 : [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
 124                 : [val] "d" (val), "0" (*ptr) : "cc", "memory");        \
 125         return old;                                                     \
 126 }
 127
 128 #define __ATOMIC_OPS(op_name, op_string)                                \
 129         __ATOMIC_OP(op_name, op_string)                                 \
 130         __ATOMIC_OP(op_name##_barrier, op_string)
 131
 132 __ATOMIC_OPS(__atomic_add, "ar")
 133 __ATOMIC_OPS(__atomic_and, "nr")
 134 __ATOMIC_OPS(__atomic_or,  "or")
 135 __ATOMIC_OPS(__atomic_xor, "xr")
 136
 137 #undef __ATOMIC_OPS
 138
 139 #define __ATOMIC64_OP(op_name, op_string)                               \
 140 static __always_inline long op_name(long val, long *ptr)                \
 141 {                                                                       \
 142         long old, new;                                                  \
 143                                                                         \
 144         asm volatile(                                                   \
 145                 "0:     lgr     %[new],%[old]\n"                        \
 146                 op_string "     %[new],%[val]\n"                        \
 147                 "       csg     %[old],%[new],%[ptr]\n"                 \
 148                 "       jl      0b"                                     \
 149                 : [old] "=d" (old), [new] "=&d" (new), [ptr] "+QS" (*ptr)\
 150                 : [val] "d" (val), "0" (*ptr) : "cc", "memory");        \
 151         return old;                                                     \
 152 }
 153
 154 #define __ATOMIC64_OPS(op_name, op_string)                              \
 155         __ATOMIC64_OP(op_name, op_string)                               \
 156         __ATOMIC64_OP(op_name##_barrier, op_string)
 157
 158 __ATOMIC64_OPS(__atomic64_add, "agr")
 159 __ATOMIC64_OPS(__atomic64_and, "ngr")
 160 __ATOMIC64_OPS(__atomic64_or,  "ogr")
 161 __ATOMIC64_OPS(__atomic64_xor, "xgr")
 162
 163 #undef __ATOMIC64_OPS
 164
 165 #define __atomic_add_const(val, ptr)            __atomic_add(val, ptr)
 166 #define __atomic_add_const_barrier(val, ptr)    __atomic_add(val, ptr)
 167 #define __atomic64_add_const(val, ptr)          __atomic64_add(val, ptr)
 168 #define __atomic64_add_const_barrier(val, ptr)  __atomic64_add(val, ptr)
 169
 170 #endif /* MARCH_HAS_Z196_FEATURES */
 171
 172 static __always_inline int __atomic_cmpxchg(int *ptr, int old, int new)
 173 {
 174         asm volatile(
 175                 "       cs      %[old],%[new],%[ptr]"
 176                 : [old] "+d" (old), [ptr] "+Q" (*ptr)
 177                 : [new] "d" (new)
 178                 : "cc", "memory");
 179         return old;
 180 }
 181
 182 static __always_inline long __atomic64_cmpxchg(long *ptr, long old, long new)
 183 {
 184         asm volatile(
 185                 "       csg     %[old],%[new],%[ptr]"
 186                 : [old] "+d" (old), [ptr] "+QS" (*ptr)
 187                 : [new] "d" (new)
 188                 : "cc", "memory");
 189         return old;
 190 }
 191
 192 /* GCC versions before 14.2.0 may die with an ICE in some configurations. */
 193 #if defined(__GCC_ASM_FLAG_OUTPUTS__) && !(IS_ENABLED(CONFIG_CC_IS_GCC) && (GCC_VERSION < 140200))
 194
 195 static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
 196 {
 197         int cc;
 198
 199         asm volatile(
 200                 "       cs      %[old],%[new],%[ptr]"
 201                 : [old] "+d" (old), [ptr] "+Q" (*ptr), "=@cc" (cc)
 202                 : [new] "d" (new)
 203                 : "memory");
 204         return cc == 0;
 205 }
 206
 207 static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)
 208 {
 209         int cc;
 210
 211         asm volatile(
 212                 "       csg     %[old],%[new],%[ptr]"
 213                 : [old] "+d" (old), [ptr] "+QS" (*ptr), "=@cc" (cc)
 214                 : [new] "d" (new)
 215                 : "memory");
 216         return cc == 0;
 217 }
 218
 219 #else /* __GCC_ASM_FLAG_OUTPUTS__ */
 220
 221 static __always_inline bool __atomic_cmpxchg_bool(int *ptr, int old, int new)
 222 {
 223         int old_expected = old;
 224
 225         asm volatile(
 226                 "       cs      %[old],%[new],%[ptr]"
 227                 : [old] "+d" (old), [ptr] "+Q" (*ptr)
 228                 : [new] "d" (new)
 229                 : "cc", "memory");
 230         return old == old_expected;
 231 }
 232
 233 static __always_inline bool __atomic64_cmpxchg_bool(long *ptr, long old, long new)
 234 {
 235         long old_expected = old;
 236
 237         asm volatile(
 238                 "       csg     %[old],%[new],%[ptr]"
 239                 : [old] "+d" (old), [ptr] "+QS" (*ptr)
 240                 : [new] "d" (new)
 241                 : "cc", "memory");
 242         return old == old_expected;
 243 }
 244
 245 #endif /* __GCC_ASM_FLAG_OUTPUTS__ */
 246
 247 #endif /* __ARCH_S390_ATOMIC_OPS__  */