arch/arc/include/asm/atomic64-arcv2.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2
   3 /*
   4  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
   5  *  - The address HAS to be 64-bit aligned
   6  */
   7
   8 #ifndef _ASM_ARC_ATOMIC64_ARCV2_H
   9 #define _ASM_ARC_ATOMIC64_ARCV2_H
  10
  11 typedef struct {
  12         s64 __aligned(8) counter;
  13 } atomic64_t;
  14
  15 #define ATOMIC64_INIT(a) { (a) }
  16
  17 static inline s64 arch_atomic64_read(const atomic64_t *v)
  18 {
  19         s64 val;
  20
  21         __asm__ __volatile__(
  22         "       ldd   %0, [%1]  \n"
  23         : "=r"(val)
  24         : "r"(&v->counter));
  25
  26         return val;
  27 }
  28
  29 static inline void arch_atomic64_set(atomic64_t *v, s64 a)
  30 {
  31         /*
  32          * This could have been a simple assignment in "C" but would need
  33          * explicit volatile. Otherwise gcc optimizers could elide the store
  34          * which borked atomic64 self-test
  35          * In the inline asm version, memory clobber needed for exact same
  36          * reason, to tell gcc about the store.
  37          *
  38          * This however is not needed for sibling atomic64_add() etc since both
  39          * load/store are explicitly done in inline asm. As long as API is used
  40          * for each access, gcc has no way to optimize away any load/store
  41          */
  42         __asm__ __volatile__(
  43         "       std   %0, [%1]  \n"
  44         :
  45         : "r"(a), "r"(&v->counter)
  46         : "memory");
  47 }
  48
  49 #define ATOMIC64_OP(op, op1, op2)                                       \
  50 static inline void arch_atomic64_##op(s64 a, atomic64_t *v)             \
  51 {                                                                       \
  52         s64 val;                                                        \
  53                                                                         \
  54         __asm__ __volatile__(                                           \
  55         "1:                             \n"                             \
  56         "       llockd  %0, [%1]        \n"                             \
  57         "       " #op1 " %L0, %L0, %L2  \n"                             \
  58         "       " #op2 " %H0, %H0, %H2  \n"                             \
  59         "       scondd   %0, [%1]       \n"                             \
  60         "       bnz     1b              \n"                             \
  61         : "=&r"(val)                                                    \
  62         : "r"(&v->counter), "ir"(a)                                     \
  63         : "cc");                                                        \
  64 }                                                                       \
  65
  66 #define ATOMIC64_OP_RETURN(op, op1, op2)                                \
  67 static inline s64 arch_atomic64_##op##_return(s64 a, atomic64_t *v)     \
  68 {                                                                       \
  69         s64 val;                                                        \
  70                                                                         \
  71         smp_mb();                                                       \
  72                                                                         \
  73         __asm__ __volatile__(                                           \
  74         "1:                             \n"                             \
  75         "       llockd   %0, [%1]       \n"                             \
  76         "       " #op1 " %L0, %L0, %L2  \n"                             \
  77         "       " #op2 " %H0, %H0, %H2  \n"                             \
  78         "       scondd   %0, [%1]       \n"                             \
  79         "       bnz     1b              \n"                             \
  80         : [val] "=&r"(val)                                              \
  81         : "r"(&v->counter), "ir"(a)                                     \
  82         : "cc");        /* memory clobber comes from smp_mb() */        \
  83                                                                         \
  84         smp_mb();                                                       \
  85                                                                         \
  86         return val;                                                     \
  87 }
  88
  89 #define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
  90 static inline s64 arch_atomic64_fetch_##op(s64 a, atomic64_t *v)        \
  91 {                                                                       \
  92         s64 val, orig;                                                  \
  93                                                                         \
  94         smp_mb();                                                       \
  95                                                                         \
  96         __asm__ __volatile__(                                           \
  97         "1:                             \n"                             \
  98         "       llockd   %0, [%2]       \n"                             \
  99         "       " #op1 " %L1, %L0, %L3  \n"                             \
 100         "       " #op2 " %H1, %H0, %H3  \n"                             \
 101         "       scondd   %1, [%2]       \n"                             \
 102         "       bnz     1b              \n"                             \
 103         : "=&r"(orig), "=&r"(val)                                       \
 104         : "r"(&v->counter), "ir"(a)                                     \
 105         : "cc");        /* memory clobber comes from smp_mb() */        \
 106                                                                         \
 107         smp_mb();                                                       \
 108                                                                         \
 109         return orig;                                                    \
 110 }
 111
 112 #define ATOMIC64_OPS(op, op1, op2)                                      \
 113         ATOMIC64_OP(op, op1, op2)                                       \
 114         ATOMIC64_OP_RETURN(op, op1, op2)                                \
 115         ATOMIC64_FETCH_OP(op, op1, op2)
 116
 117 ATOMIC64_OPS(add, add.f, adc)
 118 ATOMIC64_OPS(sub, sub.f, sbc)
 119
 120 #undef ATOMIC64_OPS
 121 #define ATOMIC64_OPS(op, op1, op2)                                      \
 122         ATOMIC64_OP(op, op1, op2)                                       \
 123         ATOMIC64_FETCH_OP(op, op1, op2)
 124
 125 ATOMIC64_OPS(and, and, and)
 126 ATOMIC64_OPS(andnot, bic, bic)
 127 ATOMIC64_OPS(or, or, or)
 128 ATOMIC64_OPS(xor, xor, xor)
 129
 130 #define arch_atomic64_andnot            arch_atomic64_andnot
 131 #define arch_atomic64_fetch_andnot      arch_atomic64_fetch_andnot
 132
 133 #undef ATOMIC64_OPS
 134 #undef ATOMIC64_FETCH_OP
 135 #undef ATOMIC64_OP_RETURN
 136 #undef ATOMIC64_OP
 137
 138 static inline s64
 139 arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)
 140 {
 141         s64 prev;
 142
 143         smp_mb();
 144
 145         __asm__ __volatile__(
 146         "1:     llockd  %0, [%1]        \n"
 147         "       brne    %L0, %L2, 2f    \n"
 148         "       brne    %H0, %H2, 2f    \n"
 149         "       scondd  %3, [%1]        \n"
 150         "       bnz     1b              \n"
 151         "2:                             \n"
 152         : "=&r"(prev)
 153         : "r"(ptr), "ir"(expected), "r"(new)
 154         : "cc");        /* memory clobber comes from smp_mb() */
 155
 156         smp_mb();
 157
 158         return prev;
 159 }
 160
 161 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
 162 {
 163         s64 prev;
 164
 165         smp_mb();
 166
 167         __asm__ __volatile__(
 168         "1:     llockd  %0, [%1]        \n"
 169         "       scondd  %2, [%1]        \n"
 170         "       bnz     1b              \n"
 171         "2:                             \n"
 172         : "=&r"(prev)
 173         : "r"(ptr), "r"(new)
 174         : "cc");        /* memory clobber comes from smp_mb() */
 175
 176         smp_mb();
 177
 178         return prev;
 179 }
 180
 181 /**
 182  * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
 183  * @v: pointer of type atomic64_t
 184  *
 185  * The function returns the old value of *v minus 1, even if
 186  * the atomic variable, v, was not decremented.
 187  */
 188
 189 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 190 {
 191         s64 val;
 192
 193         smp_mb();
 194
 195         __asm__ __volatile__(
 196         "1:     llockd  %0, [%1]        \n"
 197         "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 198         "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 199         "       brlt    %H0, 0, 2f      \n"
 200         "       scondd  %0, [%1]        \n"
 201         "       bnz     1b              \n"
 202         "2:                             \n"
 203         : "=&r"(val)
 204         : "r"(&v->counter)
 205         : "cc");        /* memory clobber comes from smp_mb() */
 206
 207         smp_mb();
 208
 209         return val;
 210 }
 211 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
 212
 213 /**
 214  * arch_atomic64_fetch_add_unless - add unless the number is a given value
 215  * @v: pointer of type atomic64_t
 216  * @a: the amount to add to v...
 217  * @u: ...unless v is equal to u.
 218  *
 219  * Atomically adds @a to @v, if it was not @u.
 220  * Returns the old value of @v
 221  */
 222 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 223 {
 224         s64 old, temp;
 225
 226         smp_mb();
 227
 228         __asm__ __volatile__(
 229         "1:     llockd  %0, [%2]        \n"
 230         "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 231         "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 232         "2:                             \n"
 233         "       add.f   %L1, %L0, %L3   \n"
 234         "       adc     %H1, %H0, %H3   \n"
 235         "       scondd  %1, [%2]        \n"
 236         "       bnz     1b              \n"
 237         "3:                             \n"
 238         : "=&r"(old), "=&r" (temp)
 239         : "r"(&v->counter), "r"(a), "r"(u)
 240         : "cc");        /* memory clobber comes from smp_mb() */
 241
 242         smp_mb();
 243
 244         return old;
 245 }
 246 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 247
 248 #endif