arch/arc/include/asm/atomic64-arcv2.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2
   3 /*
   4  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
   5  *  - The address HAS to be 64-bit aligned
   6  */
   7
   8 #ifndef _ASM_ARC_ATOMIC64_ARCV2_H
   9 #define _ASM_ARC_ATOMIC64_ARCV2_H
  10
  11 typedef struct {
  12         s64 __aligned(8) counter;
  13 } atomic64_t;
  14
  15 #define ATOMIC64_INIT(a) { (a) }
  16
  17 static inline s64 arch_atomic64_read(const atomic64_t *v)
  18 {
  19         s64 val;
  20
  21         __asm__ __volatile__(
  22         "       ldd   %0, [%1]  \n"
  23         : "=r"(val)
  24         : "r"(&v->counter));
  25
  26         return val;
  27 }
  28
  29 static inline void arch_atomic64_set(atomic64_t *v, s64 a)
  30 {
  31         /*
  32          * This could have been a simple assignment in "C" but would need
  33          * explicit volatile. Otherwise gcc optimizers could elide the store
  34          * which borked atomic64 self-test
  35          * In the inline asm version, memory clobber needed for exact same
  36          * reason, to tell gcc about the store.
  37          *
  38          * This however is not needed for sibling atomic64_add() etc since both
  39          * load/store are explicitly done in inline asm. As long as API is used
  40          * for each access, gcc has no way to optimize away any load/store
  41          */
  42         __asm__ __volatile__(
  43         "       std   %0, [%1]  \n"
  44         :
  45         : "r"(a), "r"(&v->counter)
  46         : "memory");
  47 }
  48
  49 #define ATOMIC64_OP(op, op1, op2)                                       \
  50 static inline void arch_atomic64_##op(s64 a, atomic64_t *v)             \
  51 {                                                                       \
  52         s64 val;                                                        \
  53                                                                         \
  54         __asm__ __volatile__(                                           \
  55         "1:                             \n"                             \
  56         "       llockd  %0, [%1]        \n"                             \
  57         "       " #op1 " %L0, %L0, %L2  \n"                             \
  58         "       " #op2 " %H0, %H0, %H2  \n"                             \
  59         "       scondd   %0, [%1]       \n"                             \
  60         "       bnz     1b              \n"                             \
  61         : "=&r"(val)                                                    \
  62         : "r"(&v->counter), "ir"(a)                                     \
  63         : "cc");                                                        \
  64 }                                                                       \
  65
  66 #define ATOMIC64_OP_RETURN(op, op1, op2)                                \
  67 static inline s64 arch_atomic64_##op##_return(s64 a, atomic64_t *v)     \
  68 {                                                                       \
  69         s64 val;                                                        \
  70                                                                         \
  71         smp_mb();                                                       \
  72                                                                         \
  73         __asm__ __volatile__(                                           \
  74         "1:                             \n"                             \
  75         "       llockd   %0, [%1]       \n"                             \
  76         "       " #op1 " %L0, %L0, %L2  \n"                             \
  77         "       " #op2 " %H0, %H0, %H2  \n"                             \
  78         "       scondd   %0, [%1]       \n"                             \
  79         "       bnz     1b              \n"                             \
  80         : [val] "=&r"(val)                                              \
  81         : "r"(&v->counter), "ir"(a)                                     \
  82         : "cc");        /* memory clobber comes from smp_mb() */        \
  83                                                                         \
  84         smp_mb();                                                       \
  85                                                                         \
  86         return val;                                                     \
  87 }
  88
  89 #define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
  90 static inline s64 arch_atomic64_fetch_##op(s64 a, atomic64_t *v)        \
  91 {                                                                       \
  92         s64 val, orig;                                                  \
  93                                                                         \
  94         smp_mb();                                                       \
  95                                                                         \
  96         __asm__ __volatile__(                                           \
  97         "1:                             \n"                             \
  98         "       llockd   %0, [%2]       \n"                             \
  99         "       " #op1 " %L1, %L0, %L3  \n"                             \
 100         "       " #op2 " %H1, %H0, %H3  \n"                             \
 101         "       scondd   %1, [%2]       \n"                             \
 102         "       bnz     1b              \n"                             \
 103         : "=&r"(orig), "=&r"(val)                                       \
 104         : "r"(&v->counter), "ir"(a)                                     \
 105         : "cc");        /* memory clobber comes from smp_mb() */        \
 106                                                                         \
 107         smp_mb();                                                       \
 108                                                                         \
 109         return orig;                                                    \
 110 }
 111
 112 #define ATOMIC64_OPS(op, op1, op2)                                      \
 113         ATOMIC64_OP(op, op1, op2)                                       \
 114         ATOMIC64_OP_RETURN(op, op1, op2)                                \
 115         ATOMIC64_FETCH_OP(op, op1, op2)
 116
 117 ATOMIC64_OPS(add, add.f, adc)
 118 ATOMIC64_OPS(sub, sub.f, sbc)
 119 ATOMIC64_OPS(and, and, and)
 120 ATOMIC64_OPS(andnot, bic, bic)
 121 ATOMIC64_OPS(or, or, or)
 122 ATOMIC64_OPS(xor, xor, xor)
 123
 124 #define arch_atomic64_andnot            arch_atomic64_andnot
 125 #define arch_atomic64_fetch_andnot      arch_atomic64_fetch_andnot
 126
 127 #undef ATOMIC64_OPS
 128 #undef ATOMIC64_FETCH_OP
 129 #undef ATOMIC64_OP_RETURN
 130 #undef ATOMIC64_OP
 131
 132 static inline s64
 133 arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)
 134 {
 135         s64 prev;
 136
 137         smp_mb();
 138
 139         __asm__ __volatile__(
 140         "1:     llockd  %0, [%1]        \n"
 141         "       brne    %L0, %L2, 2f    \n"
 142         "       brne    %H0, %H2, 2f    \n"
 143         "       scondd  %3, [%1]        \n"
 144         "       bnz     1b              \n"
 145         "2:                             \n"
 146         : "=&r"(prev)
 147         : "r"(ptr), "ir"(expected), "r"(new)
 148         : "cc");        /* memory clobber comes from smp_mb() */
 149
 150         smp_mb();
 151
 152         return prev;
 153 }
 154
 155 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
 156 {
 157         s64 prev;
 158
 159         smp_mb();
 160
 161         __asm__ __volatile__(
 162         "1:     llockd  %0, [%1]        \n"
 163         "       scondd  %2, [%1]        \n"
 164         "       bnz     1b              \n"
 165         "2:                             \n"
 166         : "=&r"(prev)
 167         : "r"(ptr), "r"(new)
 168         : "cc");        /* memory clobber comes from smp_mb() */
 169
 170         smp_mb();
 171
 172         return prev;
 173 }
 174
 175 /**
 176  * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
 177  * @v: pointer of type atomic64_t
 178  *
 179  * The function returns the old value of *v minus 1, even if
 180  * the atomic variable, v, was not decremented.
 181  */
 182
 183 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 184 {
 185         s64 val;
 186
 187         smp_mb();
 188
 189         __asm__ __volatile__(
 190         "1:     llockd  %0, [%1]        \n"
 191         "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 192         "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 193         "       brlt    %H0, 0, 2f      \n"
 194         "       scondd  %0, [%1]        \n"
 195         "       bnz     1b              \n"
 196         "2:                             \n"
 197         : "=&r"(val)
 198         : "r"(&v->counter)
 199         : "cc");        /* memory clobber comes from smp_mb() */
 200
 201         smp_mb();
 202
 203         return val;
 204 }
 205 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
 206
 207 /**
 208  * arch_atomic64_fetch_add_unless - add unless the number is a given value
 209  * @v: pointer of type atomic64_t
 210  * @a: the amount to add to v...
 211  * @u: ...unless v is equal to u.
 212  *
 213  * Atomically adds @a to @v, if it was not @u.
 214  * Returns the old value of @v
 215  */
 216 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 217 {
 218         s64 old, temp;
 219
 220         smp_mb();
 221
 222         __asm__ __volatile__(
 223         "1:     llockd  %0, [%2]        \n"
 224         "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 225         "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 226         "2:                             \n"
 227         "       add.f   %L1, %L0, %L3   \n"
 228         "       adc     %H1, %H0, %H3   \n"
 229         "       scondd  %1, [%2]        \n"
 230         "       bnz     1b              \n"
 231         "3:                             \n"
 232         : "=&r"(old), "=&r" (temp)
 233         : "r"(&v->counter), "r"(a), "r"(u)
 234         : "cc");        /* memory clobber comes from smp_mb() */
 235
 236         smp_mb();
 237
 238         return old;
 239 }
 240 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 241
 242 #endif