arch/arc/include/asm/bitops.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   4  */
   5
   6 #ifndef _ASM_BITOPS_H
   7 #define _ASM_BITOPS_H
   8
   9 #ifndef _LINUX_BITOPS_H
  10 #error only <linux/bitops.h> can be included directly
  11 #endif
  12
  13 #ifndef __ASSEMBLY__
  14
  15 #include <linux/types.h>
  16 #include <linux/compiler.h>
  17 #include <asm/barrier.h>
  18 #ifndef CONFIG_ARC_HAS_LLSC
  19 #include <asm/smp.h>
  20 #endif
  21
  22 #ifdef CONFIG_ARC_HAS_LLSC
  23
  24 /*
  25  * Hardware assisted Atomic-R-M-W
  26  */
  27
  28 #define BIT_OP(op, c_op, asm_op)                                        \
  29 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
  30 {                                                                       \
  31         unsigned int temp;                                              \
  32                                                                         \
  33         m += nr >> 5;                                                   \
  34                                                                         \
  35         nr &= 0x1f;                                                     \
  36                                                                         \
  37         __asm__ __volatile__(                                           \
  38         "1:     llock       %0, [%1]            \n"                     \
  39         "       " #asm_op " %0, %0, %2  \n"                             \
  40         "       scond       %0, [%1]            \n"                     \
  41         "       bnz         1b                  \n"                     \
  42         : "=&r"(temp)   /* Early clobber, to prevent reg reuse */       \
  43         : "r"(m),       /* Not "m": llock only supports reg direct addr mode */ \
  44           "ir"(nr)                                                      \
  45         : "cc");                                                        \
  46 }
  47
  48 /*
  49  * Semantically:
  50  *    Test the bit
  51  *    if clear
  52  *        set it and return 0 (old value)
  53  *    else
  54  *        return 1 (old value).
  55  *
  56  * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally
  57  * and the old value of bit is returned
  58  */
  59 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
  60 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
  61 {                                                                       \
  62         unsigned long old, temp;                                        \
  63                                                                         \
  64         m += nr >> 5;                                                   \
  65                                                                         \
  66         nr &= 0x1f;                                                     \
  67                                                                         \
  68         /*                                                              \
  69          * Explicit full memory barrier needed before/after as          \
  70          * LLOCK/SCOND themselves don't provide any such smenatic       \
  71          */                                                             \
  72         smp_mb();                                                       \
  73                                                                         \
  74         __asm__ __volatile__(                                           \
  75         "1:     llock       %0, [%2]    \n"                             \
  76         "       " #asm_op " %1, %0, %3  \n"                             \
  77         "       scond       %1, [%2]    \n"                             \
  78         "       bnz         1b          \n"                             \
  79         : "=&r"(old), "=&r"(temp)                                       \
  80         : "r"(m), "ir"(nr)                                              \
  81         : "cc");                                                        \
  82                                                                         \
  83         smp_mb();                                                       \
  84                                                                         \
  85         return (old & (1 << nr)) != 0;                                  \
  86 }
  87
  88 #elif !defined(CONFIG_ARC_PLAT_EZNPS)
  89
  90 /*
  91  * Non hardware assisted Atomic-R-M-W
  92  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
  93  *
  94  * There's "significant" micro-optimization in writing our own variants of
  95  * bitops (over generic variants)
  96  *
  97  * (1) The generic APIs have "signed" @nr while we have it "unsigned"
  98  *     This avoids extra code to be generated for pointer arithmatic, since
  99  *     is "not sure" that index is NOT -ve
 100  * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc
 101  *     only consider bottom 5 bits of @nr, so NO need to mask them off.
 102  *     (GCC Quirk: however for constant @nr we still need to do the masking
 103  *             at compile time)
 104  */
 105
 106 #define BIT_OP(op, c_op, asm_op)                                        \
 107 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
 108 {                                                                       \
 109         unsigned long temp, flags;                                      \
 110         m += nr >> 5;                                                   \
 111                                                                         \
 112         /*                                                              \
 113          * spin lock/unlock provide the needed smp_mb() before/after    \
 114          */                                                             \
 115         bitops_lock(flags);                                             \
 116                                                                         \
 117         temp = *m;                                                      \
 118         *m = temp c_op (1UL << (nr & 0x1f));                                    \
 119                                                                         \
 120         bitops_unlock(flags);                                           \
 121 }
 122
 123 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 124 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 125 {                                                                       \
 126         unsigned long old, flags;                                       \
 127         m += nr >> 5;                                                   \
 128                                                                         \
 129         bitops_lock(flags);                                             \
 130                                                                         \
 131         old = *m;                                                       \
 132         *m = old c_op (1UL << (nr & 0x1f));                             \
 133                                                                         \
 134         bitops_unlock(flags);                                           \
 135                                                                         \
 136         return (old & (1UL << (nr & 0x1f))) != 0;                       \
 137 }
 138
 139 #else /* CONFIG_ARC_PLAT_EZNPS */
 140
 141 #define BIT_OP(op, c_op, asm_op)                                        \
 142 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
 143 {                                                                       \
 144         m += nr >> 5;                                                   \
 145                                                                         \
 146         nr = (1UL << (nr & 0x1f));                                      \
 147         if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)                       \
 148                 nr = ~nr;                                               \
 149                                                                         \
 150         __asm__ __volatile__(                                           \
 151         "       mov r2, %0\n"                                           \
 152         "       mov r3, %1\n"                                           \
 153         "       .word %2\n"                                             \
 154         :                                                               \
 155         : "r"(nr), "r"(m), "i"(asm_op)                                  \
 156         : "r2", "r3", "memory");                                        \
 157 }
 158
 159 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 160 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 161 {                                                                       \
 162         unsigned long old;                                              \
 163                                                                         \
 164         m += nr >> 5;                                                   \
 165                                                                         \
 166         nr = old = (1UL << (nr & 0x1f));                                \
 167         if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)                       \
 168                 old = ~old;                                             \
 169                                                                         \
 170         /* Explicit full memory barrier needed before/after */          \
 171         smp_mb();                                                       \
 172                                                                         \
 173         __asm__ __volatile__(                                           \
 174         "       mov r2, %0\n"                                           \
 175         "       mov r3, %1\n"                                           \
 176         "       .word %2\n"                                             \
 177         "       mov %0, r2"                                             \
 178         : "+r"(old)                                                     \
 179         : "r"(m), "i"(asm_op)                                           \
 180         : "r2", "r3", "memory");                                        \
 181                                                                         \
 182         smp_mb();                                                       \
 183                                                                         \
 184         return (old & nr) != 0;                                 \
 185 }
 186
 187 #endif /* CONFIG_ARC_PLAT_EZNPS */
 188
 189 /***************************************
 190  * Non atomic variants
 191  **************************************/
 192
 193 #define __BIT_OP(op, c_op, asm_op)                                      \
 194 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m)    \
 195 {                                                                       \
 196         unsigned long temp;                                             \
 197         m += nr >> 5;                                                   \
 198                                                                         \
 199         temp = *m;                                                      \
 200         *m = temp c_op (1UL << (nr & 0x1f));                            \
 201 }
 202
 203 #define __TEST_N_BIT_OP(op, c_op, asm_op)                               \
 204 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 205 {                                                                       \
 206         unsigned long old;                                              \
 207         m += nr >> 5;                                                   \
 208                                                                         \
 209         old = *m;                                                       \
 210         *m = old c_op (1UL << (nr & 0x1f));                             \
 211                                                                         \
 212         return (old & (1UL << (nr & 0x1f))) != 0;                       \
 213 }
 214
 215 #define BIT_OPS(op, c_op, asm_op)                                       \
 216                                                                         \
 217         /* set_bit(), clear_bit(), change_bit() */                      \
 218         BIT_OP(op, c_op, asm_op)                                        \
 219                                                                         \
 220         /* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\
 221         TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 222                                                                         \
 223         /* __set_bit(), __clear_bit(), __change_bit() */                \
 224         __BIT_OP(op, c_op, asm_op)                                      \
 225                                                                         \
 226         /* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\
 227         __TEST_N_BIT_OP(op, c_op, asm_op)
 228
 229 #ifndef CONFIG_ARC_PLAT_EZNPS
 230 BIT_OPS(set, |, bset)
 231 BIT_OPS(clear, & ~, bclr)
 232 BIT_OPS(change, ^, bxor)
 233 #else
 234 BIT_OPS(set, |, CTOP_INST_AOR_DI_R2_R2_R3)
 235 BIT_OPS(clear, & ~, CTOP_INST_AAND_DI_R2_R2_R3)
 236 BIT_OPS(change, ^, CTOP_INST_AXOR_DI_R2_R2_R3)
 237 #endif
 238
 239 /*
 240  * This routine doesn't need to be atomic.
 241  */
 242 static inline int
 243 test_bit(unsigned int nr, const volatile unsigned long *addr)
 244 {
 245         unsigned long mask;
 246
 247         addr += nr >> 5;
 248
 249         mask = 1UL << (nr & 0x1f);
 250
 251         return ((mask & *addr) != 0);
 252 }
 253
 254 #ifdef CONFIG_ISA_ARCOMPACT
 255
 256 /*
 257  * Count the number of zeros, starting from MSB
 258  * Helper for fls( ) friends
 259  * This is a pure count, so (1-32) or (0-31) doesn't apply
 260  * It could be 0 to 32, based on num of 0's in there
 261  * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31
 262  */
 263 static inline __attribute__ ((const)) int clz(unsigned int x)
 264 {
 265         unsigned int res;
 266
 267         __asm__ __volatile__(
 268         "       norm.f  %0, %1          \n"
 269         "       mov.n   %0, 0           \n"
 270         "       add.p   %0, %0, 1       \n"
 271         : "=r"(res)
 272         : "r"(x)
 273         : "cc");
 274
 275         return res;
 276 }
 277
 278 static inline int constant_fls(unsigned int x)
 279 {
 280         int r = 32;
 281
 282         if (!x)
 283                 return 0;
 284         if (!(x & 0xffff0000u)) {
 285                 x <<= 16;
 286                 r -= 16;
 287         }
 288         if (!(x & 0xff000000u)) {
 289                 x <<= 8;
 290                 r -= 8;
 291         }
 292         if (!(x & 0xf0000000u)) {
 293                 x <<= 4;
 294                 r -= 4;
 295         }
 296         if (!(x & 0xc0000000u)) {
 297                 x <<= 2;
 298                 r -= 2;
 299         }
 300         if (!(x & 0x80000000u)) {
 301                 x <<= 1;
 302                 r -= 1;
 303         }
 304         return r;
 305 }
 306
 307 /*
 308  * fls = Find Last Set in word
 309  * @result: [1-32]
 310  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 311  */
 312 static inline __attribute__ ((const)) int fls(unsigned int x)
 313 {
 314         if (__builtin_constant_p(x))
 315                return constant_fls(x);
 316
 317         return 32 - clz(x);
 318 }
 319
 320 /*
 321  * __fls: Similar to fls, but zero based (0-31)
 322  */
 323 static inline __attribute__ ((const)) int __fls(unsigned long x)
 324 {
 325         if (!x)
 326                 return 0;
 327         else
 328                 return fls(x) - 1;
 329 }
 330
 331 /*
 332  * ffs = Find First Set in word (LSB to MSB)
 333  * @result: [1-32], 0 if all 0's
 334  */
 335 #define ffs(x)  ({ unsigned long __t = (x); fls(__t & -__t); })
 336
 337 /*
 338  * __ffs: Similar to ffs, but zero based (0-31)
 339  */
 340 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long word)
 341 {
 342         if (!word)
 343                 return word;
 344
 345         return ffs(word) - 1;
 346 }
 347
 348 #else   /* CONFIG_ISA_ARCV2 */
 349
 350 /*
 351  * fls = Find Last Set in word
 352  * @result: [1-32]
 353  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 354  */
 355 static inline __attribute__ ((const)) int fls(unsigned long x)
 356 {
 357         int n;
 358
 359         asm volatile(
 360         "       fls.f   %0, %1          \n"  /* 0:31; 0(Z) if src 0 */
 361         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 362         : "=r"(n)       /* Early clobber not needed */
 363         : "r"(x)
 364         : "cc");
 365
 366         return n;
 367 }
 368
 369 /*
 370  * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set
 371  */
 372 static inline __attribute__ ((const)) int __fls(unsigned long x)
 373 {
 374         /* FLS insn has exactly same semantics as the API */
 375         return  __builtin_arc_fls(x);
 376 }
 377
 378 /*
 379  * ffs = Find First Set in word (LSB to MSB)
 380  * @result: [1-32], 0 if all 0's
 381  */
 382 static inline __attribute__ ((const)) int ffs(unsigned long x)
 383 {
 384         int n;
 385
 386         asm volatile(
 387         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 388         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 389         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 390         : "=r"(n)       /* Early clobber not needed */
 391         : "r"(x)
 392         : "cc");
 393
 394         return n;
 395 }
 396
 397 /*
 398  * __ffs: Similar to ffs, but zero based (0-31)
 399  */
 400 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long x)
 401 {
 402         unsigned long n;
 403
 404         asm volatile(
 405         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 406         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 407         : "=r"(n)
 408         : "r"(x)
 409         : "cc");
 410
 411         return n;
 412
 413 }
 414
 415 #endif  /* CONFIG_ISA_ARCOMPACT */
 416
 417 /*
 418  * ffz = Find First Zero in word.
 419  * @return:[0-31], 32 if all 1's
 420  */
 421 #define ffz(x)  __ffs(~(x))
 422
 423 #include <asm-generic/bitops/hweight.h>
 424 #include <asm-generic/bitops/fls64.h>
 425 #include <asm-generic/bitops/sched.h>
 426 #include <asm-generic/bitops/lock.h>
 427
 428 #include <asm-generic/bitops/find.h>
 429 #include <asm-generic/bitops/le.h>
 430 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 431
 432 #endif /* !__ASSEMBLY__ */
 433
 434 #endif