arch/arc/include/asm/bitops.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   4  */
   5
   6 #ifndef _ASM_BITOPS_H
   7 #define _ASM_BITOPS_H
   8
   9 #ifndef _LINUX_BITOPS_H
  10 #error only <linux/bitops.h> can be included directly
  11 #endif
  12
  13 #ifndef __ASSEMBLY__
  14
  15 #include <linux/types.h>
  16 #include <linux/compiler.h>
  17 #include <asm/barrier.h>
  18 #ifndef CONFIG_ARC_HAS_LLSC
  19 #include <asm/smp.h>
  20 #endif
  21
  22 #ifdef CONFIG_ARC_HAS_LLSC
  23
  24 /*
  25  * Hardware assisted Atomic-R-M-W
  26  */
  27
  28 #define BIT_OP(op, c_op, asm_op)                                        \
  29 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
  30 {                                                                       \
  31         unsigned int temp;                                              \
  32                                                                         \
  33         m += nr >> 5;                                                   \
  34                                                                         \
  35         nr &= 0x1f;                                                     \
  36                                                                         \
  37         __asm__ __volatile__(                                           \
  38         "1:     llock       %0, [%1]            \n"                     \
  39         "       " #asm_op " %0, %0, %2  \n"                             \
  40         "       scond       %0, [%1]            \n"                     \
  41         "       bnz         1b                  \n"                     \
  42         : "=&r"(temp)   /* Early clobber, to prevent reg reuse */       \
  43         : "r"(m),       /* Not "m": llock only supports reg direct addr mode */ \
  44           "ir"(nr)                                                      \
  45         : "cc");                                                        \
  46 }
  47
  48 /*
  49  * Semantically:
  50  *    Test the bit
  51  *    if clear
  52  *        set it and return 0 (old value)
  53  *    else
  54  *        return 1 (old value).
  55  *
  56  * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally
  57  * and the old value of bit is returned
  58  */
  59 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
  60 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
  61 {                                                                       \
  62         unsigned long old, temp;                                        \
  63                                                                         \
  64         m += nr >> 5;                                                   \
  65                                                                         \
  66         nr &= 0x1f;                                                     \
  67                                                                         \
  68         /*                                                              \
  69          * Explicit full memory barrier needed before/after as          \
  70          * LLOCK/SCOND themselves don't provide any such smenatic       \
  71          */                                                             \
  72         smp_mb();                                                       \
  73                                                                         \
  74         __asm__ __volatile__(                                           \
  75         "1:     llock       %0, [%2]    \n"                             \
  76         "       " #asm_op " %1, %0, %3  \n"                             \
  77         "       scond       %1, [%2]    \n"                             \
  78         "       bnz         1b          \n"                             \
  79         : "=&r"(old), "=&r"(temp)                                       \
  80         : "r"(m), "ir"(nr)                                              \
  81         : "cc");                                                        \
  82                                                                         \
  83         smp_mb();                                                       \
  84                                                                         \
  85         return (old & (1 << nr)) != 0;                                  \
  86 }
  87
  88 #else /* !CONFIG_ARC_HAS_LLSC */
  89
  90 /*
  91  * Non hardware assisted Atomic-R-M-W
  92  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
  93  *
  94  * There's "significant" micro-optimization in writing our own variants of
  95  * bitops (over generic variants)
  96  *
  97  * (1) The generic APIs have "signed" @nr while we have it "unsigned"
  98  *     This avoids extra code to be generated for pointer arithmatic, since
  99  *     is "not sure" that index is NOT -ve
 100  * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc
 101  *     only consider bottom 5 bits of @nr, so NO need to mask them off.
 102  *     (GCC Quirk: however for constant @nr we still need to do the masking
 103  *             at compile time)
 104  */
 105
 106 #define BIT_OP(op, c_op, asm_op)                                        \
 107 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
 108 {                                                                       \
 109         unsigned long temp, flags;                                      \
 110         m += nr >> 5;                                                   \
 111                                                                         \
 112         /*                                                              \
 113          * spin lock/unlock provide the needed smp_mb() before/after    \
 114          */                                                             \
 115         bitops_lock(flags);                                             \
 116                                                                         \
 117         temp = *m;                                                      \
 118         *m = temp c_op (1UL << (nr & 0x1f));                                    \
 119                                                                         \
 120         bitops_unlock(flags);                                           \
 121 }
 122
 123 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 124 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 125 {                                                                       \
 126         unsigned long old, flags;                                       \
 127         m += nr >> 5;                                                   \
 128                                                                         \
 129         bitops_lock(flags);                                             \
 130                                                                         \
 131         old = *m;                                                       \
 132         *m = old c_op (1UL << (nr & 0x1f));                             \
 133                                                                         \
 134         bitops_unlock(flags);                                           \
 135                                                                         \
 136         return (old & (1UL << (nr & 0x1f))) != 0;                       \
 137 }
 138
 139 #endif
 140
 141 /***************************************
 142  * Non atomic variants
 143  **************************************/
 144
 145 #define __BIT_OP(op, c_op, asm_op)                                      \
 146 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m)    \
 147 {                                                                       \
 148         unsigned long temp;                                             \
 149         m += nr >> 5;                                                   \
 150                                                                         \
 151         temp = *m;                                                      \
 152         *m = temp c_op (1UL << (nr & 0x1f));                            \
 153 }
 154
 155 #define __TEST_N_BIT_OP(op, c_op, asm_op)                               \
 156 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 157 {                                                                       \
 158         unsigned long old;                                              \
 159         m += nr >> 5;                                                   \
 160                                                                         \
 161         old = *m;                                                       \
 162         *m = old c_op (1UL << (nr & 0x1f));                             \
 163                                                                         \
 164         return (old & (1UL << (nr & 0x1f))) != 0;                       \
 165 }
 166
 167 #define BIT_OPS(op, c_op, asm_op)                                       \
 168                                                                         \
 169         /* set_bit(), clear_bit(), change_bit() */                      \
 170         BIT_OP(op, c_op, asm_op)                                        \
 171                                                                         \
 172         /* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\
 173         TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 174                                                                         \
 175         /* __set_bit(), __clear_bit(), __change_bit() */                \
 176         __BIT_OP(op, c_op, asm_op)                                      \
 177                                                                         \
 178         /* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\
 179         __TEST_N_BIT_OP(op, c_op, asm_op)
 180
 181 BIT_OPS(set, |, bset)
 182 BIT_OPS(clear, & ~, bclr)
 183 BIT_OPS(change, ^, bxor)
 184
 185 /*
 186  * This routine doesn't need to be atomic.
 187  */
 188 static inline int
 189 test_bit(unsigned int nr, const volatile unsigned long *addr)
 190 {
 191         unsigned long mask;
 192
 193         addr += nr >> 5;
 194
 195         mask = 1UL << (nr & 0x1f);
 196
 197         return ((mask & *addr) != 0);
 198 }
 199
 200 #ifdef CONFIG_ISA_ARCOMPACT
 201
 202 /*
 203  * Count the number of zeros, starting from MSB
 204  * Helper for fls( ) friends
 205  * This is a pure count, so (1-32) or (0-31) doesn't apply
 206  * It could be 0 to 32, based on num of 0's in there
 207  * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31
 208  */
 209 static inline __attribute__ ((const)) int clz(unsigned int x)
 210 {
 211         unsigned int res;
 212
 213         __asm__ __volatile__(
 214         "       norm.f  %0, %1          \n"
 215         "       mov.n   %0, 0           \n"
 216         "       add.p   %0, %0, 1       \n"
 217         : "=r"(res)
 218         : "r"(x)
 219         : "cc");
 220
 221         return res;
 222 }
 223
 224 static inline int constant_fls(unsigned int x)
 225 {
 226         int r = 32;
 227
 228         if (!x)
 229                 return 0;
 230         if (!(x & 0xffff0000u)) {
 231                 x <<= 16;
 232                 r -= 16;
 233         }
 234         if (!(x & 0xff000000u)) {
 235                 x <<= 8;
 236                 r -= 8;
 237         }
 238         if (!(x & 0xf0000000u)) {
 239                 x <<= 4;
 240                 r -= 4;
 241         }
 242         if (!(x & 0xc0000000u)) {
 243                 x <<= 2;
 244                 r -= 2;
 245         }
 246         if (!(x & 0x80000000u))
 247                 r -= 1;
 248         return r;
 249 }
 250
 251 /*
 252  * fls = Find Last Set in word
 253  * @result: [1-32]
 254  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 255  */
 256 static inline __attribute__ ((const)) int fls(unsigned int x)
 257 {
 258         if (__builtin_constant_p(x))
 259                return constant_fls(x);
 260
 261         return 32 - clz(x);
 262 }
 263
 264 /*
 265  * __fls: Similar to fls, but zero based (0-31)
 266  */
 267 static inline __attribute__ ((const)) int __fls(unsigned long x)
 268 {
 269         if (!x)
 270                 return 0;
 271         else
 272                 return fls(x) - 1;
 273 }
 274
 275 /*
 276  * ffs = Find First Set in word (LSB to MSB)
 277  * @result: [1-32], 0 if all 0's
 278  */
 279 #define ffs(x)  ({ unsigned long __t = (x); fls(__t & -__t); })
 280
 281 /*
 282  * __ffs: Similar to ffs, but zero based (0-31)
 283  */
 284 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long word)
 285 {
 286         if (!word)
 287                 return word;
 288
 289         return ffs(word) - 1;
 290 }
 291
 292 #else   /* CONFIG_ISA_ARCV2 */
 293
 294 /*
 295  * fls = Find Last Set in word
 296  * @result: [1-32]
 297  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 298  */
 299 static inline __attribute__ ((const)) int fls(unsigned long x)
 300 {
 301         int n;
 302
 303         asm volatile(
 304         "       fls.f   %0, %1          \n"  /* 0:31; 0(Z) if src 0 */
 305         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 306         : "=r"(n)       /* Early clobber not needed */
 307         : "r"(x)
 308         : "cc");
 309
 310         return n;
 311 }
 312
 313 /*
 314  * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set
 315  */
 316 static inline __attribute__ ((const)) int __fls(unsigned long x)
 317 {
 318         /* FLS insn has exactly same semantics as the API */
 319         return  __builtin_arc_fls(x);
 320 }
 321
 322 /*
 323  * ffs = Find First Set in word (LSB to MSB)
 324  * @result: [1-32], 0 if all 0's
 325  */
 326 static inline __attribute__ ((const)) int ffs(unsigned long x)
 327 {
 328         int n;
 329
 330         asm volatile(
 331         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 332         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 333         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 334         : "=r"(n)       /* Early clobber not needed */
 335         : "r"(x)
 336         : "cc");
 337
 338         return n;
 339 }
 340
 341 /*
 342  * __ffs: Similar to ffs, but zero based (0-31)
 343  */
 344 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long x)
 345 {
 346         unsigned long n;
 347
 348         asm volatile(
 349         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 350         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 351         : "=r"(n)
 352         : "r"(x)
 353         : "cc");
 354
 355         return n;
 356
 357 }
 358
 359 #endif  /* CONFIG_ISA_ARCOMPACT */
 360
 361 /*
 362  * ffz = Find First Zero in word.
 363  * @return:[0-31], 32 if all 1's
 364  */
 365 #define ffz(x)  __ffs(~(x))
 366
 367 #include <asm-generic/bitops/hweight.h>
 368 #include <asm-generic/bitops/fls64.h>
 369 #include <asm-generic/bitops/sched.h>
 370 #include <asm-generic/bitops/lock.h>
 371
 372 #include <asm-generic/bitops/find.h>
 373 #include <asm-generic/bitops/le.h>
 374 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 375
 376 #endif /* !__ASSEMBLY__ */
 377
 378 #endif