Merge tag 'mips_5.3' of git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux
[linux-2.6-microblaze.git] / arch / arc / include / asm / bitops.h
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4  */
5
6 #ifndef _ASM_BITOPS_H
7 #define _ASM_BITOPS_H
8
9 #ifndef _LINUX_BITOPS_H
10 #error only <linux/bitops.h> can be included directly
11 #endif
12
13 #ifndef __ASSEMBLY__
14
15 #include <linux/types.h>
16 #include <linux/compiler.h>
17 #include <asm/barrier.h>
18 #ifndef CONFIG_ARC_HAS_LLSC
19 #include <asm/smp.h>
20 #endif
21
22 #ifdef CONFIG_ARC_HAS_LLSC
23
24 /*
25  * Hardware assisted Atomic-R-M-W
26  */
27
28 #define BIT_OP(op, c_op, asm_op)                                        \
29 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
30 {                                                                       \
31         unsigned int temp;                                              \
32                                                                         \
33         m += nr >> 5;                                                   \
34                                                                         \
35         nr &= 0x1f;                                                     \
36                                                                         \
37         __asm__ __volatile__(                                           \
38         "1:     llock       %0, [%1]            \n"                     \
39         "       " #asm_op " %0, %0, %2  \n"                             \
40         "       scond       %0, [%1]            \n"                     \
41         "       bnz         1b                  \n"                     \
42         : "=&r"(temp)   /* Early clobber, to prevent reg reuse */       \
43         : "r"(m),       /* Not "m": llock only supports reg direct addr mode */ \
44           "ir"(nr)                                                      \
45         : "cc");                                                        \
46 }
47
48 /*
49  * Semantically:
50  *    Test the bit
51  *    if clear
52  *        set it and return 0 (old value)
53  *    else
54  *        return 1 (old value).
55  *
56  * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally
57  * and the old value of bit is returned
58  */
59 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
60 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
61 {                                                                       \
62         unsigned long old, temp;                                        \
63                                                                         \
64         m += nr >> 5;                                                   \
65                                                                         \
66         nr &= 0x1f;                                                     \
67                                                                         \
68         /*                                                              \
69          * Explicit full memory barrier needed before/after as          \
70          * LLOCK/SCOND themselves don't provide any such smenatic       \
71          */                                                             \
72         smp_mb();                                                       \
73                                                                         \
74         __asm__ __volatile__(                                           \
75         "1:     llock       %0, [%2]    \n"                             \
76         "       " #asm_op " %1, %0, %3  \n"                             \
77         "       scond       %1, [%2]    \n"                             \
78         "       bnz         1b          \n"                             \
79         : "=&r"(old), "=&r"(temp)                                       \
80         : "r"(m), "ir"(nr)                                              \
81         : "cc");                                                        \
82                                                                         \
83         smp_mb();                                                       \
84                                                                         \
85         return (old & (1 << nr)) != 0;                                  \
86 }
87
88 #elif !defined(CONFIG_ARC_PLAT_EZNPS)
89
90 /*
91  * Non hardware assisted Atomic-R-M-W
92  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
93  *
94  * There's "significant" micro-optimization in writing our own variants of
95  * bitops (over generic variants)
96  *
97  * (1) The generic APIs have "signed" @nr while we have it "unsigned"
98  *     This avoids extra code to be generated for pointer arithmatic, since
99  *     is "not sure" that index is NOT -ve
100  * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc
101  *     only consider bottom 5 bits of @nr, so NO need to mask them off.
102  *     (GCC Quirk: however for constant @nr we still need to do the masking
103  *             at compile time)
104  */
105
106 #define BIT_OP(op, c_op, asm_op)                                        \
107 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
108 {                                                                       \
109         unsigned long temp, flags;                                      \
110         m += nr >> 5;                                                   \
111                                                                         \
112         /*                                                              \
113          * spin lock/unlock provide the needed smp_mb() before/after    \
114          */                                                             \
115         bitops_lock(flags);                                             \
116                                                                         \
117         temp = *m;                                                      \
118         *m = temp c_op (1UL << (nr & 0x1f));                                    \
119                                                                         \
120         bitops_unlock(flags);                                           \
121 }
122
123 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
124 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
125 {                                                                       \
126         unsigned long old, flags;                                       \
127         m += nr >> 5;                                                   \
128                                                                         \
129         bitops_lock(flags);                                             \
130                                                                         \
131         old = *m;                                                       \
132         *m = old c_op (1UL << (nr & 0x1f));                             \
133                                                                         \
134         bitops_unlock(flags);                                           \
135                                                                         \
136         return (old & (1UL << (nr & 0x1f))) != 0;                       \
137 }
138
139 #else /* CONFIG_ARC_PLAT_EZNPS */
140
141 #define BIT_OP(op, c_op, asm_op)                                        \
142 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
143 {                                                                       \
144         m += nr >> 5;                                                   \
145                                                                         \
146         nr = (1UL << (nr & 0x1f));                                      \
147         if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)                       \
148                 nr = ~nr;                                               \
149                                                                         \
150         __asm__ __volatile__(                                           \
151         "       mov r2, %0\n"                                           \
152         "       mov r3, %1\n"                                           \
153         "       .word %2\n"                                             \
154         :                                                               \
155         : "r"(nr), "r"(m), "i"(asm_op)                                  \
156         : "r2", "r3", "memory");                                        \
157 }
158
159 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
160 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
161 {                                                                       \
162         unsigned long old;                                              \
163                                                                         \
164         m += nr >> 5;                                                   \
165                                                                         \
166         nr = old = (1UL << (nr & 0x1f));                                \
167         if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)                       \
168                 old = ~old;                                             \
169                                                                         \
170         /* Explicit full memory barrier needed before/after */          \
171         smp_mb();                                                       \
172                                                                         \
173         __asm__ __volatile__(                                           \
174         "       mov r2, %0\n"                                           \
175         "       mov r3, %1\n"                                           \
176         "       .word %2\n"                                             \
177         "       mov %0, r2"                                             \
178         : "+r"(old)                                                     \
179         : "r"(m), "i"(asm_op)                                           \
180         : "r2", "r3", "memory");                                        \
181                                                                         \
182         smp_mb();                                                       \
183                                                                         \
184         return (old & nr) != 0;                                 \
185 }
186
187 #endif /* CONFIG_ARC_PLAT_EZNPS */
188
189 /***************************************
190  * Non atomic variants
191  **************************************/
192
193 #define __BIT_OP(op, c_op, asm_op)                                      \
194 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m)    \
195 {                                                                       \
196         unsigned long temp;                                             \
197         m += nr >> 5;                                                   \
198                                                                         \
199         temp = *m;                                                      \
200         *m = temp c_op (1UL << (nr & 0x1f));                            \
201 }
202
203 #define __TEST_N_BIT_OP(op, c_op, asm_op)                               \
204 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
205 {                                                                       \
206         unsigned long old;                                              \
207         m += nr >> 5;                                                   \
208                                                                         \
209         old = *m;                                                       \
210         *m = old c_op (1UL << (nr & 0x1f));                             \
211                                                                         \
212         return (old & (1UL << (nr & 0x1f))) != 0;                       \
213 }
214
215 #define BIT_OPS(op, c_op, asm_op)                                       \
216                                                                         \
217         /* set_bit(), clear_bit(), change_bit() */                      \
218         BIT_OP(op, c_op, asm_op)                                        \
219                                                                         \
220         /* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\
221         TEST_N_BIT_OP(op, c_op, asm_op)                                 \
222                                                                         \
223         /* __set_bit(), __clear_bit(), __change_bit() */                \
224         __BIT_OP(op, c_op, asm_op)                                      \
225                                                                         \
226         /* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\
227         __TEST_N_BIT_OP(op, c_op, asm_op)
228
229 #ifndef CONFIG_ARC_PLAT_EZNPS
230 BIT_OPS(set, |, bset)
231 BIT_OPS(clear, & ~, bclr)
232 BIT_OPS(change, ^, bxor)
233 #else
234 BIT_OPS(set, |, CTOP_INST_AOR_DI_R2_R2_R3)
235 BIT_OPS(clear, & ~, CTOP_INST_AAND_DI_R2_R2_R3)
236 BIT_OPS(change, ^, CTOP_INST_AXOR_DI_R2_R2_R3)
237 #endif
238
239 /*
240  * This routine doesn't need to be atomic.
241  */
242 static inline int
243 test_bit(unsigned int nr, const volatile unsigned long *addr)
244 {
245         unsigned long mask;
246
247         addr += nr >> 5;
248
249         mask = 1UL << (nr & 0x1f);
250
251         return ((mask & *addr) != 0);
252 }
253
254 #ifdef CONFIG_ISA_ARCOMPACT
255
256 /*
257  * Count the number of zeros, starting from MSB
258  * Helper for fls( ) friends
259  * This is a pure count, so (1-32) or (0-31) doesn't apply
260  * It could be 0 to 32, based on num of 0's in there
261  * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31
262  */
263 static inline __attribute__ ((const)) int clz(unsigned int x)
264 {
265         unsigned int res;
266
267         __asm__ __volatile__(
268         "       norm.f  %0, %1          \n"
269         "       mov.n   %0, 0           \n"
270         "       add.p   %0, %0, 1       \n"
271         : "=r"(res)
272         : "r"(x)
273         : "cc");
274
275         return res;
276 }
277
278 static inline int constant_fls(unsigned int x)
279 {
280         int r = 32;
281
282         if (!x)
283                 return 0;
284         if (!(x & 0xffff0000u)) {
285                 x <<= 16;
286                 r -= 16;
287         }
288         if (!(x & 0xff000000u)) {
289                 x <<= 8;
290                 r -= 8;
291         }
292         if (!(x & 0xf0000000u)) {
293                 x <<= 4;
294                 r -= 4;
295         }
296         if (!(x & 0xc0000000u)) {
297                 x <<= 2;
298                 r -= 2;
299         }
300         if (!(x & 0x80000000u)) {
301                 x <<= 1;
302                 r -= 1;
303         }
304         return r;
305 }
306
307 /*
308  * fls = Find Last Set in word
309  * @result: [1-32]
310  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
311  */
312 static inline __attribute__ ((const)) int fls(unsigned int x)
313 {
314         if (__builtin_constant_p(x))
315                return constant_fls(x);
316
317         return 32 - clz(x);
318 }
319
320 /*
321  * __fls: Similar to fls, but zero based (0-31)
322  */
323 static inline __attribute__ ((const)) int __fls(unsigned long x)
324 {
325         if (!x)
326                 return 0;
327         else
328                 return fls(x) - 1;
329 }
330
331 /*
332  * ffs = Find First Set in word (LSB to MSB)
333  * @result: [1-32], 0 if all 0's
334  */
335 #define ffs(x)  ({ unsigned long __t = (x); fls(__t & -__t); })
336
337 /*
338  * __ffs: Similar to ffs, but zero based (0-31)
339  */
340 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long word)
341 {
342         if (!word)
343                 return word;
344
345         return ffs(word) - 1;
346 }
347
348 #else   /* CONFIG_ISA_ARCV2 */
349
350 /*
351  * fls = Find Last Set in word
352  * @result: [1-32]
353  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
354  */
355 static inline __attribute__ ((const)) int fls(unsigned long x)
356 {
357         int n;
358
359         asm volatile(
360         "       fls.f   %0, %1          \n"  /* 0:31; 0(Z) if src 0 */
361         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
362         : "=r"(n)       /* Early clobber not needed */
363         : "r"(x)
364         : "cc");
365
366         return n;
367 }
368
369 /*
370  * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set
371  */
372 static inline __attribute__ ((const)) int __fls(unsigned long x)
373 {
374         /* FLS insn has exactly same semantics as the API */
375         return  __builtin_arc_fls(x);
376 }
377
378 /*
379  * ffs = Find First Set in word (LSB to MSB)
380  * @result: [1-32], 0 if all 0's
381  */
382 static inline __attribute__ ((const)) int ffs(unsigned long x)
383 {
384         int n;
385
386         asm volatile(
387         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
388         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
389         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
390         : "=r"(n)       /* Early clobber not needed */
391         : "r"(x)
392         : "cc");
393
394         return n;
395 }
396
397 /*
398  * __ffs: Similar to ffs, but zero based (0-31)
399  */
400 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long x)
401 {
402         unsigned long n;
403
404         asm volatile(
405         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
406         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
407         : "=r"(n)
408         : "r"(x)
409         : "cc");
410
411         return n;
412
413 }
414
415 #endif  /* CONFIG_ISA_ARCOMPACT */
416
417 /*
418  * ffz = Find First Zero in word.
419  * @return:[0-31], 32 if all 1's
420  */
421 #define ffz(x)  __ffs(~(x))
422
423 #include <asm-generic/bitops/hweight.h>
424 #include <asm-generic/bitops/fls64.h>
425 #include <asm-generic/bitops/sched.h>
426 #include <asm-generic/bitops/lock.h>
427
428 #include <asm-generic/bitops/find.h>
429 #include <asm-generic/bitops/le.h>
430 #include <asm-generic/bitops/ext2-atomic-setbit.h>
431
432 #endif /* !__ASSEMBLY__ */
433
434 #endif