1 // SPDX-License-Identifier: GPL-2.0
3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
7 * This file provides wrappers with KASAN instrumentation for atomic operations.
8 * To use this functionality an arch's atomic.h file needs to define all
9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 * this file at the end. This file provides atomic_read() that forwards to
11 * arch_atomic_read() for actual atomic operation.
12 * Note: if an arch atomic operation is implemented by means of other atomic
13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 * double instrumentation.
17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18 #define _LINUX_ATOMIC_INSTRUMENTED_H
20 #include <linux/build_bug.h>
21 #include <linux/compiler.h>
22 #include <linux/instrumented.h>
24 static __always_inline int
25 atomic_read(const atomic_t *v)
27 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);
31 static __always_inline int
32 atomic_read_acquire(const atomic_t *v)
34 instrument_atomic_read(v, sizeof(*v));
35 return arch_atomic_read_acquire(v);
38 static __always_inline void
39 atomic_set(atomic_t *v, int i)
41 instrument_atomic_write(v, sizeof(*v));
42 arch_atomic_set(v, i);
45 static __always_inline void
46 atomic_set_release(atomic_t *v, int i)
49 instrument_atomic_write(v, sizeof(*v));
50 arch_atomic_set_release(v, i);
53 static __always_inline void
54 atomic_add(int i, atomic_t *v)
56 instrument_atomic_read_write(v, sizeof(*v));
57 arch_atomic_add(i, v);
60 static __always_inline int
61 atomic_add_return(int i, atomic_t *v)
64 instrument_atomic_read_write(v, sizeof(*v));
65 return arch_atomic_add_return(i, v);
68 static __always_inline int
69 atomic_add_return_acquire(int i, atomic_t *v)
71 instrument_atomic_read_write(v, sizeof(*v));
72 return arch_atomic_add_return_acquire(i, v);
75 static __always_inline int
76 atomic_add_return_release(int i, atomic_t *v)
79 instrument_atomic_read_write(v, sizeof(*v));
80 return arch_atomic_add_return_release(i, v);
83 static __always_inline int
84 atomic_add_return_relaxed(int i, atomic_t *v)
86 instrument_atomic_read_write(v, sizeof(*v));
87 return arch_atomic_add_return_relaxed(i, v);
90 static __always_inline int
91 atomic_fetch_add(int i, atomic_t *v)
94 instrument_atomic_read_write(v, sizeof(*v));
95 return arch_atomic_fetch_add(i, v);
98 static __always_inline int
99 atomic_fetch_add_acquire(int i, atomic_t *v)
101 instrument_atomic_read_write(v, sizeof(*v));
102 return arch_atomic_fetch_add_acquire(i, v);
105 static __always_inline int
106 atomic_fetch_add_release(int i, atomic_t *v)
109 instrument_atomic_read_write(v, sizeof(*v));
110 return arch_atomic_fetch_add_release(i, v);
113 static __always_inline int
114 atomic_fetch_add_relaxed(int i, atomic_t *v)
116 instrument_atomic_read_write(v, sizeof(*v));
117 return arch_atomic_fetch_add_relaxed(i, v);
120 static __always_inline void
121 atomic_sub(int i, atomic_t *v)
123 instrument_atomic_read_write(v, sizeof(*v));
124 arch_atomic_sub(i, v);
127 static __always_inline int
128 atomic_sub_return(int i, atomic_t *v)
131 instrument_atomic_read_write(v, sizeof(*v));
132 return arch_atomic_sub_return(i, v);
135 static __always_inline int
136 atomic_sub_return_acquire(int i, atomic_t *v)
138 instrument_atomic_read_write(v, sizeof(*v));
139 return arch_atomic_sub_return_acquire(i, v);
142 static __always_inline int
143 atomic_sub_return_release(int i, atomic_t *v)
146 instrument_atomic_read_write(v, sizeof(*v));
147 return arch_atomic_sub_return_release(i, v);
150 static __always_inline int
151 atomic_sub_return_relaxed(int i, atomic_t *v)
153 instrument_atomic_read_write(v, sizeof(*v));
154 return arch_atomic_sub_return_relaxed(i, v);
157 static __always_inline int
158 atomic_fetch_sub(int i, atomic_t *v)
161 instrument_atomic_read_write(v, sizeof(*v));
162 return arch_atomic_fetch_sub(i, v);
165 static __always_inline int
166 atomic_fetch_sub_acquire(int i, atomic_t *v)
168 instrument_atomic_read_write(v, sizeof(*v));
169 return arch_atomic_fetch_sub_acquire(i, v);
172 static __always_inline int
173 atomic_fetch_sub_release(int i, atomic_t *v)
176 instrument_atomic_read_write(v, sizeof(*v));
177 return arch_atomic_fetch_sub_release(i, v);
180 static __always_inline int
181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
183 instrument_atomic_read_write(v, sizeof(*v));
184 return arch_atomic_fetch_sub_relaxed(i, v);
187 static __always_inline void
188 atomic_inc(atomic_t *v)
190 instrument_atomic_read_write(v, sizeof(*v));
194 static __always_inline int
195 atomic_inc_return(atomic_t *v)
198 instrument_atomic_read_write(v, sizeof(*v));
199 return arch_atomic_inc_return(v);
202 static __always_inline int
203 atomic_inc_return_acquire(atomic_t *v)
205 instrument_atomic_read_write(v, sizeof(*v));
206 return arch_atomic_inc_return_acquire(v);
209 static __always_inline int
210 atomic_inc_return_release(atomic_t *v)
213 instrument_atomic_read_write(v, sizeof(*v));
214 return arch_atomic_inc_return_release(v);
217 static __always_inline int
218 atomic_inc_return_relaxed(atomic_t *v)
220 instrument_atomic_read_write(v, sizeof(*v));
221 return arch_atomic_inc_return_relaxed(v);
224 static __always_inline int
225 atomic_fetch_inc(atomic_t *v)
228 instrument_atomic_read_write(v, sizeof(*v));
229 return arch_atomic_fetch_inc(v);
232 static __always_inline int
233 atomic_fetch_inc_acquire(atomic_t *v)
235 instrument_atomic_read_write(v, sizeof(*v));
236 return arch_atomic_fetch_inc_acquire(v);
239 static __always_inline int
240 atomic_fetch_inc_release(atomic_t *v)
243 instrument_atomic_read_write(v, sizeof(*v));
244 return arch_atomic_fetch_inc_release(v);
247 static __always_inline int
248 atomic_fetch_inc_relaxed(atomic_t *v)
250 instrument_atomic_read_write(v, sizeof(*v));
251 return arch_atomic_fetch_inc_relaxed(v);
254 static __always_inline void
255 atomic_dec(atomic_t *v)
257 instrument_atomic_read_write(v, sizeof(*v));
261 static __always_inline int
262 atomic_dec_return(atomic_t *v)
265 instrument_atomic_read_write(v, sizeof(*v));
266 return arch_atomic_dec_return(v);
269 static __always_inline int
270 atomic_dec_return_acquire(atomic_t *v)
272 instrument_atomic_read_write(v, sizeof(*v));
273 return arch_atomic_dec_return_acquire(v);
276 static __always_inline int
277 atomic_dec_return_release(atomic_t *v)
280 instrument_atomic_read_write(v, sizeof(*v));
281 return arch_atomic_dec_return_release(v);
284 static __always_inline int
285 atomic_dec_return_relaxed(atomic_t *v)
287 instrument_atomic_read_write(v, sizeof(*v));
288 return arch_atomic_dec_return_relaxed(v);
291 static __always_inline int
292 atomic_fetch_dec(atomic_t *v)
295 instrument_atomic_read_write(v, sizeof(*v));
296 return arch_atomic_fetch_dec(v);
299 static __always_inline int
300 atomic_fetch_dec_acquire(atomic_t *v)
302 instrument_atomic_read_write(v, sizeof(*v));
303 return arch_atomic_fetch_dec_acquire(v);
306 static __always_inline int
307 atomic_fetch_dec_release(atomic_t *v)
310 instrument_atomic_read_write(v, sizeof(*v));
311 return arch_atomic_fetch_dec_release(v);
314 static __always_inline int
315 atomic_fetch_dec_relaxed(atomic_t *v)
317 instrument_atomic_read_write(v, sizeof(*v));
318 return arch_atomic_fetch_dec_relaxed(v);
321 static __always_inline void
322 atomic_and(int i, atomic_t *v)
324 instrument_atomic_read_write(v, sizeof(*v));
325 arch_atomic_and(i, v);
328 static __always_inline int
329 atomic_fetch_and(int i, atomic_t *v)
332 instrument_atomic_read_write(v, sizeof(*v));
333 return arch_atomic_fetch_and(i, v);
336 static __always_inline int
337 atomic_fetch_and_acquire(int i, atomic_t *v)
339 instrument_atomic_read_write(v, sizeof(*v));
340 return arch_atomic_fetch_and_acquire(i, v);
343 static __always_inline int
344 atomic_fetch_and_release(int i, atomic_t *v)
347 instrument_atomic_read_write(v, sizeof(*v));
348 return arch_atomic_fetch_and_release(i, v);
351 static __always_inline int
352 atomic_fetch_and_relaxed(int i, atomic_t *v)
354 instrument_atomic_read_write(v, sizeof(*v));
355 return arch_atomic_fetch_and_relaxed(i, v);
358 static __always_inline void
359 atomic_andnot(int i, atomic_t *v)
361 instrument_atomic_read_write(v, sizeof(*v));
362 arch_atomic_andnot(i, v);
365 static __always_inline int
366 atomic_fetch_andnot(int i, atomic_t *v)
369 instrument_atomic_read_write(v, sizeof(*v));
370 return arch_atomic_fetch_andnot(i, v);
373 static __always_inline int
374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
376 instrument_atomic_read_write(v, sizeof(*v));
377 return arch_atomic_fetch_andnot_acquire(i, v);
380 static __always_inline int
381 atomic_fetch_andnot_release(int i, atomic_t *v)
384 instrument_atomic_read_write(v, sizeof(*v));
385 return arch_atomic_fetch_andnot_release(i, v);
388 static __always_inline int
389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
391 instrument_atomic_read_write(v, sizeof(*v));
392 return arch_atomic_fetch_andnot_relaxed(i, v);
395 static __always_inline void
396 atomic_or(int i, atomic_t *v)
398 instrument_atomic_read_write(v, sizeof(*v));
399 arch_atomic_or(i, v);
402 static __always_inline int
403 atomic_fetch_or(int i, atomic_t *v)
406 instrument_atomic_read_write(v, sizeof(*v));
407 return arch_atomic_fetch_or(i, v);
410 static __always_inline int
411 atomic_fetch_or_acquire(int i, atomic_t *v)
413 instrument_atomic_read_write(v, sizeof(*v));
414 return arch_atomic_fetch_or_acquire(i, v);
417 static __always_inline int
418 atomic_fetch_or_release(int i, atomic_t *v)
421 instrument_atomic_read_write(v, sizeof(*v));
422 return arch_atomic_fetch_or_release(i, v);
425 static __always_inline int
426 atomic_fetch_or_relaxed(int i, atomic_t *v)
428 instrument_atomic_read_write(v, sizeof(*v));
429 return arch_atomic_fetch_or_relaxed(i, v);
432 static __always_inline void
433 atomic_xor(int i, atomic_t *v)
435 instrument_atomic_read_write(v, sizeof(*v));
436 arch_atomic_xor(i, v);
439 static __always_inline int
440 atomic_fetch_xor(int i, atomic_t *v)
443 instrument_atomic_read_write(v, sizeof(*v));
444 return arch_atomic_fetch_xor(i, v);
447 static __always_inline int
448 atomic_fetch_xor_acquire(int i, atomic_t *v)
450 instrument_atomic_read_write(v, sizeof(*v));
451 return arch_atomic_fetch_xor_acquire(i, v);
454 static __always_inline int
455 atomic_fetch_xor_release(int i, atomic_t *v)
458 instrument_atomic_read_write(v, sizeof(*v));
459 return arch_atomic_fetch_xor_release(i, v);
462 static __always_inline int
463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
465 instrument_atomic_read_write(v, sizeof(*v));
466 return arch_atomic_fetch_xor_relaxed(i, v);
469 static __always_inline int
470 atomic_xchg(atomic_t *v, int i)
473 instrument_atomic_read_write(v, sizeof(*v));
474 return arch_atomic_xchg(v, i);
477 static __always_inline int
478 atomic_xchg_acquire(atomic_t *v, int i)
480 instrument_atomic_read_write(v, sizeof(*v));
481 return arch_atomic_xchg_acquire(v, i);
484 static __always_inline int
485 atomic_xchg_release(atomic_t *v, int i)
488 instrument_atomic_read_write(v, sizeof(*v));
489 return arch_atomic_xchg_release(v, i);
492 static __always_inline int
493 atomic_xchg_relaxed(atomic_t *v, int i)
495 instrument_atomic_read_write(v, sizeof(*v));
496 return arch_atomic_xchg_relaxed(v, i);
499 static __always_inline int
500 atomic_cmpxchg(atomic_t *v, int old, int new)
503 instrument_atomic_read_write(v, sizeof(*v));
504 return arch_atomic_cmpxchg(v, old, new);
507 static __always_inline int
508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
510 instrument_atomic_read_write(v, sizeof(*v));
511 return arch_atomic_cmpxchg_acquire(v, old, new);
514 static __always_inline int
515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
518 instrument_atomic_read_write(v, sizeof(*v));
519 return arch_atomic_cmpxchg_release(v, old, new);
522 static __always_inline int
523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
525 instrument_atomic_read_write(v, sizeof(*v));
526 return arch_atomic_cmpxchg_relaxed(v, old, new);
529 static __always_inline bool
530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
533 instrument_atomic_read_write(v, sizeof(*v));
534 instrument_atomic_read_write(old, sizeof(*old));
535 return arch_atomic_try_cmpxchg(v, old, new);
538 static __always_inline bool
539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
541 instrument_atomic_read_write(v, sizeof(*v));
542 instrument_atomic_read_write(old, sizeof(*old));
543 return arch_atomic_try_cmpxchg_acquire(v, old, new);
546 static __always_inline bool
547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
550 instrument_atomic_read_write(v, sizeof(*v));
551 instrument_atomic_read_write(old, sizeof(*old));
552 return arch_atomic_try_cmpxchg_release(v, old, new);
555 static __always_inline bool
556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
558 instrument_atomic_read_write(v, sizeof(*v));
559 instrument_atomic_read_write(old, sizeof(*old));
560 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
563 static __always_inline bool
564 atomic_sub_and_test(int i, atomic_t *v)
567 instrument_atomic_read_write(v, sizeof(*v));
568 return arch_atomic_sub_and_test(i, v);
571 static __always_inline bool
572 atomic_dec_and_test(atomic_t *v)
575 instrument_atomic_read_write(v, sizeof(*v));
576 return arch_atomic_dec_and_test(v);
579 static __always_inline bool
580 atomic_inc_and_test(atomic_t *v)
583 instrument_atomic_read_write(v, sizeof(*v));
584 return arch_atomic_inc_and_test(v);
587 static __always_inline bool
588 atomic_add_negative(int i, atomic_t *v)
591 instrument_atomic_read_write(v, sizeof(*v));
592 return arch_atomic_add_negative(i, v);
595 static __always_inline int
596 atomic_fetch_add_unless(atomic_t *v, int a, int u)
599 instrument_atomic_read_write(v, sizeof(*v));
600 return arch_atomic_fetch_add_unless(v, a, u);
603 static __always_inline bool
604 atomic_add_unless(atomic_t *v, int a, int u)
607 instrument_atomic_read_write(v, sizeof(*v));
608 return arch_atomic_add_unless(v, a, u);
611 static __always_inline bool
612 atomic_inc_not_zero(atomic_t *v)
615 instrument_atomic_read_write(v, sizeof(*v));
616 return arch_atomic_inc_not_zero(v);
619 static __always_inline bool
620 atomic_inc_unless_negative(atomic_t *v)
623 instrument_atomic_read_write(v, sizeof(*v));
624 return arch_atomic_inc_unless_negative(v);
627 static __always_inline bool
628 atomic_dec_unless_positive(atomic_t *v)
631 instrument_atomic_read_write(v, sizeof(*v));
632 return arch_atomic_dec_unless_positive(v);
635 static __always_inline int
636 atomic_dec_if_positive(atomic_t *v)
639 instrument_atomic_read_write(v, sizeof(*v));
640 return arch_atomic_dec_if_positive(v);
643 static __always_inline s64
644 atomic64_read(const atomic64_t *v)
646 instrument_atomic_read(v, sizeof(*v));
647 return arch_atomic64_read(v);
650 static __always_inline s64
651 atomic64_read_acquire(const atomic64_t *v)
653 instrument_atomic_read(v, sizeof(*v));
654 return arch_atomic64_read_acquire(v);
657 static __always_inline void
658 atomic64_set(atomic64_t *v, s64 i)
660 instrument_atomic_write(v, sizeof(*v));
661 arch_atomic64_set(v, i);
664 static __always_inline void
665 atomic64_set_release(atomic64_t *v, s64 i)
668 instrument_atomic_write(v, sizeof(*v));
669 arch_atomic64_set_release(v, i);
672 static __always_inline void
673 atomic64_add(s64 i, atomic64_t *v)
675 instrument_atomic_read_write(v, sizeof(*v));
676 arch_atomic64_add(i, v);
679 static __always_inline s64
680 atomic64_add_return(s64 i, atomic64_t *v)
683 instrument_atomic_read_write(v, sizeof(*v));
684 return arch_atomic64_add_return(i, v);
687 static __always_inline s64
688 atomic64_add_return_acquire(s64 i, atomic64_t *v)
690 instrument_atomic_read_write(v, sizeof(*v));
691 return arch_atomic64_add_return_acquire(i, v);
694 static __always_inline s64
695 atomic64_add_return_release(s64 i, atomic64_t *v)
698 instrument_atomic_read_write(v, sizeof(*v));
699 return arch_atomic64_add_return_release(i, v);
702 static __always_inline s64
703 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
705 instrument_atomic_read_write(v, sizeof(*v));
706 return arch_atomic64_add_return_relaxed(i, v);
709 static __always_inline s64
710 atomic64_fetch_add(s64 i, atomic64_t *v)
713 instrument_atomic_read_write(v, sizeof(*v));
714 return arch_atomic64_fetch_add(i, v);
717 static __always_inline s64
718 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
720 instrument_atomic_read_write(v, sizeof(*v));
721 return arch_atomic64_fetch_add_acquire(i, v);
724 static __always_inline s64
725 atomic64_fetch_add_release(s64 i, atomic64_t *v)
728 instrument_atomic_read_write(v, sizeof(*v));
729 return arch_atomic64_fetch_add_release(i, v);
732 static __always_inline s64
733 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
735 instrument_atomic_read_write(v, sizeof(*v));
736 return arch_atomic64_fetch_add_relaxed(i, v);
739 static __always_inline void
740 atomic64_sub(s64 i, atomic64_t *v)
742 instrument_atomic_read_write(v, sizeof(*v));
743 arch_atomic64_sub(i, v);
746 static __always_inline s64
747 atomic64_sub_return(s64 i, atomic64_t *v)
750 instrument_atomic_read_write(v, sizeof(*v));
751 return arch_atomic64_sub_return(i, v);
754 static __always_inline s64
755 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
757 instrument_atomic_read_write(v, sizeof(*v));
758 return arch_atomic64_sub_return_acquire(i, v);
761 static __always_inline s64
762 atomic64_sub_return_release(s64 i, atomic64_t *v)
765 instrument_atomic_read_write(v, sizeof(*v));
766 return arch_atomic64_sub_return_release(i, v);
769 static __always_inline s64
770 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
772 instrument_atomic_read_write(v, sizeof(*v));
773 return arch_atomic64_sub_return_relaxed(i, v);
776 static __always_inline s64
777 atomic64_fetch_sub(s64 i, atomic64_t *v)
780 instrument_atomic_read_write(v, sizeof(*v));
781 return arch_atomic64_fetch_sub(i, v);
784 static __always_inline s64
785 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
787 instrument_atomic_read_write(v, sizeof(*v));
788 return arch_atomic64_fetch_sub_acquire(i, v);
791 static __always_inline s64
792 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
795 instrument_atomic_read_write(v, sizeof(*v));
796 return arch_atomic64_fetch_sub_release(i, v);
799 static __always_inline s64
800 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
802 instrument_atomic_read_write(v, sizeof(*v));
803 return arch_atomic64_fetch_sub_relaxed(i, v);
806 static __always_inline void
807 atomic64_inc(atomic64_t *v)
809 instrument_atomic_read_write(v, sizeof(*v));
810 arch_atomic64_inc(v);
813 static __always_inline s64
814 atomic64_inc_return(atomic64_t *v)
817 instrument_atomic_read_write(v, sizeof(*v));
818 return arch_atomic64_inc_return(v);
821 static __always_inline s64
822 atomic64_inc_return_acquire(atomic64_t *v)
824 instrument_atomic_read_write(v, sizeof(*v));
825 return arch_atomic64_inc_return_acquire(v);
828 static __always_inline s64
829 atomic64_inc_return_release(atomic64_t *v)
832 instrument_atomic_read_write(v, sizeof(*v));
833 return arch_atomic64_inc_return_release(v);
836 static __always_inline s64
837 atomic64_inc_return_relaxed(atomic64_t *v)
839 instrument_atomic_read_write(v, sizeof(*v));
840 return arch_atomic64_inc_return_relaxed(v);
843 static __always_inline s64
844 atomic64_fetch_inc(atomic64_t *v)
847 instrument_atomic_read_write(v, sizeof(*v));
848 return arch_atomic64_fetch_inc(v);
851 static __always_inline s64
852 atomic64_fetch_inc_acquire(atomic64_t *v)
854 instrument_atomic_read_write(v, sizeof(*v));
855 return arch_atomic64_fetch_inc_acquire(v);
858 static __always_inline s64
859 atomic64_fetch_inc_release(atomic64_t *v)
862 instrument_atomic_read_write(v, sizeof(*v));
863 return arch_atomic64_fetch_inc_release(v);
866 static __always_inline s64
867 atomic64_fetch_inc_relaxed(atomic64_t *v)
869 instrument_atomic_read_write(v, sizeof(*v));
870 return arch_atomic64_fetch_inc_relaxed(v);
873 static __always_inline void
874 atomic64_dec(atomic64_t *v)
876 instrument_atomic_read_write(v, sizeof(*v));
877 arch_atomic64_dec(v);
880 static __always_inline s64
881 atomic64_dec_return(atomic64_t *v)
884 instrument_atomic_read_write(v, sizeof(*v));
885 return arch_atomic64_dec_return(v);
888 static __always_inline s64
889 atomic64_dec_return_acquire(atomic64_t *v)
891 instrument_atomic_read_write(v, sizeof(*v));
892 return arch_atomic64_dec_return_acquire(v);
895 static __always_inline s64
896 atomic64_dec_return_release(atomic64_t *v)
899 instrument_atomic_read_write(v, sizeof(*v));
900 return arch_atomic64_dec_return_release(v);
903 static __always_inline s64
904 atomic64_dec_return_relaxed(atomic64_t *v)
906 instrument_atomic_read_write(v, sizeof(*v));
907 return arch_atomic64_dec_return_relaxed(v);
910 static __always_inline s64
911 atomic64_fetch_dec(atomic64_t *v)
914 instrument_atomic_read_write(v, sizeof(*v));
915 return arch_atomic64_fetch_dec(v);
918 static __always_inline s64
919 atomic64_fetch_dec_acquire(atomic64_t *v)
921 instrument_atomic_read_write(v, sizeof(*v));
922 return arch_atomic64_fetch_dec_acquire(v);
925 static __always_inline s64
926 atomic64_fetch_dec_release(atomic64_t *v)
929 instrument_atomic_read_write(v, sizeof(*v));
930 return arch_atomic64_fetch_dec_release(v);
933 static __always_inline s64
934 atomic64_fetch_dec_relaxed(atomic64_t *v)
936 instrument_atomic_read_write(v, sizeof(*v));
937 return arch_atomic64_fetch_dec_relaxed(v);
940 static __always_inline void
941 atomic64_and(s64 i, atomic64_t *v)
943 instrument_atomic_read_write(v, sizeof(*v));
944 arch_atomic64_and(i, v);
947 static __always_inline s64
948 atomic64_fetch_and(s64 i, atomic64_t *v)
951 instrument_atomic_read_write(v, sizeof(*v));
952 return arch_atomic64_fetch_and(i, v);
955 static __always_inline s64
956 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
958 instrument_atomic_read_write(v, sizeof(*v));
959 return arch_atomic64_fetch_and_acquire(i, v);
962 static __always_inline s64
963 atomic64_fetch_and_release(s64 i, atomic64_t *v)
966 instrument_atomic_read_write(v, sizeof(*v));
967 return arch_atomic64_fetch_and_release(i, v);
970 static __always_inline s64
971 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
973 instrument_atomic_read_write(v, sizeof(*v));
974 return arch_atomic64_fetch_and_relaxed(i, v);
977 static __always_inline void
978 atomic64_andnot(s64 i, atomic64_t *v)
980 instrument_atomic_read_write(v, sizeof(*v));
981 arch_atomic64_andnot(i, v);
984 static __always_inline s64
985 atomic64_fetch_andnot(s64 i, atomic64_t *v)
988 instrument_atomic_read_write(v, sizeof(*v));
989 return arch_atomic64_fetch_andnot(i, v);
992 static __always_inline s64
993 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
995 instrument_atomic_read_write(v, sizeof(*v));
996 return arch_atomic64_fetch_andnot_acquire(i, v);
999 static __always_inline s64
1000 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1003 instrument_atomic_read_write(v, sizeof(*v));
1004 return arch_atomic64_fetch_andnot_release(i, v);
1007 static __always_inline s64
1008 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1010 instrument_atomic_read_write(v, sizeof(*v));
1011 return arch_atomic64_fetch_andnot_relaxed(i, v);
1014 static __always_inline void
1015 atomic64_or(s64 i, atomic64_t *v)
1017 instrument_atomic_read_write(v, sizeof(*v));
1018 arch_atomic64_or(i, v);
1021 static __always_inline s64
1022 atomic64_fetch_or(s64 i, atomic64_t *v)
1025 instrument_atomic_read_write(v, sizeof(*v));
1026 return arch_atomic64_fetch_or(i, v);
1029 static __always_inline s64
1030 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1032 instrument_atomic_read_write(v, sizeof(*v));
1033 return arch_atomic64_fetch_or_acquire(i, v);
1036 static __always_inline s64
1037 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1040 instrument_atomic_read_write(v, sizeof(*v));
1041 return arch_atomic64_fetch_or_release(i, v);
1044 static __always_inline s64
1045 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1047 instrument_atomic_read_write(v, sizeof(*v));
1048 return arch_atomic64_fetch_or_relaxed(i, v);
1051 static __always_inline void
1052 atomic64_xor(s64 i, atomic64_t *v)
1054 instrument_atomic_read_write(v, sizeof(*v));
1055 arch_atomic64_xor(i, v);
1058 static __always_inline s64
1059 atomic64_fetch_xor(s64 i, atomic64_t *v)
1062 instrument_atomic_read_write(v, sizeof(*v));
1063 return arch_atomic64_fetch_xor(i, v);
1066 static __always_inline s64
1067 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1069 instrument_atomic_read_write(v, sizeof(*v));
1070 return arch_atomic64_fetch_xor_acquire(i, v);
1073 static __always_inline s64
1074 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1077 instrument_atomic_read_write(v, sizeof(*v));
1078 return arch_atomic64_fetch_xor_release(i, v);
1081 static __always_inline s64
1082 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1084 instrument_atomic_read_write(v, sizeof(*v));
1085 return arch_atomic64_fetch_xor_relaxed(i, v);
1088 static __always_inline s64
1089 atomic64_xchg(atomic64_t *v, s64 i)
1092 instrument_atomic_read_write(v, sizeof(*v));
1093 return arch_atomic64_xchg(v, i);
1096 static __always_inline s64
1097 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1099 instrument_atomic_read_write(v, sizeof(*v));
1100 return arch_atomic64_xchg_acquire(v, i);
1103 static __always_inline s64
1104 atomic64_xchg_release(atomic64_t *v, s64 i)
1107 instrument_atomic_read_write(v, sizeof(*v));
1108 return arch_atomic64_xchg_release(v, i);
1111 static __always_inline s64
1112 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1114 instrument_atomic_read_write(v, sizeof(*v));
1115 return arch_atomic64_xchg_relaxed(v, i);
1118 static __always_inline s64
1119 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1122 instrument_atomic_read_write(v, sizeof(*v));
1123 return arch_atomic64_cmpxchg(v, old, new);
1126 static __always_inline s64
1127 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1129 instrument_atomic_read_write(v, sizeof(*v));
1130 return arch_atomic64_cmpxchg_acquire(v, old, new);
1133 static __always_inline s64
1134 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1137 instrument_atomic_read_write(v, sizeof(*v));
1138 return arch_atomic64_cmpxchg_release(v, old, new);
1141 static __always_inline s64
1142 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1144 instrument_atomic_read_write(v, sizeof(*v));
1145 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1148 static __always_inline bool
1149 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1152 instrument_atomic_read_write(v, sizeof(*v));
1153 instrument_atomic_read_write(old, sizeof(*old));
1154 return arch_atomic64_try_cmpxchg(v, old, new);
1157 static __always_inline bool
1158 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1160 instrument_atomic_read_write(v, sizeof(*v));
1161 instrument_atomic_read_write(old, sizeof(*old));
1162 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1165 static __always_inline bool
1166 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1169 instrument_atomic_read_write(v, sizeof(*v));
1170 instrument_atomic_read_write(old, sizeof(*old));
1171 return arch_atomic64_try_cmpxchg_release(v, old, new);
1174 static __always_inline bool
1175 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1177 instrument_atomic_read_write(v, sizeof(*v));
1178 instrument_atomic_read_write(old, sizeof(*old));
1179 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1182 static __always_inline bool
1183 atomic64_sub_and_test(s64 i, atomic64_t *v)
1186 instrument_atomic_read_write(v, sizeof(*v));
1187 return arch_atomic64_sub_and_test(i, v);
1190 static __always_inline bool
1191 atomic64_dec_and_test(atomic64_t *v)
1194 instrument_atomic_read_write(v, sizeof(*v));
1195 return arch_atomic64_dec_and_test(v);
1198 static __always_inline bool
1199 atomic64_inc_and_test(atomic64_t *v)
1202 instrument_atomic_read_write(v, sizeof(*v));
1203 return arch_atomic64_inc_and_test(v);
1206 static __always_inline bool
1207 atomic64_add_negative(s64 i, atomic64_t *v)
1210 instrument_atomic_read_write(v, sizeof(*v));
1211 return arch_atomic64_add_negative(i, v);
1214 static __always_inline s64
1215 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1218 instrument_atomic_read_write(v, sizeof(*v));
1219 return arch_atomic64_fetch_add_unless(v, a, u);
1222 static __always_inline bool
1223 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1226 instrument_atomic_read_write(v, sizeof(*v));
1227 return arch_atomic64_add_unless(v, a, u);
1230 static __always_inline bool
1231 atomic64_inc_not_zero(atomic64_t *v)
1234 instrument_atomic_read_write(v, sizeof(*v));
1235 return arch_atomic64_inc_not_zero(v);
1238 static __always_inline bool
1239 atomic64_inc_unless_negative(atomic64_t *v)
1242 instrument_atomic_read_write(v, sizeof(*v));
1243 return arch_atomic64_inc_unless_negative(v);
1246 static __always_inline bool
1247 atomic64_dec_unless_positive(atomic64_t *v)
1250 instrument_atomic_read_write(v, sizeof(*v));
1251 return arch_atomic64_dec_unless_positive(v);
1254 static __always_inline s64
1255 atomic64_dec_if_positive(atomic64_t *v)
1258 instrument_atomic_read_write(v, sizeof(*v));
1259 return arch_atomic64_dec_if_positive(v);
1262 static __always_inline long
1263 atomic_long_read(const atomic_long_t *v)
1265 instrument_atomic_read(v, sizeof(*v));
1266 return arch_atomic_long_read(v);
1269 static __always_inline long
1270 atomic_long_read_acquire(const atomic_long_t *v)
1272 instrument_atomic_read(v, sizeof(*v));
1273 return arch_atomic_long_read_acquire(v);
1276 static __always_inline void
1277 atomic_long_set(atomic_long_t *v, long i)
1279 instrument_atomic_write(v, sizeof(*v));
1280 arch_atomic_long_set(v, i);
1283 static __always_inline void
1284 atomic_long_set_release(atomic_long_t *v, long i)
1287 instrument_atomic_write(v, sizeof(*v));
1288 arch_atomic_long_set_release(v, i);
1291 static __always_inline void
1292 atomic_long_add(long i, atomic_long_t *v)
1294 instrument_atomic_read_write(v, sizeof(*v));
1295 arch_atomic_long_add(i, v);
1298 static __always_inline long
1299 atomic_long_add_return(long i, atomic_long_t *v)
1302 instrument_atomic_read_write(v, sizeof(*v));
1303 return arch_atomic_long_add_return(i, v);
1306 static __always_inline long
1307 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1309 instrument_atomic_read_write(v, sizeof(*v));
1310 return arch_atomic_long_add_return_acquire(i, v);
1313 static __always_inline long
1314 atomic_long_add_return_release(long i, atomic_long_t *v)
1317 instrument_atomic_read_write(v, sizeof(*v));
1318 return arch_atomic_long_add_return_release(i, v);
1321 static __always_inline long
1322 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1324 instrument_atomic_read_write(v, sizeof(*v));
1325 return arch_atomic_long_add_return_relaxed(i, v);
1328 static __always_inline long
1329 atomic_long_fetch_add(long i, atomic_long_t *v)
1332 instrument_atomic_read_write(v, sizeof(*v));
1333 return arch_atomic_long_fetch_add(i, v);
1336 static __always_inline long
1337 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1339 instrument_atomic_read_write(v, sizeof(*v));
1340 return arch_atomic_long_fetch_add_acquire(i, v);
1343 static __always_inline long
1344 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1347 instrument_atomic_read_write(v, sizeof(*v));
1348 return arch_atomic_long_fetch_add_release(i, v);
1351 static __always_inline long
1352 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1354 instrument_atomic_read_write(v, sizeof(*v));
1355 return arch_atomic_long_fetch_add_relaxed(i, v);
1358 static __always_inline void
1359 atomic_long_sub(long i, atomic_long_t *v)
1361 instrument_atomic_read_write(v, sizeof(*v));
1362 arch_atomic_long_sub(i, v);
1365 static __always_inline long
1366 atomic_long_sub_return(long i, atomic_long_t *v)
1369 instrument_atomic_read_write(v, sizeof(*v));
1370 return arch_atomic_long_sub_return(i, v);
1373 static __always_inline long
1374 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1376 instrument_atomic_read_write(v, sizeof(*v));
1377 return arch_atomic_long_sub_return_acquire(i, v);
1380 static __always_inline long
1381 atomic_long_sub_return_release(long i, atomic_long_t *v)
1384 instrument_atomic_read_write(v, sizeof(*v));
1385 return arch_atomic_long_sub_return_release(i, v);
1388 static __always_inline long
1389 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1391 instrument_atomic_read_write(v, sizeof(*v));
1392 return arch_atomic_long_sub_return_relaxed(i, v);
1395 static __always_inline long
1396 atomic_long_fetch_sub(long i, atomic_long_t *v)
1399 instrument_atomic_read_write(v, sizeof(*v));
1400 return arch_atomic_long_fetch_sub(i, v);
1403 static __always_inline long
1404 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1406 instrument_atomic_read_write(v, sizeof(*v));
1407 return arch_atomic_long_fetch_sub_acquire(i, v);
1410 static __always_inline long
1411 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1414 instrument_atomic_read_write(v, sizeof(*v));
1415 return arch_atomic_long_fetch_sub_release(i, v);
1418 static __always_inline long
1419 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1421 instrument_atomic_read_write(v, sizeof(*v));
1422 return arch_atomic_long_fetch_sub_relaxed(i, v);
1425 static __always_inline void
1426 atomic_long_inc(atomic_long_t *v)
1428 instrument_atomic_read_write(v, sizeof(*v));
1429 arch_atomic_long_inc(v);
1432 static __always_inline long
1433 atomic_long_inc_return(atomic_long_t *v)
1436 instrument_atomic_read_write(v, sizeof(*v));
1437 return arch_atomic_long_inc_return(v);
1440 static __always_inline long
1441 atomic_long_inc_return_acquire(atomic_long_t *v)
1443 instrument_atomic_read_write(v, sizeof(*v));
1444 return arch_atomic_long_inc_return_acquire(v);
1447 static __always_inline long
1448 atomic_long_inc_return_release(atomic_long_t *v)
1451 instrument_atomic_read_write(v, sizeof(*v));
1452 return arch_atomic_long_inc_return_release(v);
1455 static __always_inline long
1456 atomic_long_inc_return_relaxed(atomic_long_t *v)
1458 instrument_atomic_read_write(v, sizeof(*v));
1459 return arch_atomic_long_inc_return_relaxed(v);
1462 static __always_inline long
1463 atomic_long_fetch_inc(atomic_long_t *v)
1466 instrument_atomic_read_write(v, sizeof(*v));
1467 return arch_atomic_long_fetch_inc(v);
1470 static __always_inline long
1471 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1473 instrument_atomic_read_write(v, sizeof(*v));
1474 return arch_atomic_long_fetch_inc_acquire(v);
1477 static __always_inline long
1478 atomic_long_fetch_inc_release(atomic_long_t *v)
1481 instrument_atomic_read_write(v, sizeof(*v));
1482 return arch_atomic_long_fetch_inc_release(v);
1485 static __always_inline long
1486 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1488 instrument_atomic_read_write(v, sizeof(*v));
1489 return arch_atomic_long_fetch_inc_relaxed(v);
1492 static __always_inline void
1493 atomic_long_dec(atomic_long_t *v)
1495 instrument_atomic_read_write(v, sizeof(*v));
1496 arch_atomic_long_dec(v);
1499 static __always_inline long
1500 atomic_long_dec_return(atomic_long_t *v)
1503 instrument_atomic_read_write(v, sizeof(*v));
1504 return arch_atomic_long_dec_return(v);
1507 static __always_inline long
1508 atomic_long_dec_return_acquire(atomic_long_t *v)
1510 instrument_atomic_read_write(v, sizeof(*v));
1511 return arch_atomic_long_dec_return_acquire(v);
1514 static __always_inline long
1515 atomic_long_dec_return_release(atomic_long_t *v)
1518 instrument_atomic_read_write(v, sizeof(*v));
1519 return arch_atomic_long_dec_return_release(v);
1522 static __always_inline long
1523 atomic_long_dec_return_relaxed(atomic_long_t *v)
1525 instrument_atomic_read_write(v, sizeof(*v));
1526 return arch_atomic_long_dec_return_relaxed(v);
1529 static __always_inline long
1530 atomic_long_fetch_dec(atomic_long_t *v)
1533 instrument_atomic_read_write(v, sizeof(*v));
1534 return arch_atomic_long_fetch_dec(v);
1537 static __always_inline long
1538 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1540 instrument_atomic_read_write(v, sizeof(*v));
1541 return arch_atomic_long_fetch_dec_acquire(v);
1544 static __always_inline long
1545 atomic_long_fetch_dec_release(atomic_long_t *v)
1548 instrument_atomic_read_write(v, sizeof(*v));
1549 return arch_atomic_long_fetch_dec_release(v);
1552 static __always_inline long
1553 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1555 instrument_atomic_read_write(v, sizeof(*v));
1556 return arch_atomic_long_fetch_dec_relaxed(v);
1559 static __always_inline void
1560 atomic_long_and(long i, atomic_long_t *v)
1562 instrument_atomic_read_write(v, sizeof(*v));
1563 arch_atomic_long_and(i, v);
1566 static __always_inline long
1567 atomic_long_fetch_and(long i, atomic_long_t *v)
1570 instrument_atomic_read_write(v, sizeof(*v));
1571 return arch_atomic_long_fetch_and(i, v);
1574 static __always_inline long
1575 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1577 instrument_atomic_read_write(v, sizeof(*v));
1578 return arch_atomic_long_fetch_and_acquire(i, v);
1581 static __always_inline long
1582 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1585 instrument_atomic_read_write(v, sizeof(*v));
1586 return arch_atomic_long_fetch_and_release(i, v);
1589 static __always_inline long
1590 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1592 instrument_atomic_read_write(v, sizeof(*v));
1593 return arch_atomic_long_fetch_and_relaxed(i, v);
1596 static __always_inline void
1597 atomic_long_andnot(long i, atomic_long_t *v)
1599 instrument_atomic_read_write(v, sizeof(*v));
1600 arch_atomic_long_andnot(i, v);
1603 static __always_inline long
1604 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1607 instrument_atomic_read_write(v, sizeof(*v));
1608 return arch_atomic_long_fetch_andnot(i, v);
1611 static __always_inline long
1612 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1614 instrument_atomic_read_write(v, sizeof(*v));
1615 return arch_atomic_long_fetch_andnot_acquire(i, v);
1618 static __always_inline long
1619 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1622 instrument_atomic_read_write(v, sizeof(*v));
1623 return arch_atomic_long_fetch_andnot_release(i, v);
1626 static __always_inline long
1627 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1629 instrument_atomic_read_write(v, sizeof(*v));
1630 return arch_atomic_long_fetch_andnot_relaxed(i, v);
1633 static __always_inline void
1634 atomic_long_or(long i, atomic_long_t *v)
1636 instrument_atomic_read_write(v, sizeof(*v));
1637 arch_atomic_long_or(i, v);
1640 static __always_inline long
1641 atomic_long_fetch_or(long i, atomic_long_t *v)
1644 instrument_atomic_read_write(v, sizeof(*v));
1645 return arch_atomic_long_fetch_or(i, v);
1648 static __always_inline long
1649 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1651 instrument_atomic_read_write(v, sizeof(*v));
1652 return arch_atomic_long_fetch_or_acquire(i, v);
1655 static __always_inline long
1656 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1659 instrument_atomic_read_write(v, sizeof(*v));
1660 return arch_atomic_long_fetch_or_release(i, v);
1663 static __always_inline long
1664 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1666 instrument_atomic_read_write(v, sizeof(*v));
1667 return arch_atomic_long_fetch_or_relaxed(i, v);
1670 static __always_inline void
1671 atomic_long_xor(long i, atomic_long_t *v)
1673 instrument_atomic_read_write(v, sizeof(*v));
1674 arch_atomic_long_xor(i, v);
1677 static __always_inline long
1678 atomic_long_fetch_xor(long i, atomic_long_t *v)
1681 instrument_atomic_read_write(v, sizeof(*v));
1682 return arch_atomic_long_fetch_xor(i, v);
1685 static __always_inline long
1686 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1688 instrument_atomic_read_write(v, sizeof(*v));
1689 return arch_atomic_long_fetch_xor_acquire(i, v);
1692 static __always_inline long
1693 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1696 instrument_atomic_read_write(v, sizeof(*v));
1697 return arch_atomic_long_fetch_xor_release(i, v);
1700 static __always_inline long
1701 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1703 instrument_atomic_read_write(v, sizeof(*v));
1704 return arch_atomic_long_fetch_xor_relaxed(i, v);
1707 static __always_inline long
1708 atomic_long_xchg(atomic_long_t *v, long i)
1711 instrument_atomic_read_write(v, sizeof(*v));
1712 return arch_atomic_long_xchg(v, i);
1715 static __always_inline long
1716 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1718 instrument_atomic_read_write(v, sizeof(*v));
1719 return arch_atomic_long_xchg_acquire(v, i);
1722 static __always_inline long
1723 atomic_long_xchg_release(atomic_long_t *v, long i)
1726 instrument_atomic_read_write(v, sizeof(*v));
1727 return arch_atomic_long_xchg_release(v, i);
1730 static __always_inline long
1731 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1733 instrument_atomic_read_write(v, sizeof(*v));
1734 return arch_atomic_long_xchg_relaxed(v, i);
1737 static __always_inline long
1738 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1741 instrument_atomic_read_write(v, sizeof(*v));
1742 return arch_atomic_long_cmpxchg(v, old, new);
1745 static __always_inline long
1746 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1748 instrument_atomic_read_write(v, sizeof(*v));
1749 return arch_atomic_long_cmpxchg_acquire(v, old, new);
1752 static __always_inline long
1753 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1756 instrument_atomic_read_write(v, sizeof(*v));
1757 return arch_atomic_long_cmpxchg_release(v, old, new);
1760 static __always_inline long
1761 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1763 instrument_atomic_read_write(v, sizeof(*v));
1764 return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1767 static __always_inline bool
1768 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1771 instrument_atomic_read_write(v, sizeof(*v));
1772 instrument_atomic_read_write(old, sizeof(*old));
1773 return arch_atomic_long_try_cmpxchg(v, old, new);
1776 static __always_inline bool
1777 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1779 instrument_atomic_read_write(v, sizeof(*v));
1780 instrument_atomic_read_write(old, sizeof(*old));
1781 return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1784 static __always_inline bool
1785 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1788 instrument_atomic_read_write(v, sizeof(*v));
1789 instrument_atomic_read_write(old, sizeof(*old));
1790 return arch_atomic_long_try_cmpxchg_release(v, old, new);
1793 static __always_inline bool
1794 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1796 instrument_atomic_read_write(v, sizeof(*v));
1797 instrument_atomic_read_write(old, sizeof(*old));
1798 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1801 static __always_inline bool
1802 atomic_long_sub_and_test(long i, atomic_long_t *v)
1805 instrument_atomic_read_write(v, sizeof(*v));
1806 return arch_atomic_long_sub_and_test(i, v);
1809 static __always_inline bool
1810 atomic_long_dec_and_test(atomic_long_t *v)
1813 instrument_atomic_read_write(v, sizeof(*v));
1814 return arch_atomic_long_dec_and_test(v);
1817 static __always_inline bool
1818 atomic_long_inc_and_test(atomic_long_t *v)
1821 instrument_atomic_read_write(v, sizeof(*v));
1822 return arch_atomic_long_inc_and_test(v);
1825 static __always_inline bool
1826 atomic_long_add_negative(long i, atomic_long_t *v)
1829 instrument_atomic_read_write(v, sizeof(*v));
1830 return arch_atomic_long_add_negative(i, v);
1833 static __always_inline long
1834 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1837 instrument_atomic_read_write(v, sizeof(*v));
1838 return arch_atomic_long_fetch_add_unless(v, a, u);
1841 static __always_inline bool
1842 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1845 instrument_atomic_read_write(v, sizeof(*v));
1846 return arch_atomic_long_add_unless(v, a, u);
1849 static __always_inline bool
1850 atomic_long_inc_not_zero(atomic_long_t *v)
1853 instrument_atomic_read_write(v, sizeof(*v));
1854 return arch_atomic_long_inc_not_zero(v);
1857 static __always_inline bool
1858 atomic_long_inc_unless_negative(atomic_long_t *v)
1861 instrument_atomic_read_write(v, sizeof(*v));
1862 return arch_atomic_long_inc_unless_negative(v);
1865 static __always_inline bool
1866 atomic_long_dec_unless_positive(atomic_long_t *v)
1869 instrument_atomic_read_write(v, sizeof(*v));
1870 return arch_atomic_long_dec_unless_positive(v);
1873 static __always_inline long
1874 atomic_long_dec_if_positive(atomic_long_t *v)
1877 instrument_atomic_read_write(v, sizeof(*v));
1878 return arch_atomic_long_dec_if_positive(v);
1881 #define xchg(ptr, ...) \
1883 typeof(ptr) __ai_ptr = (ptr); \
1885 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886 arch_xchg(__ai_ptr, __VA_ARGS__); \
1889 #define xchg_acquire(ptr, ...) \
1891 typeof(ptr) __ai_ptr = (ptr); \
1892 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1896 #define xchg_release(ptr, ...) \
1898 typeof(ptr) __ai_ptr = (ptr); \
1900 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1904 #define xchg_relaxed(ptr, ...) \
1906 typeof(ptr) __ai_ptr = (ptr); \
1907 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1911 #define cmpxchg(ptr, ...) \
1913 typeof(ptr) __ai_ptr = (ptr); \
1915 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1919 #define cmpxchg_acquire(ptr, ...) \
1921 typeof(ptr) __ai_ptr = (ptr); \
1922 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1926 #define cmpxchg_release(ptr, ...) \
1928 typeof(ptr) __ai_ptr = (ptr); \
1930 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1934 #define cmpxchg_relaxed(ptr, ...) \
1936 typeof(ptr) __ai_ptr = (ptr); \
1937 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1941 #define cmpxchg64(ptr, ...) \
1943 typeof(ptr) __ai_ptr = (ptr); \
1945 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1949 #define cmpxchg64_acquire(ptr, ...) \
1951 typeof(ptr) __ai_ptr = (ptr); \
1952 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1956 #define cmpxchg64_release(ptr, ...) \
1958 typeof(ptr) __ai_ptr = (ptr); \
1960 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1964 #define cmpxchg64_relaxed(ptr, ...) \
1966 typeof(ptr) __ai_ptr = (ptr); \
1967 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1971 #define try_cmpxchg(ptr, oldp, ...) \
1973 typeof(ptr) __ai_ptr = (ptr); \
1974 typeof(oldp) __ai_oldp = (oldp); \
1976 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1981 #define try_cmpxchg_acquire(ptr, oldp, ...) \
1983 typeof(ptr) __ai_ptr = (ptr); \
1984 typeof(oldp) __ai_oldp = (oldp); \
1985 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1990 #define try_cmpxchg_release(ptr, oldp, ...) \
1992 typeof(ptr) __ai_ptr = (ptr); \
1993 typeof(oldp) __ai_oldp = (oldp); \
1995 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2000 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2002 typeof(ptr) __ai_ptr = (ptr); \
2003 typeof(oldp) __ai_oldp = (oldp); \
2004 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2009 #define cmpxchg_local(ptr, ...) \
2011 typeof(ptr) __ai_ptr = (ptr); \
2012 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2013 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2016 #define cmpxchg64_local(ptr, ...) \
2018 typeof(ptr) __ai_ptr = (ptr); \
2019 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2020 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2023 #define sync_cmpxchg(ptr, ...) \
2025 typeof(ptr) __ai_ptr = (ptr); \
2027 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2028 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2031 #define cmpxchg_double(ptr, ...) \
2033 typeof(ptr) __ai_ptr = (ptr); \
2035 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2036 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2040 #define cmpxchg_double_local(ptr, ...) \
2042 typeof(ptr) __ai_ptr = (ptr); \
2043 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2044 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2047 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2048 // 87c974b93032afd42143613434d1a7788fa598f9