1 // SPDX-License-Identifier: GPL-2.0
3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
7 * This file provides wrappers with KASAN instrumentation for atomic operations.
8 * To use this functionality an arch's atomic.h file needs to define all
9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 * this file at the end. This file provides atomic_read() that forwards to
11 * arch_atomic_read() for actual atomic operation.
12 * Note: if an arch atomic operation is implemented by means of other atomic
13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 * double instrumentation.
17 #ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
18 #define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
20 #include <linux/build_bug.h>
21 #include <linux/compiler.h>
22 #include <linux/instrumented.h>
24 static __always_inline int
25 atomic_read(const atomic_t *v)
27 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);
31 static __always_inline int
32 atomic_read_acquire(const atomic_t *v)
34 instrument_atomic_read(v, sizeof(*v));
35 return arch_atomic_read_acquire(v);
38 static __always_inline void
39 atomic_set(atomic_t *v, int i)
41 instrument_atomic_write(v, sizeof(*v));
42 arch_atomic_set(v, i);
45 static __always_inline void
46 atomic_set_release(atomic_t *v, int i)
48 instrument_atomic_write(v, sizeof(*v));
49 arch_atomic_set_release(v, i);
52 static __always_inline void
53 atomic_add(int i, atomic_t *v)
55 instrument_atomic_read_write(v, sizeof(*v));
56 arch_atomic_add(i, v);
59 static __always_inline int
60 atomic_add_return(int i, atomic_t *v)
62 instrument_atomic_read_write(v, sizeof(*v));
63 return arch_atomic_add_return(i, v);
66 static __always_inline int
67 atomic_add_return_acquire(int i, atomic_t *v)
69 instrument_atomic_read_write(v, sizeof(*v));
70 return arch_atomic_add_return_acquire(i, v);
73 static __always_inline int
74 atomic_add_return_release(int i, atomic_t *v)
76 instrument_atomic_read_write(v, sizeof(*v));
77 return arch_atomic_add_return_release(i, v);
80 static __always_inline int
81 atomic_add_return_relaxed(int i, atomic_t *v)
83 instrument_atomic_read_write(v, sizeof(*v));
84 return arch_atomic_add_return_relaxed(i, v);
87 static __always_inline int
88 atomic_fetch_add(int i, atomic_t *v)
90 instrument_atomic_read_write(v, sizeof(*v));
91 return arch_atomic_fetch_add(i, v);
94 static __always_inline int
95 atomic_fetch_add_acquire(int i, atomic_t *v)
97 instrument_atomic_read_write(v, sizeof(*v));
98 return arch_atomic_fetch_add_acquire(i, v);
101 static __always_inline int
102 atomic_fetch_add_release(int i, atomic_t *v)
104 instrument_atomic_read_write(v, sizeof(*v));
105 return arch_atomic_fetch_add_release(i, v);
108 static __always_inline int
109 atomic_fetch_add_relaxed(int i, atomic_t *v)
111 instrument_atomic_read_write(v, sizeof(*v));
112 return arch_atomic_fetch_add_relaxed(i, v);
115 static __always_inline void
116 atomic_sub(int i, atomic_t *v)
118 instrument_atomic_read_write(v, sizeof(*v));
119 arch_atomic_sub(i, v);
122 static __always_inline int
123 atomic_sub_return(int i, atomic_t *v)
125 instrument_atomic_read_write(v, sizeof(*v));
126 return arch_atomic_sub_return(i, v);
129 static __always_inline int
130 atomic_sub_return_acquire(int i, atomic_t *v)
132 instrument_atomic_read_write(v, sizeof(*v));
133 return arch_atomic_sub_return_acquire(i, v);
136 static __always_inline int
137 atomic_sub_return_release(int i, atomic_t *v)
139 instrument_atomic_read_write(v, sizeof(*v));
140 return arch_atomic_sub_return_release(i, v);
143 static __always_inline int
144 atomic_sub_return_relaxed(int i, atomic_t *v)
146 instrument_atomic_read_write(v, sizeof(*v));
147 return arch_atomic_sub_return_relaxed(i, v);
150 static __always_inline int
151 atomic_fetch_sub(int i, atomic_t *v)
153 instrument_atomic_read_write(v, sizeof(*v));
154 return arch_atomic_fetch_sub(i, v);
157 static __always_inline int
158 atomic_fetch_sub_acquire(int i, atomic_t *v)
160 instrument_atomic_read_write(v, sizeof(*v));
161 return arch_atomic_fetch_sub_acquire(i, v);
164 static __always_inline int
165 atomic_fetch_sub_release(int i, atomic_t *v)
167 instrument_atomic_read_write(v, sizeof(*v));
168 return arch_atomic_fetch_sub_release(i, v);
171 static __always_inline int
172 atomic_fetch_sub_relaxed(int i, atomic_t *v)
174 instrument_atomic_read_write(v, sizeof(*v));
175 return arch_atomic_fetch_sub_relaxed(i, v);
178 static __always_inline void
179 atomic_inc(atomic_t *v)
181 instrument_atomic_read_write(v, sizeof(*v));
185 static __always_inline int
186 atomic_inc_return(atomic_t *v)
188 instrument_atomic_read_write(v, sizeof(*v));
189 return arch_atomic_inc_return(v);
192 static __always_inline int
193 atomic_inc_return_acquire(atomic_t *v)
195 instrument_atomic_read_write(v, sizeof(*v));
196 return arch_atomic_inc_return_acquire(v);
199 static __always_inline int
200 atomic_inc_return_release(atomic_t *v)
202 instrument_atomic_read_write(v, sizeof(*v));
203 return arch_atomic_inc_return_release(v);
206 static __always_inline int
207 atomic_inc_return_relaxed(atomic_t *v)
209 instrument_atomic_read_write(v, sizeof(*v));
210 return arch_atomic_inc_return_relaxed(v);
213 static __always_inline int
214 atomic_fetch_inc(atomic_t *v)
216 instrument_atomic_read_write(v, sizeof(*v));
217 return arch_atomic_fetch_inc(v);
220 static __always_inline int
221 atomic_fetch_inc_acquire(atomic_t *v)
223 instrument_atomic_read_write(v, sizeof(*v));
224 return arch_atomic_fetch_inc_acquire(v);
227 static __always_inline int
228 atomic_fetch_inc_release(atomic_t *v)
230 instrument_atomic_read_write(v, sizeof(*v));
231 return arch_atomic_fetch_inc_release(v);
234 static __always_inline int
235 atomic_fetch_inc_relaxed(atomic_t *v)
237 instrument_atomic_read_write(v, sizeof(*v));
238 return arch_atomic_fetch_inc_relaxed(v);
241 static __always_inline void
242 atomic_dec(atomic_t *v)
244 instrument_atomic_read_write(v, sizeof(*v));
248 static __always_inline int
249 atomic_dec_return(atomic_t *v)
251 instrument_atomic_read_write(v, sizeof(*v));
252 return arch_atomic_dec_return(v);
255 static __always_inline int
256 atomic_dec_return_acquire(atomic_t *v)
258 instrument_atomic_read_write(v, sizeof(*v));
259 return arch_atomic_dec_return_acquire(v);
262 static __always_inline int
263 atomic_dec_return_release(atomic_t *v)
265 instrument_atomic_read_write(v, sizeof(*v));
266 return arch_atomic_dec_return_release(v);
269 static __always_inline int
270 atomic_dec_return_relaxed(atomic_t *v)
272 instrument_atomic_read_write(v, sizeof(*v));
273 return arch_atomic_dec_return_relaxed(v);
276 static __always_inline int
277 atomic_fetch_dec(atomic_t *v)
279 instrument_atomic_read_write(v, sizeof(*v));
280 return arch_atomic_fetch_dec(v);
283 static __always_inline int
284 atomic_fetch_dec_acquire(atomic_t *v)
286 instrument_atomic_read_write(v, sizeof(*v));
287 return arch_atomic_fetch_dec_acquire(v);
290 static __always_inline int
291 atomic_fetch_dec_release(atomic_t *v)
293 instrument_atomic_read_write(v, sizeof(*v));
294 return arch_atomic_fetch_dec_release(v);
297 static __always_inline int
298 atomic_fetch_dec_relaxed(atomic_t *v)
300 instrument_atomic_read_write(v, sizeof(*v));
301 return arch_atomic_fetch_dec_relaxed(v);
304 static __always_inline void
305 atomic_and(int i, atomic_t *v)
307 instrument_atomic_read_write(v, sizeof(*v));
308 arch_atomic_and(i, v);
311 static __always_inline int
312 atomic_fetch_and(int i, atomic_t *v)
314 instrument_atomic_read_write(v, sizeof(*v));
315 return arch_atomic_fetch_and(i, v);
318 static __always_inline int
319 atomic_fetch_and_acquire(int i, atomic_t *v)
321 instrument_atomic_read_write(v, sizeof(*v));
322 return arch_atomic_fetch_and_acquire(i, v);
325 static __always_inline int
326 atomic_fetch_and_release(int i, atomic_t *v)
328 instrument_atomic_read_write(v, sizeof(*v));
329 return arch_atomic_fetch_and_release(i, v);
332 static __always_inline int
333 atomic_fetch_and_relaxed(int i, atomic_t *v)
335 instrument_atomic_read_write(v, sizeof(*v));
336 return arch_atomic_fetch_and_relaxed(i, v);
339 static __always_inline void
340 atomic_andnot(int i, atomic_t *v)
342 instrument_atomic_read_write(v, sizeof(*v));
343 arch_atomic_andnot(i, v);
346 static __always_inline int
347 atomic_fetch_andnot(int i, atomic_t *v)
349 instrument_atomic_read_write(v, sizeof(*v));
350 return arch_atomic_fetch_andnot(i, v);
353 static __always_inline int
354 atomic_fetch_andnot_acquire(int i, atomic_t *v)
356 instrument_atomic_read_write(v, sizeof(*v));
357 return arch_atomic_fetch_andnot_acquire(i, v);
360 static __always_inline int
361 atomic_fetch_andnot_release(int i, atomic_t *v)
363 instrument_atomic_read_write(v, sizeof(*v));
364 return arch_atomic_fetch_andnot_release(i, v);
367 static __always_inline int
368 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
370 instrument_atomic_read_write(v, sizeof(*v));
371 return arch_atomic_fetch_andnot_relaxed(i, v);
374 static __always_inline void
375 atomic_or(int i, atomic_t *v)
377 instrument_atomic_read_write(v, sizeof(*v));
378 arch_atomic_or(i, v);
381 static __always_inline int
382 atomic_fetch_or(int i, atomic_t *v)
384 instrument_atomic_read_write(v, sizeof(*v));
385 return arch_atomic_fetch_or(i, v);
388 static __always_inline int
389 atomic_fetch_or_acquire(int i, atomic_t *v)
391 instrument_atomic_read_write(v, sizeof(*v));
392 return arch_atomic_fetch_or_acquire(i, v);
395 static __always_inline int
396 atomic_fetch_or_release(int i, atomic_t *v)
398 instrument_atomic_read_write(v, sizeof(*v));
399 return arch_atomic_fetch_or_release(i, v);
402 static __always_inline int
403 atomic_fetch_or_relaxed(int i, atomic_t *v)
405 instrument_atomic_read_write(v, sizeof(*v));
406 return arch_atomic_fetch_or_relaxed(i, v);
409 static __always_inline void
410 atomic_xor(int i, atomic_t *v)
412 instrument_atomic_read_write(v, sizeof(*v));
413 arch_atomic_xor(i, v);
416 static __always_inline int
417 atomic_fetch_xor(int i, atomic_t *v)
419 instrument_atomic_read_write(v, sizeof(*v));
420 return arch_atomic_fetch_xor(i, v);
423 static __always_inline int
424 atomic_fetch_xor_acquire(int i, atomic_t *v)
426 instrument_atomic_read_write(v, sizeof(*v));
427 return arch_atomic_fetch_xor_acquire(i, v);
430 static __always_inline int
431 atomic_fetch_xor_release(int i, atomic_t *v)
433 instrument_atomic_read_write(v, sizeof(*v));
434 return arch_atomic_fetch_xor_release(i, v);
437 static __always_inline int
438 atomic_fetch_xor_relaxed(int i, atomic_t *v)
440 instrument_atomic_read_write(v, sizeof(*v));
441 return arch_atomic_fetch_xor_relaxed(i, v);
444 static __always_inline int
445 atomic_xchg(atomic_t *v, int i)
447 instrument_atomic_read_write(v, sizeof(*v));
448 return arch_atomic_xchg(v, i);
451 static __always_inline int
452 atomic_xchg_acquire(atomic_t *v, int i)
454 instrument_atomic_read_write(v, sizeof(*v));
455 return arch_atomic_xchg_acquire(v, i);
458 static __always_inline int
459 atomic_xchg_release(atomic_t *v, int i)
461 instrument_atomic_read_write(v, sizeof(*v));
462 return arch_atomic_xchg_release(v, i);
465 static __always_inline int
466 atomic_xchg_relaxed(atomic_t *v, int i)
468 instrument_atomic_read_write(v, sizeof(*v));
469 return arch_atomic_xchg_relaxed(v, i);
472 static __always_inline int
473 atomic_cmpxchg(atomic_t *v, int old, int new)
475 instrument_atomic_read_write(v, sizeof(*v));
476 return arch_atomic_cmpxchg(v, old, new);
479 static __always_inline int
480 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
482 instrument_atomic_read_write(v, sizeof(*v));
483 return arch_atomic_cmpxchg_acquire(v, old, new);
486 static __always_inline int
487 atomic_cmpxchg_release(atomic_t *v, int old, int new)
489 instrument_atomic_read_write(v, sizeof(*v));
490 return arch_atomic_cmpxchg_release(v, old, new);
493 static __always_inline int
494 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
496 instrument_atomic_read_write(v, sizeof(*v));
497 return arch_atomic_cmpxchg_relaxed(v, old, new);
500 static __always_inline bool
501 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
503 instrument_atomic_read_write(v, sizeof(*v));
504 instrument_atomic_read_write(old, sizeof(*old));
505 return arch_atomic_try_cmpxchg(v, old, new);
508 static __always_inline bool
509 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
511 instrument_atomic_read_write(v, sizeof(*v));
512 instrument_atomic_read_write(old, sizeof(*old));
513 return arch_atomic_try_cmpxchg_acquire(v, old, new);
516 static __always_inline bool
517 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
519 instrument_atomic_read_write(v, sizeof(*v));
520 instrument_atomic_read_write(old, sizeof(*old));
521 return arch_atomic_try_cmpxchg_release(v, old, new);
524 static __always_inline bool
525 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
527 instrument_atomic_read_write(v, sizeof(*v));
528 instrument_atomic_read_write(old, sizeof(*old));
529 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
532 static __always_inline bool
533 atomic_sub_and_test(int i, atomic_t *v)
535 instrument_atomic_read_write(v, sizeof(*v));
536 return arch_atomic_sub_and_test(i, v);
539 static __always_inline bool
540 atomic_dec_and_test(atomic_t *v)
542 instrument_atomic_read_write(v, sizeof(*v));
543 return arch_atomic_dec_and_test(v);
546 static __always_inline bool
547 atomic_inc_and_test(atomic_t *v)
549 instrument_atomic_read_write(v, sizeof(*v));
550 return arch_atomic_inc_and_test(v);
553 static __always_inline bool
554 atomic_add_negative(int i, atomic_t *v)
556 instrument_atomic_read_write(v, sizeof(*v));
557 return arch_atomic_add_negative(i, v);
560 static __always_inline int
561 atomic_fetch_add_unless(atomic_t *v, int a, int u)
563 instrument_atomic_read_write(v, sizeof(*v));
564 return arch_atomic_fetch_add_unless(v, a, u);
567 static __always_inline bool
568 atomic_add_unless(atomic_t *v, int a, int u)
570 instrument_atomic_read_write(v, sizeof(*v));
571 return arch_atomic_add_unless(v, a, u);
574 static __always_inline bool
575 atomic_inc_not_zero(atomic_t *v)
577 instrument_atomic_read_write(v, sizeof(*v));
578 return arch_atomic_inc_not_zero(v);
581 static __always_inline bool
582 atomic_inc_unless_negative(atomic_t *v)
584 instrument_atomic_read_write(v, sizeof(*v));
585 return arch_atomic_inc_unless_negative(v);
588 static __always_inline bool
589 atomic_dec_unless_positive(atomic_t *v)
591 instrument_atomic_read_write(v, sizeof(*v));
592 return arch_atomic_dec_unless_positive(v);
595 static __always_inline int
596 atomic_dec_if_positive(atomic_t *v)
598 instrument_atomic_read_write(v, sizeof(*v));
599 return arch_atomic_dec_if_positive(v);
602 static __always_inline s64
603 atomic64_read(const atomic64_t *v)
605 instrument_atomic_read(v, sizeof(*v));
606 return arch_atomic64_read(v);
609 static __always_inline s64
610 atomic64_read_acquire(const atomic64_t *v)
612 instrument_atomic_read(v, sizeof(*v));
613 return arch_atomic64_read_acquire(v);
616 static __always_inline void
617 atomic64_set(atomic64_t *v, s64 i)
619 instrument_atomic_write(v, sizeof(*v));
620 arch_atomic64_set(v, i);
623 static __always_inline void
624 atomic64_set_release(atomic64_t *v, s64 i)
626 instrument_atomic_write(v, sizeof(*v));
627 arch_atomic64_set_release(v, i);
630 static __always_inline void
631 atomic64_add(s64 i, atomic64_t *v)
633 instrument_atomic_read_write(v, sizeof(*v));
634 arch_atomic64_add(i, v);
637 static __always_inline s64
638 atomic64_add_return(s64 i, atomic64_t *v)
640 instrument_atomic_read_write(v, sizeof(*v));
641 return arch_atomic64_add_return(i, v);
644 static __always_inline s64
645 atomic64_add_return_acquire(s64 i, atomic64_t *v)
647 instrument_atomic_read_write(v, sizeof(*v));
648 return arch_atomic64_add_return_acquire(i, v);
651 static __always_inline s64
652 atomic64_add_return_release(s64 i, atomic64_t *v)
654 instrument_atomic_read_write(v, sizeof(*v));
655 return arch_atomic64_add_return_release(i, v);
658 static __always_inline s64
659 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
661 instrument_atomic_read_write(v, sizeof(*v));
662 return arch_atomic64_add_return_relaxed(i, v);
665 static __always_inline s64
666 atomic64_fetch_add(s64 i, atomic64_t *v)
668 instrument_atomic_read_write(v, sizeof(*v));
669 return arch_atomic64_fetch_add(i, v);
672 static __always_inline s64
673 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
675 instrument_atomic_read_write(v, sizeof(*v));
676 return arch_atomic64_fetch_add_acquire(i, v);
679 static __always_inline s64
680 atomic64_fetch_add_release(s64 i, atomic64_t *v)
682 instrument_atomic_read_write(v, sizeof(*v));
683 return arch_atomic64_fetch_add_release(i, v);
686 static __always_inline s64
687 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
689 instrument_atomic_read_write(v, sizeof(*v));
690 return arch_atomic64_fetch_add_relaxed(i, v);
693 static __always_inline void
694 atomic64_sub(s64 i, atomic64_t *v)
696 instrument_atomic_read_write(v, sizeof(*v));
697 arch_atomic64_sub(i, v);
700 static __always_inline s64
701 atomic64_sub_return(s64 i, atomic64_t *v)
703 instrument_atomic_read_write(v, sizeof(*v));
704 return arch_atomic64_sub_return(i, v);
707 static __always_inline s64
708 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
710 instrument_atomic_read_write(v, sizeof(*v));
711 return arch_atomic64_sub_return_acquire(i, v);
714 static __always_inline s64
715 atomic64_sub_return_release(s64 i, atomic64_t *v)
717 instrument_atomic_read_write(v, sizeof(*v));
718 return arch_atomic64_sub_return_release(i, v);
721 static __always_inline s64
722 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
724 instrument_atomic_read_write(v, sizeof(*v));
725 return arch_atomic64_sub_return_relaxed(i, v);
728 static __always_inline s64
729 atomic64_fetch_sub(s64 i, atomic64_t *v)
731 instrument_atomic_read_write(v, sizeof(*v));
732 return arch_atomic64_fetch_sub(i, v);
735 static __always_inline s64
736 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
738 instrument_atomic_read_write(v, sizeof(*v));
739 return arch_atomic64_fetch_sub_acquire(i, v);
742 static __always_inline s64
743 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
745 instrument_atomic_read_write(v, sizeof(*v));
746 return arch_atomic64_fetch_sub_release(i, v);
749 static __always_inline s64
750 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
752 instrument_atomic_read_write(v, sizeof(*v));
753 return arch_atomic64_fetch_sub_relaxed(i, v);
756 static __always_inline void
757 atomic64_inc(atomic64_t *v)
759 instrument_atomic_read_write(v, sizeof(*v));
760 arch_atomic64_inc(v);
763 static __always_inline s64
764 atomic64_inc_return(atomic64_t *v)
766 instrument_atomic_read_write(v, sizeof(*v));
767 return arch_atomic64_inc_return(v);
770 static __always_inline s64
771 atomic64_inc_return_acquire(atomic64_t *v)
773 instrument_atomic_read_write(v, sizeof(*v));
774 return arch_atomic64_inc_return_acquire(v);
777 static __always_inline s64
778 atomic64_inc_return_release(atomic64_t *v)
780 instrument_atomic_read_write(v, sizeof(*v));
781 return arch_atomic64_inc_return_release(v);
784 static __always_inline s64
785 atomic64_inc_return_relaxed(atomic64_t *v)
787 instrument_atomic_read_write(v, sizeof(*v));
788 return arch_atomic64_inc_return_relaxed(v);
791 static __always_inline s64
792 atomic64_fetch_inc(atomic64_t *v)
794 instrument_atomic_read_write(v, sizeof(*v));
795 return arch_atomic64_fetch_inc(v);
798 static __always_inline s64
799 atomic64_fetch_inc_acquire(atomic64_t *v)
801 instrument_atomic_read_write(v, sizeof(*v));
802 return arch_atomic64_fetch_inc_acquire(v);
805 static __always_inline s64
806 atomic64_fetch_inc_release(atomic64_t *v)
808 instrument_atomic_read_write(v, sizeof(*v));
809 return arch_atomic64_fetch_inc_release(v);
812 static __always_inline s64
813 atomic64_fetch_inc_relaxed(atomic64_t *v)
815 instrument_atomic_read_write(v, sizeof(*v));
816 return arch_atomic64_fetch_inc_relaxed(v);
819 static __always_inline void
820 atomic64_dec(atomic64_t *v)
822 instrument_atomic_read_write(v, sizeof(*v));
823 arch_atomic64_dec(v);
826 static __always_inline s64
827 atomic64_dec_return(atomic64_t *v)
829 instrument_atomic_read_write(v, sizeof(*v));
830 return arch_atomic64_dec_return(v);
833 static __always_inline s64
834 atomic64_dec_return_acquire(atomic64_t *v)
836 instrument_atomic_read_write(v, sizeof(*v));
837 return arch_atomic64_dec_return_acquire(v);
840 static __always_inline s64
841 atomic64_dec_return_release(atomic64_t *v)
843 instrument_atomic_read_write(v, sizeof(*v));
844 return arch_atomic64_dec_return_release(v);
847 static __always_inline s64
848 atomic64_dec_return_relaxed(atomic64_t *v)
850 instrument_atomic_read_write(v, sizeof(*v));
851 return arch_atomic64_dec_return_relaxed(v);
854 static __always_inline s64
855 atomic64_fetch_dec(atomic64_t *v)
857 instrument_atomic_read_write(v, sizeof(*v));
858 return arch_atomic64_fetch_dec(v);
861 static __always_inline s64
862 atomic64_fetch_dec_acquire(atomic64_t *v)
864 instrument_atomic_read_write(v, sizeof(*v));
865 return arch_atomic64_fetch_dec_acquire(v);
868 static __always_inline s64
869 atomic64_fetch_dec_release(atomic64_t *v)
871 instrument_atomic_read_write(v, sizeof(*v));
872 return arch_atomic64_fetch_dec_release(v);
875 static __always_inline s64
876 atomic64_fetch_dec_relaxed(atomic64_t *v)
878 instrument_atomic_read_write(v, sizeof(*v));
879 return arch_atomic64_fetch_dec_relaxed(v);
882 static __always_inline void
883 atomic64_and(s64 i, atomic64_t *v)
885 instrument_atomic_read_write(v, sizeof(*v));
886 arch_atomic64_and(i, v);
889 static __always_inline s64
890 atomic64_fetch_and(s64 i, atomic64_t *v)
892 instrument_atomic_read_write(v, sizeof(*v));
893 return arch_atomic64_fetch_and(i, v);
896 static __always_inline s64
897 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
899 instrument_atomic_read_write(v, sizeof(*v));
900 return arch_atomic64_fetch_and_acquire(i, v);
903 static __always_inline s64
904 atomic64_fetch_and_release(s64 i, atomic64_t *v)
906 instrument_atomic_read_write(v, sizeof(*v));
907 return arch_atomic64_fetch_and_release(i, v);
910 static __always_inline s64
911 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
913 instrument_atomic_read_write(v, sizeof(*v));
914 return arch_atomic64_fetch_and_relaxed(i, v);
917 static __always_inline void
918 atomic64_andnot(s64 i, atomic64_t *v)
920 instrument_atomic_read_write(v, sizeof(*v));
921 arch_atomic64_andnot(i, v);
924 static __always_inline s64
925 atomic64_fetch_andnot(s64 i, atomic64_t *v)
927 instrument_atomic_read_write(v, sizeof(*v));
928 return arch_atomic64_fetch_andnot(i, v);
931 static __always_inline s64
932 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
934 instrument_atomic_read_write(v, sizeof(*v));
935 return arch_atomic64_fetch_andnot_acquire(i, v);
938 static __always_inline s64
939 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
941 instrument_atomic_read_write(v, sizeof(*v));
942 return arch_atomic64_fetch_andnot_release(i, v);
945 static __always_inline s64
946 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
948 instrument_atomic_read_write(v, sizeof(*v));
949 return arch_atomic64_fetch_andnot_relaxed(i, v);
952 static __always_inline void
953 atomic64_or(s64 i, atomic64_t *v)
955 instrument_atomic_read_write(v, sizeof(*v));
956 arch_atomic64_or(i, v);
959 static __always_inline s64
960 atomic64_fetch_or(s64 i, atomic64_t *v)
962 instrument_atomic_read_write(v, sizeof(*v));
963 return arch_atomic64_fetch_or(i, v);
966 static __always_inline s64
967 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
969 instrument_atomic_read_write(v, sizeof(*v));
970 return arch_atomic64_fetch_or_acquire(i, v);
973 static __always_inline s64
974 atomic64_fetch_or_release(s64 i, atomic64_t *v)
976 instrument_atomic_read_write(v, sizeof(*v));
977 return arch_atomic64_fetch_or_release(i, v);
980 static __always_inline s64
981 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
983 instrument_atomic_read_write(v, sizeof(*v));
984 return arch_atomic64_fetch_or_relaxed(i, v);
987 static __always_inline void
988 atomic64_xor(s64 i, atomic64_t *v)
990 instrument_atomic_read_write(v, sizeof(*v));
991 arch_atomic64_xor(i, v);
994 static __always_inline s64
995 atomic64_fetch_xor(s64 i, atomic64_t *v)
997 instrument_atomic_read_write(v, sizeof(*v));
998 return arch_atomic64_fetch_xor(i, v);
1001 static __always_inline s64
1002 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1004 instrument_atomic_read_write(v, sizeof(*v));
1005 return arch_atomic64_fetch_xor_acquire(i, v);
1008 static __always_inline s64
1009 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1011 instrument_atomic_read_write(v, sizeof(*v));
1012 return arch_atomic64_fetch_xor_release(i, v);
1015 static __always_inline s64
1016 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1018 instrument_atomic_read_write(v, sizeof(*v));
1019 return arch_atomic64_fetch_xor_relaxed(i, v);
1022 static __always_inline s64
1023 atomic64_xchg(atomic64_t *v, s64 i)
1025 instrument_atomic_read_write(v, sizeof(*v));
1026 return arch_atomic64_xchg(v, i);
1029 static __always_inline s64
1030 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1032 instrument_atomic_read_write(v, sizeof(*v));
1033 return arch_atomic64_xchg_acquire(v, i);
1036 static __always_inline s64
1037 atomic64_xchg_release(atomic64_t *v, s64 i)
1039 instrument_atomic_read_write(v, sizeof(*v));
1040 return arch_atomic64_xchg_release(v, i);
1043 static __always_inline s64
1044 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1046 instrument_atomic_read_write(v, sizeof(*v));
1047 return arch_atomic64_xchg_relaxed(v, i);
1050 static __always_inline s64
1051 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1053 instrument_atomic_read_write(v, sizeof(*v));
1054 return arch_atomic64_cmpxchg(v, old, new);
1057 static __always_inline s64
1058 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1060 instrument_atomic_read_write(v, sizeof(*v));
1061 return arch_atomic64_cmpxchg_acquire(v, old, new);
1064 static __always_inline s64
1065 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1067 instrument_atomic_read_write(v, sizeof(*v));
1068 return arch_atomic64_cmpxchg_release(v, old, new);
1071 static __always_inline s64
1072 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1074 instrument_atomic_read_write(v, sizeof(*v));
1075 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1078 static __always_inline bool
1079 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1081 instrument_atomic_read_write(v, sizeof(*v));
1082 instrument_atomic_read_write(old, sizeof(*old));
1083 return arch_atomic64_try_cmpxchg(v, old, new);
1086 static __always_inline bool
1087 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1089 instrument_atomic_read_write(v, sizeof(*v));
1090 instrument_atomic_read_write(old, sizeof(*old));
1091 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1094 static __always_inline bool
1095 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1097 instrument_atomic_read_write(v, sizeof(*v));
1098 instrument_atomic_read_write(old, sizeof(*old));
1099 return arch_atomic64_try_cmpxchg_release(v, old, new);
1102 static __always_inline bool
1103 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1105 instrument_atomic_read_write(v, sizeof(*v));
1106 instrument_atomic_read_write(old, sizeof(*old));
1107 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1110 static __always_inline bool
1111 atomic64_sub_and_test(s64 i, atomic64_t *v)
1113 instrument_atomic_read_write(v, sizeof(*v));
1114 return arch_atomic64_sub_and_test(i, v);
1117 static __always_inline bool
1118 atomic64_dec_and_test(atomic64_t *v)
1120 instrument_atomic_read_write(v, sizeof(*v));
1121 return arch_atomic64_dec_and_test(v);
1124 static __always_inline bool
1125 atomic64_inc_and_test(atomic64_t *v)
1127 instrument_atomic_read_write(v, sizeof(*v));
1128 return arch_atomic64_inc_and_test(v);
1131 static __always_inline bool
1132 atomic64_add_negative(s64 i, atomic64_t *v)
1134 instrument_atomic_read_write(v, sizeof(*v));
1135 return arch_atomic64_add_negative(i, v);
1138 static __always_inline s64
1139 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1141 instrument_atomic_read_write(v, sizeof(*v));
1142 return arch_atomic64_fetch_add_unless(v, a, u);
1145 static __always_inline bool
1146 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1148 instrument_atomic_read_write(v, sizeof(*v));
1149 return arch_atomic64_add_unless(v, a, u);
1152 static __always_inline bool
1153 atomic64_inc_not_zero(atomic64_t *v)
1155 instrument_atomic_read_write(v, sizeof(*v));
1156 return arch_atomic64_inc_not_zero(v);
1159 static __always_inline bool
1160 atomic64_inc_unless_negative(atomic64_t *v)
1162 instrument_atomic_read_write(v, sizeof(*v));
1163 return arch_atomic64_inc_unless_negative(v);
1166 static __always_inline bool
1167 atomic64_dec_unless_positive(atomic64_t *v)
1169 instrument_atomic_read_write(v, sizeof(*v));
1170 return arch_atomic64_dec_unless_positive(v);
1173 static __always_inline s64
1174 atomic64_dec_if_positive(atomic64_t *v)
1176 instrument_atomic_read_write(v, sizeof(*v));
1177 return arch_atomic64_dec_if_positive(v);
1180 #define xchg(ptr, ...) \
1182 typeof(ptr) __ai_ptr = (ptr); \
1183 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1184 arch_xchg(__ai_ptr, __VA_ARGS__); \
1187 #define xchg_acquire(ptr, ...) \
1189 typeof(ptr) __ai_ptr = (ptr); \
1190 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1191 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1194 #define xchg_release(ptr, ...) \
1196 typeof(ptr) __ai_ptr = (ptr); \
1197 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1198 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1201 #define xchg_relaxed(ptr, ...) \
1203 typeof(ptr) __ai_ptr = (ptr); \
1204 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1205 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1208 #define cmpxchg(ptr, ...) \
1210 typeof(ptr) __ai_ptr = (ptr); \
1211 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1212 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1215 #define cmpxchg_acquire(ptr, ...) \
1217 typeof(ptr) __ai_ptr = (ptr); \
1218 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1219 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1222 #define cmpxchg_release(ptr, ...) \
1224 typeof(ptr) __ai_ptr = (ptr); \
1225 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1226 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1229 #define cmpxchg_relaxed(ptr, ...) \
1231 typeof(ptr) __ai_ptr = (ptr); \
1232 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1233 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1236 #define cmpxchg64(ptr, ...) \
1238 typeof(ptr) __ai_ptr = (ptr); \
1239 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1240 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1243 #define cmpxchg64_acquire(ptr, ...) \
1245 typeof(ptr) __ai_ptr = (ptr); \
1246 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1247 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1250 #define cmpxchg64_release(ptr, ...) \
1252 typeof(ptr) __ai_ptr = (ptr); \
1253 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1254 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1257 #define cmpxchg64_relaxed(ptr, ...) \
1259 typeof(ptr) __ai_ptr = (ptr); \
1260 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1261 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1264 #define try_cmpxchg(ptr, oldp, ...) \
1266 typeof(ptr) __ai_ptr = (ptr); \
1267 typeof(oldp) __ai_oldp = (oldp); \
1268 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1269 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1270 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1273 #define try_cmpxchg_acquire(ptr, oldp, ...) \
1275 typeof(ptr) __ai_ptr = (ptr); \
1276 typeof(oldp) __ai_oldp = (oldp); \
1277 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1278 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1279 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1282 #define try_cmpxchg_release(ptr, oldp, ...) \
1284 typeof(ptr) __ai_ptr = (ptr); \
1285 typeof(oldp) __ai_oldp = (oldp); \
1286 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1287 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1288 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1291 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
1293 typeof(ptr) __ai_ptr = (ptr); \
1294 typeof(oldp) __ai_oldp = (oldp); \
1295 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1296 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1297 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1300 #define cmpxchg_local(ptr, ...) \
1302 typeof(ptr) __ai_ptr = (ptr); \
1303 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1304 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
1307 #define cmpxchg64_local(ptr, ...) \
1309 typeof(ptr) __ai_ptr = (ptr); \
1310 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1311 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
1314 #define sync_cmpxchg(ptr, ...) \
1316 typeof(ptr) __ai_ptr = (ptr); \
1317 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1318 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
1321 #define cmpxchg_double(ptr, ...) \
1323 typeof(ptr) __ai_ptr = (ptr); \
1324 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1325 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
1329 #define cmpxchg_double_local(ptr, ...) \
1331 typeof(ptr) __ai_ptr = (ptr); \
1332 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1333 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
1336 #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
1337 // 1d7c3a25aca5c7fb031c307be4c3d24c7b48fcd5