1// SPDX-License-Identifier: GPL-2.0
2
3// Generated by scripts/atomic/gen-atomic-instrumented.sh
4// DO NOT MODIFY THIS FILE DIRECTLY
5
6/*
7 * This file provides wrappers with KASAN instrumentation for atomic operations.
8 * To use this functionality an arch's atomic.h file needs to define all
9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 * this file at the end. This file provides atomic_read() that forwards to
11 * arch_atomic_read() for actual atomic operation.
12 * Note: if an arch atomic operation is implemented by means of other atomic
13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 * double instrumentation.
16 */
17#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18#define _LINUX_ATOMIC_INSTRUMENTED_H
19
20#include <linux/build_bug.h>
21#include <linux/compiler.h>
22#include <linux/instrumented.h>
23
24static __always_inline int
25atomic_read(const atomic_t *v)
26{
27 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);
29}
30
31static __always_inline int
32atomic_read_acquire(const atomic_t *v)
33{
34 instrument_atomic_read(v, sizeof(*v));
35 return arch_atomic_read_acquire(v);
36}
37
38static __always_inline void
39atomic_set(atomic_t *v, int i)
40{
41 instrument_atomic_write(v, sizeof(*v));
42 arch_atomic_set(v, i);
43}
44
45static __always_inline void
46atomic_set_release(atomic_t *v, int i)
47{
48 kcsan_release();
49 instrument_atomic_write(v, sizeof(*v));
50 arch_atomic_set_release(v, i);
51}
52
53static __always_inline void
54atomic_add(int i, atomic_t *v)
55{
56 instrument_atomic_read_write(v, sizeof(*v));
57 arch_atomic_add(i, v);
58}
59
60static __always_inline int
61atomic_add_return(int i, atomic_t *v)
62{
63 kcsan_mb();
64 instrument_atomic_read_write(v, sizeof(*v));
65 return arch_atomic_add_return(i, v);
66}
67
68static __always_inline int
69atomic_add_return_acquire(int i, atomic_t *v)
70{
71 instrument_atomic_read_write(v, sizeof(*v));
72 return arch_atomic_add_return_acquire(i, v);
73}
74
75static __always_inline int
76atomic_add_return_release(int i, atomic_t *v)
77{
78 kcsan_release();
79 instrument_atomic_read_write(v, sizeof(*v));
80 return arch_atomic_add_return_release(i, v);
81}
82
83static __always_inline int
84atomic_add_return_relaxed(int i, atomic_t *v)
85{
86 instrument_atomic_read_write(v, sizeof(*v));
87 return arch_atomic_add_return_relaxed(i, v);
88}
89
90static __always_inline int
91atomic_fetch_add(int i, atomic_t *v)
92{
93 kcsan_mb();
94 instrument_atomic_read_write(v, sizeof(*v));
95 return arch_atomic_fetch_add(i, v);
96}
97
98static __always_inline int
99atomic_fetch_add_acquire(int i, atomic_t *v)
100{
101 instrument_atomic_read_write(v, sizeof(*v));
102 return arch_atomic_fetch_add_acquire(i, v);
103}
104
105static __always_inline int
106atomic_fetch_add_release(int i, atomic_t *v)
107{
108 kcsan_release();
109 instrument_atomic_read_write(v, sizeof(*v));
110 return arch_atomic_fetch_add_release(i, v);
111}
112
113static __always_inline int
114atomic_fetch_add_relaxed(int i, atomic_t *v)
115{
116 instrument_atomic_read_write(v, sizeof(*v));
117 return arch_atomic_fetch_add_relaxed(i, v);
118}
119
120static __always_inline void
121atomic_sub(int i, atomic_t *v)
122{
123 instrument_atomic_read_write(v, sizeof(*v));
124 arch_atomic_sub(i, v);
125}
126
127static __always_inline int
128atomic_sub_return(int i, atomic_t *v)
129{
130 kcsan_mb();
131 instrument_atomic_read_write(v, sizeof(*v));
132 return arch_atomic_sub_return(i, v);
133}
134
135static __always_inline int
136atomic_sub_return_acquire(int i, atomic_t *v)
137{
138 instrument_atomic_read_write(v, sizeof(*v));
139 return arch_atomic_sub_return_acquire(i, v);
140}
141
142static __always_inline int
143atomic_sub_return_release(int i, atomic_t *v)
144{
145 kcsan_release();
146 instrument_atomic_read_write(v, sizeof(*v));
147 return arch_atomic_sub_return_release(i, v);
148}
149
150static __always_inline int
151atomic_sub_return_relaxed(int i, atomic_t *v)
152{
153 instrument_atomic_read_write(v, sizeof(*v));
154 return arch_atomic_sub_return_relaxed(i, v);
155}
156
157static __always_inline int
158atomic_fetch_sub(int i, atomic_t *v)
159{
160 kcsan_mb();
161 instrument_atomic_read_write(v, sizeof(*v));
162 return arch_atomic_fetch_sub(i, v);
163}
164
165static __always_inline int
166atomic_fetch_sub_acquire(int i, atomic_t *v)
167{
168 instrument_atomic_read_write(v, sizeof(*v));
169 return arch_atomic_fetch_sub_acquire(i, v);
170}
171
172static __always_inline int
173atomic_fetch_sub_release(int i, atomic_t *v)
174{
175 kcsan_release();
176 instrument_atomic_read_write(v, sizeof(*v));
177 return arch_atomic_fetch_sub_release(i, v);
178}
179
180static __always_inline int
181atomic_fetch_sub_relaxed(int i, atomic_t *v)
182{
183 instrument_atomic_read_write(v, sizeof(*v));
184 return arch_atomic_fetch_sub_relaxed(i, v);
185}
186
187static __always_inline void
188atomic_inc(atomic_t *v)
189{
190 instrument_atomic_read_write(v, sizeof(*v));
191 arch_atomic_inc(v);
192}
193
194static __always_inline int
195atomic_inc_return(atomic_t *v)
196{
197 kcsan_mb();
198 instrument_atomic_read_write(v, sizeof(*v));
199 return arch_atomic_inc_return(v);
200}
201
202static __always_inline int
203atomic_inc_return_acquire(atomic_t *v)
204{
205 instrument_atomic_read_write(v, sizeof(*v));
206 return arch_atomic_inc_return_acquire(v);
207}
208
209static __always_inline int
210atomic_inc_return_release(atomic_t *v)
211{
212 kcsan_release();
213 instrument_atomic_read_write(v, sizeof(*v));
214 return arch_atomic_inc_return_release(v);
215}
216
217static __always_inline int
218atomic_inc_return_relaxed(atomic_t *v)
219{
220 instrument_atomic_read_write(v, sizeof(*v));
221 return arch_atomic_inc_return_relaxed(v);
222}
223
224static __always_inline int
225atomic_fetch_inc(atomic_t *v)
226{
227 kcsan_mb();
228 instrument_atomic_read_write(v, sizeof(*v));
229 return arch_atomic_fetch_inc(v);
230}
231
232static __always_inline int
233atomic_fetch_inc_acquire(atomic_t *v)
234{
235 instrument_atomic_read_write(v, sizeof(*v));
236 return arch_atomic_fetch_inc_acquire(v);
237}
238
239static __always_inline int
240atomic_fetch_inc_release(atomic_t *v)
241{
242 kcsan_release();
243 instrument_atomic_read_write(v, sizeof(*v));
244 return arch_atomic_fetch_inc_release(v);
245}
246
247static __always_inline int
248atomic_fetch_inc_relaxed(atomic_t *v)
249{
250 instrument_atomic_read_write(v, sizeof(*v));
251 return arch_atomic_fetch_inc_relaxed(v);
252}
253
254static __always_inline void
255atomic_dec(atomic_t *v)
256{
257 instrument_atomic_read_write(v, sizeof(*v));
258 arch_atomic_dec(v);
259}
260
261static __always_inline int
262atomic_dec_return(atomic_t *v)
263{
264 kcsan_mb();
265 instrument_atomic_read_write(v, sizeof(*v));
266 return arch_atomic_dec_return(v);
267}
268
269static __always_inline int
270atomic_dec_return_acquire(atomic_t *v)
271{
272 instrument_atomic_read_write(v, sizeof(*v));
273 return arch_atomic_dec_return_acquire(v);
274}
275
276static __always_inline int
277atomic_dec_return_release(atomic_t *v)
278{
279 kcsan_release();
280 instrument_atomic_read_write(v, sizeof(*v));
281 return arch_atomic_dec_return_release(v);
282}
283
284static __always_inline int
285atomic_dec_return_relaxed(atomic_t *v)
286{
287 instrument_atomic_read_write(v, sizeof(*v));
288 return arch_atomic_dec_return_relaxed(v);
289}
290
291static __always_inline int
292atomic_fetch_dec(atomic_t *v)
293{
294 kcsan_mb();
295 instrument_atomic_read_write(v, sizeof(*v));
296 return arch_atomic_fetch_dec(v);
297}
298
299static __always_inline int
300atomic_fetch_dec_acquire(atomic_t *v)
301{
302 instrument_atomic_read_write(v, sizeof(*v));
303 return arch_atomic_fetch_dec_acquire(v);
304}
305
306static __always_inline int
307atomic_fetch_dec_release(atomic_t *v)
308{
309 kcsan_release();
310 instrument_atomic_read_write(v, sizeof(*v));
311 return arch_atomic_fetch_dec_release(v);
312}
313
314static __always_inline int
315atomic_fetch_dec_relaxed(atomic_t *v)
316{
317 instrument_atomic_read_write(v, sizeof(*v));
318 return arch_atomic_fetch_dec_relaxed(v);
319}
320
321static __always_inline void
322atomic_and(int i, atomic_t *v)
323{
324 instrument_atomic_read_write(v, sizeof(*v));
325 arch_atomic_and(i, v);
326}
327
328static __always_inline int
329atomic_fetch_and(int i, atomic_t *v)
330{
331 kcsan_mb();
332 instrument_atomic_read_write(v, sizeof(*v));
333 return arch_atomic_fetch_and(i, v);
334}
335
336static __always_inline int
337atomic_fetch_and_acquire(int i, atomic_t *v)
338{
339 instrument_atomic_read_write(v, sizeof(*v));
340 return arch_atomic_fetch_and_acquire(i, v);
341}
342
343static __always_inline int
344atomic_fetch_and_release(int i, atomic_t *v)
345{
346 kcsan_release();
347 instrument_atomic_read_write(v, sizeof(*v));
348 return arch_atomic_fetch_and_release(i, v);
349}
350
351static __always_inline int
352atomic_fetch_and_relaxed(int i, atomic_t *v)
353{
354 instrument_atomic_read_write(v, sizeof(*v));
355 return arch_atomic_fetch_and_relaxed(i, v);
356}
357
358static __always_inline void
359atomic_andnot(int i, atomic_t *v)
360{
361 instrument_atomic_read_write(v, sizeof(*v));
362 arch_atomic_andnot(i, v);
363}
364
365static __always_inline int
366atomic_fetch_andnot(int i, atomic_t *v)
367{
368 kcsan_mb();
369 instrument_atomic_read_write(v, sizeof(*v));
370 return arch_atomic_fetch_andnot(i, v);
371}
372
373static __always_inline int
374atomic_fetch_andnot_acquire(int i, atomic_t *v)
375{
376 instrument_atomic_read_write(v, sizeof(*v));
377 return arch_atomic_fetch_andnot_acquire(i, v);
378}
379
380static __always_inline int
381atomic_fetch_andnot_release(int i, atomic_t *v)
382{
383 kcsan_release();
384 instrument_atomic_read_write(v, sizeof(*v));
385 return arch_atomic_fetch_andnot_release(i, v);
386}
387
388static __always_inline int
389atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390{
391 instrument_atomic_read_write(v, sizeof(*v));
392 return arch_atomic_fetch_andnot_relaxed(i, v);
393}
394
395static __always_inline void
396atomic_or(int i, atomic_t *v)
397{
398 instrument_atomic_read_write(v, sizeof(*v));
399 arch_atomic_or(i, v);
400}
401
402static __always_inline int
403atomic_fetch_or(int i, atomic_t *v)
404{
405 kcsan_mb();
406 instrument_atomic_read_write(v, sizeof(*v));
407 return arch_atomic_fetch_or(i, v);
408}
409
410static __always_inline int
411atomic_fetch_or_acquire(int i, atomic_t *v)
412{
413 instrument_atomic_read_write(v, sizeof(*v));
414 return arch_atomic_fetch_or_acquire(i, v);
415}
416
417static __always_inline int
418atomic_fetch_or_release(int i, atomic_t *v)
419{
420 kcsan_release();
421 instrument_atomic_read_write(v, sizeof(*v));
422 return arch_atomic_fetch_or_release(i, v);
423}
424
425static __always_inline int
426atomic_fetch_or_relaxed(int i, atomic_t *v)
427{
428 instrument_atomic_read_write(v, sizeof(*v));
429 return arch_atomic_fetch_or_relaxed(i, v);
430}
431
432static __always_inline void
433atomic_xor(int i, atomic_t *v)
434{
435 instrument_atomic_read_write(v, sizeof(*v));
436 arch_atomic_xor(i, v);
437}
438
439static __always_inline int
440atomic_fetch_xor(int i, atomic_t *v)
441{
442 kcsan_mb();
443 instrument_atomic_read_write(v, sizeof(*v));
444 return arch_atomic_fetch_xor(i, v);
445}
446
447static __always_inline int
448atomic_fetch_xor_acquire(int i, atomic_t *v)
449{
450 instrument_atomic_read_write(v, sizeof(*v));
451 return arch_atomic_fetch_xor_acquire(i, v);
452}
453
454static __always_inline int
455atomic_fetch_xor_release(int i, atomic_t *v)
456{
457 kcsan_release();
458 instrument_atomic_read_write(v, sizeof(*v));
459 return arch_atomic_fetch_xor_release(i, v);
460}
461
462static __always_inline int
463atomic_fetch_xor_relaxed(int i, atomic_t *v)
464{
465 instrument_atomic_read_write(v, sizeof(*v));
466 return arch_atomic_fetch_xor_relaxed(i, v);
467}
468
469static __always_inline int
470atomic_xchg(atomic_t *v, int i)
471{
472 kcsan_mb();
473 instrument_atomic_read_write(v, sizeof(*v));
474 return arch_atomic_xchg(v, i);
475}
476
477static __always_inline int
478atomic_xchg_acquire(atomic_t *v, int i)
479{
480 instrument_atomic_read_write(v, sizeof(*v));
481 return arch_atomic_xchg_acquire(v, i);
482}
483
484static __always_inline int
485atomic_xchg_release(atomic_t *v, int i)
486{
487 kcsan_release();
488 instrument_atomic_read_write(v, sizeof(*v));
489 return arch_atomic_xchg_release(v, i);
490}
491
492static __always_inline int
493atomic_xchg_relaxed(atomic_t *v, int i)
494{
495 instrument_atomic_read_write(v, sizeof(*v));
496 return arch_atomic_xchg_relaxed(v, i);
497}
498
499static __always_inline int
500atomic_cmpxchg(atomic_t *v, int old, int new)
501{
502 kcsan_mb();
503 instrument_atomic_read_write(v, sizeof(*v));
504 return arch_atomic_cmpxchg(v, old, new);
505}
506
507static __always_inline int
508atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509{
510 instrument_atomic_read_write(v, sizeof(*v));
511 return arch_atomic_cmpxchg_acquire(v, old, new);
512}
513
514static __always_inline int
515atomic_cmpxchg_release(atomic_t *v, int old, int new)
516{
517 kcsan_release();
518 instrument_atomic_read_write(v, sizeof(*v));
519 return arch_atomic_cmpxchg_release(v, old, new);
520}
521
522static __always_inline int
523atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524{
525 instrument_atomic_read_write(v, sizeof(*v));
526 return arch_atomic_cmpxchg_relaxed(v, old, new);
527}
528
529static __always_inline bool
530atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531{
532 kcsan_mb();
533 instrument_atomic_read_write(v, sizeof(*v));
534 instrument_atomic_read_write(old, sizeof(*old));
535 return arch_atomic_try_cmpxchg(v, old, new);
536}
537
538static __always_inline bool
539atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540{
541 instrument_atomic_read_write(v, sizeof(*v));
542 instrument_atomic_read_write(old, sizeof(*old));
543 return arch_atomic_try_cmpxchg_acquire(v, old, new);
544}
545
546static __always_inline bool
547atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548{
549 kcsan_release();
550 instrument_atomic_read_write(v, sizeof(*v));
551 instrument_atomic_read_write(old, sizeof(*old));
552 return arch_atomic_try_cmpxchg_release(v, old, new);
553}
554
555static __always_inline bool
556atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557{
558 instrument_atomic_read_write(v, sizeof(*v));
559 instrument_atomic_read_write(old, sizeof(*old));
560 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561}
562
563static __always_inline bool
564atomic_sub_and_test(int i, atomic_t *v)
565{
566 kcsan_mb();
567 instrument_atomic_read_write(v, sizeof(*v));
568 return arch_atomic_sub_and_test(i, v);
569}
570
571static __always_inline bool
572atomic_dec_and_test(atomic_t *v)
573{
574 kcsan_mb();
575 instrument_atomic_read_write(v, sizeof(*v));
576 return arch_atomic_dec_and_test(v);
577}
578
579static __always_inline bool
580atomic_inc_and_test(atomic_t *v)
581{
582 kcsan_mb();
583 instrument_atomic_read_write(v, sizeof(*v));
584 return arch_atomic_inc_and_test(v);
585}
586
587static __always_inline bool
588atomic_add_negative(int i, atomic_t *v)
589{
590 kcsan_mb();
591 instrument_atomic_read_write(v, sizeof(*v));
592 return arch_atomic_add_negative(i, v);
593}
594
595static __always_inline int
596atomic_fetch_add_unless(atomic_t *v, int a, int u)
597{
598 kcsan_mb();
599 instrument_atomic_read_write(v, sizeof(*v));
600 return arch_atomic_fetch_add_unless(v, a, u);
601}
602
603static __always_inline bool
604atomic_add_unless(atomic_t *v, int a, int u)
605{
606 kcsan_mb();
607 instrument_atomic_read_write(v, sizeof(*v));
608 return arch_atomic_add_unless(v, a, u);
609}
610
611static __always_inline bool
612atomic_inc_not_zero(atomic_t *v)
613{
614 kcsan_mb();
615 instrument_atomic_read_write(v, sizeof(*v));
616 return arch_atomic_inc_not_zero(v);
617}
618
619static __always_inline bool
620atomic_inc_unless_negative(atomic_t *v)
621{
622 kcsan_mb();
623 instrument_atomic_read_write(v, sizeof(*v));
624 return arch_atomic_inc_unless_negative(v);
625}
626
627static __always_inline bool
628atomic_dec_unless_positive(atomic_t *v)
629{
630 kcsan_mb();
631 instrument_atomic_read_write(v, sizeof(*v));
632 return arch_atomic_dec_unless_positive(v);
633}
634
635static __always_inline int
636atomic_dec_if_positive(atomic_t *v)
637{
638 kcsan_mb();
639 instrument_atomic_read_write(v, sizeof(*v));
640 return arch_atomic_dec_if_positive(v);
641}
642
643static __always_inline s64
644atomic64_read(const atomic64_t *v)
645{
646 instrument_atomic_read(v, sizeof(*v));
647 return arch_atomic64_read(v);
648}
649
650static __always_inline s64
651atomic64_read_acquire(const atomic64_t *v)
652{
653 instrument_atomic_read(v, sizeof(*v));
654 return arch_atomic64_read_acquire(v);
655}
656
657static __always_inline void
658atomic64_set(atomic64_t *v, s64 i)
659{
660 instrument_atomic_write(v, sizeof(*v));
661 arch_atomic64_set(v, i);
662}
663
664static __always_inline void
665atomic64_set_release(atomic64_t *v, s64 i)
666{
667 kcsan_release();
668 instrument_atomic_write(v, sizeof(*v));
669 arch_atomic64_set_release(v, i);
670}
671
672static __always_inline void
673atomic64_add(s64 i, atomic64_t *v)
674{
675 instrument_atomic_read_write(v, sizeof(*v));
676 arch_atomic64_add(i, v);
677}
678
679static __always_inline s64
680atomic64_add_return(s64 i, atomic64_t *v)
681{
682 kcsan_mb();
683 instrument_atomic_read_write(v, sizeof(*v));
684 return arch_atomic64_add_return(i, v);
685}
686
687static __always_inline s64
688atomic64_add_return_acquire(s64 i, atomic64_t *v)
689{
690 instrument_atomic_read_write(v, sizeof(*v));
691 return arch_atomic64_add_return_acquire(i, v);
692}
693
694static __always_inline s64
695atomic64_add_return_release(s64 i, atomic64_t *v)
696{
697 kcsan_release();
698 instrument_atomic_read_write(v, sizeof(*v));
699 return arch_atomic64_add_return_release(i, v);
700}
701
702static __always_inline s64
703atomic64_add_return_relaxed(s64 i, atomic64_t *v)
704{
705 instrument_atomic_read_write(v, sizeof(*v));
706 return arch_atomic64_add_return_relaxed(i, v);
707}
708
709static __always_inline s64
710atomic64_fetch_add(s64 i, atomic64_t *v)
711{
712 kcsan_mb();
713 instrument_atomic_read_write(v, sizeof(*v));
714 return arch_atomic64_fetch_add(i, v);
715}
716
717static __always_inline s64
718atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
719{
720 instrument_atomic_read_write(v, sizeof(*v));
721 return arch_atomic64_fetch_add_acquire(i, v);
722}
723
724static __always_inline s64
725atomic64_fetch_add_release(s64 i, atomic64_t *v)
726{
727 kcsan_release();
728 instrument_atomic_read_write(v, sizeof(*v));
729 return arch_atomic64_fetch_add_release(i, v);
730}
731
732static __always_inline s64
733atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
734{
735 instrument_atomic_read_write(v, sizeof(*v));
736 return arch_atomic64_fetch_add_relaxed(i, v);
737}
738
739static __always_inline void
740atomic64_sub(s64 i, atomic64_t *v)
741{
742 instrument_atomic_read_write(v, sizeof(*v));
743 arch_atomic64_sub(i, v);
744}
745
746static __always_inline s64
747atomic64_sub_return(s64 i, atomic64_t *v)
748{
749 kcsan_mb();
750 instrument_atomic_read_write(v, sizeof(*v));
751 return arch_atomic64_sub_return(i, v);
752}
753
754static __always_inline s64
755atomic64_sub_return_acquire(s64 i, atomic64_t *v)
756{
757 instrument_atomic_read_write(v, sizeof(*v));
758 return arch_atomic64_sub_return_acquire(i, v);
759}
760
761static __always_inline s64
762atomic64_sub_return_release(s64 i, atomic64_t *v)
763{
764 kcsan_release();
765 instrument_atomic_read_write(v, sizeof(*v));
766 return arch_atomic64_sub_return_release(i, v);
767}
768
769static __always_inline s64
770atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
771{
772 instrument_atomic_read_write(v, sizeof(*v));
773 return arch_atomic64_sub_return_relaxed(i, v);
774}
775
776static __always_inline s64
777atomic64_fetch_sub(s64 i, atomic64_t *v)
778{
779 kcsan_mb();
780 instrument_atomic_read_write(v, sizeof(*v));
781 return arch_atomic64_fetch_sub(i, v);
782}
783
784static __always_inline s64
785atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
786{
787 instrument_atomic_read_write(v, sizeof(*v));
788 return arch_atomic64_fetch_sub_acquire(i, v);
789}
790
791static __always_inline s64
792atomic64_fetch_sub_release(s64 i, atomic64_t *v)
793{
794 kcsan_release();
795 instrument_atomic_read_write(v, sizeof(*v));
796 return arch_atomic64_fetch_sub_release(i, v);
797}
798
799static __always_inline s64
800atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
801{
802 instrument_atomic_read_write(v, sizeof(*v));
803 return arch_atomic64_fetch_sub_relaxed(i, v);
804}
805
806static __always_inline void
807atomic64_inc(atomic64_t *v)
808{
809 instrument_atomic_read_write(v, sizeof(*v));
810 arch_atomic64_inc(v);
811}
812
813static __always_inline s64
814atomic64_inc_return(atomic64_t *v)
815{
816 kcsan_mb();
817 instrument_atomic_read_write(v, sizeof(*v));
818 return arch_atomic64_inc_return(v);
819}
820
821static __always_inline s64
822atomic64_inc_return_acquire(atomic64_t *v)
823{
824 instrument_atomic_read_write(v, sizeof(*v));
825 return arch_atomic64_inc_return_acquire(v);
826}
827
828static __always_inline s64
829atomic64_inc_return_release(atomic64_t *v)
830{
831 kcsan_release();
832 instrument_atomic_read_write(v, sizeof(*v));
833 return arch_atomic64_inc_return_release(v);
834}
835
836static __always_inline s64
837atomic64_inc_return_relaxed(atomic64_t *v)
838{
839 instrument_atomic_read_write(v, sizeof(*v));
840 return arch_atomic64_inc_return_relaxed(v);
841}
842
843static __always_inline s64
844atomic64_fetch_inc(atomic64_t *v)
845{
846 kcsan_mb();
847 instrument_atomic_read_write(v, sizeof(*v));
848 return arch_atomic64_fetch_inc(v);
849}
850
851static __always_inline s64
852atomic64_fetch_inc_acquire(atomic64_t *v)
853{
854 instrument_atomic_read_write(v, sizeof(*v));
855 return arch_atomic64_fetch_inc_acquire(v);
856}
857
858static __always_inline s64
859atomic64_fetch_inc_release(atomic64_t *v)
860{
861 kcsan_release();
862 instrument_atomic_read_write(v, sizeof(*v));
863 return arch_atomic64_fetch_inc_release(v);
864}
865
866static __always_inline s64
867atomic64_fetch_inc_relaxed(atomic64_t *v)
868{
869 instrument_atomic_read_write(v, sizeof(*v));
870 return arch_atomic64_fetch_inc_relaxed(v);
871}
872
873static __always_inline void
874atomic64_dec(atomic64_t *v)
875{
876 instrument_atomic_read_write(v, sizeof(*v));
877 arch_atomic64_dec(v);
878}
879
880static __always_inline s64
881atomic64_dec_return(atomic64_t *v)
882{
883 kcsan_mb();
884 instrument_atomic_read_write(v, sizeof(*v));
885 return arch_atomic64_dec_return(v);
886}
887
888static __always_inline s64
889atomic64_dec_return_acquire(atomic64_t *v)
890{
891 instrument_atomic_read_write(v, sizeof(*v));
892 return arch_atomic64_dec_return_acquire(v);
893}
894
895static __always_inline s64
896atomic64_dec_return_release(atomic64_t *v)
897{
898 kcsan_release();
899 instrument_atomic_read_write(v, sizeof(*v));
900 return arch_atomic64_dec_return_release(v);
901}
902
903static __always_inline s64
904atomic64_dec_return_relaxed(atomic64_t *v)
905{
906 instrument_atomic_read_write(v, sizeof(*v));
907 return arch_atomic64_dec_return_relaxed(v);
908}
909
910static __always_inline s64
911atomic64_fetch_dec(atomic64_t *v)
912{
913 kcsan_mb();
914 instrument_atomic_read_write(v, sizeof(*v));
915 return arch_atomic64_fetch_dec(v);
916}
917
918static __always_inline s64
919atomic64_fetch_dec_acquire(atomic64_t *v)
920{
921 instrument_atomic_read_write(v, sizeof(*v));
922 return arch_atomic64_fetch_dec_acquire(v);
923}
924
925static __always_inline s64
926atomic64_fetch_dec_release(atomic64_t *v)
927{
928 kcsan_release();
929 instrument_atomic_read_write(v, sizeof(*v));
930 return arch_atomic64_fetch_dec_release(v);
931}
932
933static __always_inline s64
934atomic64_fetch_dec_relaxed(atomic64_t *v)
935{
936 instrument_atomic_read_write(v, sizeof(*v));
937 return arch_atomic64_fetch_dec_relaxed(v);
938}
939
940static __always_inline void
941atomic64_and(s64 i, atomic64_t *v)
942{
943 instrument_atomic_read_write(v, sizeof(*v));
944 arch_atomic64_and(i, v);
945}
946
947static __always_inline s64
948atomic64_fetch_and(s64 i, atomic64_t *v)
949{
950 kcsan_mb();
951 instrument_atomic_read_write(v, sizeof(*v));
952 return arch_atomic64_fetch_and(i, v);
953}
954
955static __always_inline s64
956atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
957{
958 instrument_atomic_read_write(v, sizeof(*v));
959 return arch_atomic64_fetch_and_acquire(i, v);
960}
961
962static __always_inline s64
963atomic64_fetch_and_release(s64 i, atomic64_t *v)
964{
965 kcsan_release();
966 instrument_atomic_read_write(v, sizeof(*v));
967 return arch_atomic64_fetch_and_release(i, v);
968}
969
970static __always_inline s64
971atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
972{
973 instrument_atomic_read_write(v, sizeof(*v));
974 return arch_atomic64_fetch_and_relaxed(i, v);
975}
976
977static __always_inline void
978atomic64_andnot(s64 i, atomic64_t *v)
979{
980 instrument_atomic_read_write(v, sizeof(*v));
981 arch_atomic64_andnot(i, v);
982}
983
984static __always_inline s64
985atomic64_fetch_andnot(s64 i, atomic64_t *v)
986{
987 kcsan_mb();
988 instrument_atomic_read_write(v, sizeof(*v));
989 return arch_atomic64_fetch_andnot(i, v);
990}
991
992static __always_inline s64
993atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
994{
995 instrument_atomic_read_write(v, sizeof(*v));
996 return arch_atomic64_fetch_andnot_acquire(i, v);
997}
998
999static __always_inline s64
1000atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1001{
1002 kcsan_release();
1003 instrument_atomic_read_write(v, sizeof(*v));
1004 return arch_atomic64_fetch_andnot_release(i, v);
1005}
1006
1007static __always_inline s64
1008atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1009{
1010 instrument_atomic_read_write(v, sizeof(*v));
1011 return arch_atomic64_fetch_andnot_relaxed(i, v);
1012}
1013
1014static __always_inline void
1015atomic64_or(s64 i, atomic64_t *v)
1016{
1017 instrument_atomic_read_write(v, sizeof(*v));
1018 arch_atomic64_or(i, v);
1019}
1020
1021static __always_inline s64
1022atomic64_fetch_or(s64 i, atomic64_t *v)
1023{
1024 kcsan_mb();
1025 instrument_atomic_read_write(v, sizeof(*v));
1026 return arch_atomic64_fetch_or(i, v);
1027}
1028
1029static __always_inline s64
1030atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1031{
1032 instrument_atomic_read_write(v, sizeof(*v));
1033 return arch_atomic64_fetch_or_acquire(i, v);
1034}
1035
1036static __always_inline s64
1037atomic64_fetch_or_release(s64 i, atomic64_t *v)
1038{
1039 kcsan_release();
1040 instrument_atomic_read_write(v, sizeof(*v));
1041 return arch_atomic64_fetch_or_release(i, v);
1042}
1043
1044static __always_inline s64
1045atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1046{
1047 instrument_atomic_read_write(v, sizeof(*v));
1048 return arch_atomic64_fetch_or_relaxed(i, v);
1049}
1050
1051static __always_inline void
1052atomic64_xor(s64 i, atomic64_t *v)
1053{
1054 instrument_atomic_read_write(v, sizeof(*v));
1055 arch_atomic64_xor(i, v);
1056}
1057
1058static __always_inline s64
1059atomic64_fetch_xor(s64 i, atomic64_t *v)
1060{
1061 kcsan_mb();
1062 instrument_atomic_read_write(v, sizeof(*v));
1063 return arch_atomic64_fetch_xor(i, v);
1064}
1065
1066static __always_inline s64
1067atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1068{
1069 instrument_atomic_read_write(v, sizeof(*v));
1070 return arch_atomic64_fetch_xor_acquire(i, v);
1071}
1072
1073static __always_inline s64
1074atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1075{
1076 kcsan_release();
1077 instrument_atomic_read_write(v, sizeof(*v));
1078 return arch_atomic64_fetch_xor_release(i, v);
1079}
1080
1081static __always_inline s64
1082atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1083{
1084 instrument_atomic_read_write(v, sizeof(*v));
1085 return arch_atomic64_fetch_xor_relaxed(i, v);
1086}
1087
1088static __always_inline s64
1089atomic64_xchg(atomic64_t *v, s64 i)
1090{
1091 kcsan_mb();
1092 instrument_atomic_read_write(v, sizeof(*v));
1093 return arch_atomic64_xchg(v, i);
1094}
1095
1096static __always_inline s64
1097atomic64_xchg_acquire(atomic64_t *v, s64 i)
1098{
1099 instrument_atomic_read_write(v, sizeof(*v));
1100 return arch_atomic64_xchg_acquire(v, i);
1101}
1102
1103static __always_inline s64
1104atomic64_xchg_release(atomic64_t *v, s64 i)
1105{
1106 kcsan_release();
1107 instrument_atomic_read_write(v, sizeof(*v));
1108 return arch_atomic64_xchg_release(v, i);
1109}
1110
1111static __always_inline s64
1112atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1113{
1114 instrument_atomic_read_write(v, sizeof(*v));
1115 return arch_atomic64_xchg_relaxed(v, i);
1116}
1117
1118static __always_inline s64
1119atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1120{
1121 kcsan_mb();
1122 instrument_atomic_read_write(v, sizeof(*v));
1123 return arch_atomic64_cmpxchg(v, old, new);
1124}
1125
1126static __always_inline s64
1127atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1128{
1129 instrument_atomic_read_write(v, sizeof(*v));
1130 return arch_atomic64_cmpxchg_acquire(v, old, new);
1131}
1132
1133static __always_inline s64
1134atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1135{
1136 kcsan_release();
1137 instrument_atomic_read_write(v, sizeof(*v));
1138 return arch_atomic64_cmpxchg_release(v, old, new);
1139}
1140
1141static __always_inline s64
1142atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1143{
1144 instrument_atomic_read_write(v, sizeof(*v));
1145 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1146}
1147
1148static __always_inline bool
1149atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1150{
1151 kcsan_mb();
1152 instrument_atomic_read_write(v, sizeof(*v));
1153 instrument_atomic_read_write(old, sizeof(*old));
1154 return arch_atomic64_try_cmpxchg(v, old, new);
1155}
1156
1157static __always_inline bool
1158atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1159{
1160 instrument_atomic_read_write(v, sizeof(*v));
1161 instrument_atomic_read_write(old, sizeof(*old));
1162 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1163}
1164
1165static __always_inline bool
1166atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1167{
1168 kcsan_release();
1169 instrument_atomic_read_write(v, sizeof(*v));
1170 instrument_atomic_read_write(old, sizeof(*old));
1171 return arch_atomic64_try_cmpxchg_release(v, old, new);
1172}
1173
1174static __always_inline bool
1175atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1176{
1177 instrument_atomic_read_write(v, sizeof(*v));
1178 instrument_atomic_read_write(old, sizeof(*old));
1179 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1180}
1181
1182static __always_inline bool
1183atomic64_sub_and_test(s64 i, atomic64_t *v)
1184{
1185 kcsan_mb();
1186 instrument_atomic_read_write(v, sizeof(*v));
1187 return arch_atomic64_sub_and_test(i, v);
1188}
1189
1190static __always_inline bool
1191atomic64_dec_and_test(atomic64_t *v)
1192{
1193 kcsan_mb();
1194 instrument_atomic_read_write(v, sizeof(*v));
1195 return arch_atomic64_dec_and_test(v);
1196}
1197
1198static __always_inline bool
1199atomic64_inc_and_test(atomic64_t *v)
1200{
1201 kcsan_mb();
1202 instrument_atomic_read_write(v, sizeof(*v));
1203 return arch_atomic64_inc_and_test(v);
1204}
1205
1206static __always_inline bool
1207atomic64_add_negative(s64 i, atomic64_t *v)
1208{
1209 kcsan_mb();
1210 instrument_atomic_read_write(v, sizeof(*v));
1211 return arch_atomic64_add_negative(i, v);
1212}
1213
1214static __always_inline s64
1215atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216{
1217 kcsan_mb();
1218 instrument_atomic_read_write(v, sizeof(*v));
1219 return arch_atomic64_fetch_add_unless(v, a, u);
1220}
1221
1222static __always_inline bool
1223atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1224{
1225 kcsan_mb();
1226 instrument_atomic_read_write(v, sizeof(*v));
1227 return arch_atomic64_add_unless(v, a, u);
1228}
1229
1230static __always_inline bool
1231atomic64_inc_not_zero(atomic64_t *v)
1232{
1233 kcsan_mb();
1234 instrument_atomic_read_write(v, sizeof(*v));
1235 return arch_atomic64_inc_not_zero(v);
1236}
1237
1238static __always_inline bool
1239atomic64_inc_unless_negative(atomic64_t *v)
1240{
1241 kcsan_mb();
1242 instrument_atomic_read_write(v, sizeof(*v));
1243 return arch_atomic64_inc_unless_negative(v);
1244}
1245
1246static __always_inline bool
1247atomic64_dec_unless_positive(atomic64_t *v)
1248{
1249 kcsan_mb();
1250 instrument_atomic_read_write(v, sizeof(*v));
1251 return arch_atomic64_dec_unless_positive(v);
1252}
1253
1254static __always_inline s64
1255atomic64_dec_if_positive(atomic64_t *v)
1256{
1257 kcsan_mb();
1258 instrument_atomic_read_write(v, sizeof(*v));
1259 return arch_atomic64_dec_if_positive(v);
1260}
1261
1262static __always_inline long
1263atomic_long_read(const atomic_long_t *v)
1264{
1265 instrument_atomic_read(v, sizeof(*v));
1266 return arch_atomic_long_read(v);
1267}
1268
1269static __always_inline long
1270atomic_long_read_acquire(const atomic_long_t *v)
1271{
1272 instrument_atomic_read(v, sizeof(*v));
1273 return arch_atomic_long_read_acquire(v);
1274}
1275
1276static __always_inline void
1277atomic_long_set(atomic_long_t *v, long i)
1278{
1279 instrument_atomic_write(v, sizeof(*v));
1280 arch_atomic_long_set(v, i);
1281}
1282
1283static __always_inline void
1284atomic_long_set_release(atomic_long_t *v, long i)
1285{
1286 kcsan_release();
1287 instrument_atomic_write(v, sizeof(*v));
1288 arch_atomic_long_set_release(v, i);
1289}
1290
1291static __always_inline void
1292atomic_long_add(long i, atomic_long_t *v)
1293{
1294 instrument_atomic_read_write(v, sizeof(*v));
1295 arch_atomic_long_add(i, v);
1296}
1297
1298static __always_inline long
1299atomic_long_add_return(long i, atomic_long_t *v)
1300{
1301 kcsan_mb();
1302 instrument_atomic_read_write(v, sizeof(*v));
1303 return arch_atomic_long_add_return(i, v);
1304}
1305
1306static __always_inline long
1307atomic_long_add_return_acquire(long i, atomic_long_t *v)
1308{
1309 instrument_atomic_read_write(v, sizeof(*v));
1310 return arch_atomic_long_add_return_acquire(i, v);
1311}
1312
1313static __always_inline long
1314atomic_long_add_return_release(long i, atomic_long_t *v)
1315{
1316 kcsan_release();
1317 instrument_atomic_read_write(v, sizeof(*v));
1318 return arch_atomic_long_add_return_release(i, v);
1319}
1320
1321static __always_inline long
1322atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1323{
1324 instrument_atomic_read_write(v, sizeof(*v));
1325 return arch_atomic_long_add_return_relaxed(i, v);
1326}
1327
1328static __always_inline long
1329atomic_long_fetch_add(long i, atomic_long_t *v)
1330{
1331 kcsan_mb();
1332 instrument_atomic_read_write(v, sizeof(*v));
1333 return arch_atomic_long_fetch_add(i, v);
1334}
1335
1336static __always_inline long
1337atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1338{
1339 instrument_atomic_read_write(v, sizeof(*v));
1340 return arch_atomic_long_fetch_add_acquire(i, v);
1341}
1342
1343static __always_inline long
1344atomic_long_fetch_add_release(long i, atomic_long_t *v)
1345{
1346 kcsan_release();
1347 instrument_atomic_read_write(v, sizeof(*v));
1348 return arch_atomic_long_fetch_add_release(i, v);
1349}
1350
1351static __always_inline long
1352atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1353{
1354 instrument_atomic_read_write(v, sizeof(*v));
1355 return arch_atomic_long_fetch_add_relaxed(i, v);
1356}
1357
1358static __always_inline void
1359atomic_long_sub(long i, atomic_long_t *v)
1360{
1361 instrument_atomic_read_write(v, sizeof(*v));
1362 arch_atomic_long_sub(i, v);
1363}
1364
1365static __always_inline long
1366atomic_long_sub_return(long i, atomic_long_t *v)
1367{
1368 kcsan_mb();
1369 instrument_atomic_read_write(v, sizeof(*v));
1370 return arch_atomic_long_sub_return(i, v);
1371}
1372
1373static __always_inline long
1374atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1375{
1376 instrument_atomic_read_write(v, sizeof(*v));
1377 return arch_atomic_long_sub_return_acquire(i, v);
1378}
1379
1380static __always_inline long
1381atomic_long_sub_return_release(long i, atomic_long_t *v)
1382{
1383 kcsan_release();
1384 instrument_atomic_read_write(v, sizeof(*v));
1385 return arch_atomic_long_sub_return_release(i, v);
1386}
1387
1388static __always_inline long
1389atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1390{
1391 instrument_atomic_read_write(v, sizeof(*v));
1392 return arch_atomic_long_sub_return_relaxed(i, v);
1393}
1394
1395static __always_inline long
1396atomic_long_fetch_sub(long i, atomic_long_t *v)
1397{
1398 kcsan_mb();
1399 instrument_atomic_read_write(v, sizeof(*v));
1400 return arch_atomic_long_fetch_sub(i, v);
1401}
1402
1403static __always_inline long
1404atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1405{
1406 instrument_atomic_read_write(v, sizeof(*v));
1407 return arch_atomic_long_fetch_sub_acquire(i, v);
1408}
1409
1410static __always_inline long
1411atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1412{
1413 kcsan_release();
1414 instrument_atomic_read_write(v, sizeof(*v));
1415 return arch_atomic_long_fetch_sub_release(i, v);
1416}
1417
1418static __always_inline long
1419atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1420{
1421 instrument_atomic_read_write(v, sizeof(*v));
1422 return arch_atomic_long_fetch_sub_relaxed(i, v);
1423}
1424
1425static __always_inline void
1426atomic_long_inc(atomic_long_t *v)
1427{
1428 instrument_atomic_read_write(v, sizeof(*v));
1429 arch_atomic_long_inc(v);
1430}
1431
1432static __always_inline long
1433atomic_long_inc_return(atomic_long_t *v)
1434{
1435 kcsan_mb();
1436 instrument_atomic_read_write(v, sizeof(*v));
1437 return arch_atomic_long_inc_return(v);
1438}
1439
1440static __always_inline long
1441atomic_long_inc_return_acquire(atomic_long_t *v)
1442{
1443 instrument_atomic_read_write(v, sizeof(*v));
1444 return arch_atomic_long_inc_return_acquire(v);
1445}
1446
1447static __always_inline long
1448atomic_long_inc_return_release(atomic_long_t *v)
1449{
1450 kcsan_release();
1451 instrument_atomic_read_write(v, sizeof(*v));
1452 return arch_atomic_long_inc_return_release(v);
1453}
1454
1455static __always_inline long
1456atomic_long_inc_return_relaxed(atomic_long_t *v)
1457{
1458 instrument_atomic_read_write(v, sizeof(*v));
1459 return arch_atomic_long_inc_return_relaxed(v);
1460}
1461
1462static __always_inline long
1463atomic_long_fetch_inc(atomic_long_t *v)
1464{
1465 kcsan_mb();
1466 instrument_atomic_read_write(v, sizeof(*v));
1467 return arch_atomic_long_fetch_inc(v);
1468}
1469
1470static __always_inline long
1471atomic_long_fetch_inc_acquire(atomic_long_t *v)
1472{
1473 instrument_atomic_read_write(v, sizeof(*v));
1474 return arch_atomic_long_fetch_inc_acquire(v);
1475}
1476
1477static __always_inline long
1478atomic_long_fetch_inc_release(atomic_long_t *v)
1479{
1480 kcsan_release();
1481 instrument_atomic_read_write(v, sizeof(*v));
1482 return arch_atomic_long_fetch_inc_release(v);
1483}
1484
1485static __always_inline long
1486atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1487{
1488 instrument_atomic_read_write(v, sizeof(*v));
1489 return arch_atomic_long_fetch_inc_relaxed(v);
1490}
1491
1492static __always_inline void
1493atomic_long_dec(atomic_long_t *v)
1494{
1495 instrument_atomic_read_write(v, sizeof(*v));
1496 arch_atomic_long_dec(v);
1497}
1498
1499static __always_inline long
1500atomic_long_dec_return(atomic_long_t *v)
1501{
1502 kcsan_mb();
1503 instrument_atomic_read_write(v, sizeof(*v));
1504 return arch_atomic_long_dec_return(v);
1505}
1506
1507static __always_inline long
1508atomic_long_dec_return_acquire(atomic_long_t *v)
1509{
1510 instrument_atomic_read_write(v, sizeof(*v));
1511 return arch_atomic_long_dec_return_acquire(v);
1512}
1513
1514static __always_inline long
1515atomic_long_dec_return_release(atomic_long_t *v)
1516{
1517 kcsan_release();
1518 instrument_atomic_read_write(v, sizeof(*v));
1519 return arch_atomic_long_dec_return_release(v);
1520}
1521
1522static __always_inline long
1523atomic_long_dec_return_relaxed(atomic_long_t *v)
1524{
1525 instrument_atomic_read_write(v, sizeof(*v));
1526 return arch_atomic_long_dec_return_relaxed(v);
1527}
1528
1529static __always_inline long
1530atomic_long_fetch_dec(atomic_long_t *v)
1531{
1532 kcsan_mb();
1533 instrument_atomic_read_write(v, sizeof(*v));
1534 return arch_atomic_long_fetch_dec(v);
1535}
1536
1537static __always_inline long
1538atomic_long_fetch_dec_acquire(atomic_long_t *v)
1539{
1540 instrument_atomic_read_write(v, sizeof(*v));
1541 return arch_atomic_long_fetch_dec_acquire(v);
1542}
1543
1544static __always_inline long
1545atomic_long_fetch_dec_release(atomic_long_t *v)
1546{
1547 kcsan_release();
1548 instrument_atomic_read_write(v, sizeof(*v));
1549 return arch_atomic_long_fetch_dec_release(v);
1550}
1551
1552static __always_inline long
1553atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1554{
1555 instrument_atomic_read_write(v, sizeof(*v));
1556 return arch_atomic_long_fetch_dec_relaxed(v);
1557}
1558
1559static __always_inline void
1560atomic_long_and(long i, atomic_long_t *v)
1561{
1562 instrument_atomic_read_write(v, sizeof(*v));
1563 arch_atomic_long_and(i, v);
1564}
1565
1566static __always_inline long
1567atomic_long_fetch_and(long i, atomic_long_t *v)
1568{
1569 kcsan_mb();
1570 instrument_atomic_read_write(v, sizeof(*v));
1571 return arch_atomic_long_fetch_and(i, v);
1572}
1573
1574static __always_inline long
1575atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1576{
1577 instrument_atomic_read_write(v, sizeof(*v));
1578 return arch_atomic_long_fetch_and_acquire(i, v);
1579}
1580
1581static __always_inline long
1582atomic_long_fetch_and_release(long i, atomic_long_t *v)
1583{
1584 kcsan_release();
1585 instrument_atomic_read_write(v, sizeof(*v));
1586 return arch_atomic_long_fetch_and_release(i, v);
1587}
1588
1589static __always_inline long
1590atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1591{
1592 instrument_atomic_read_write(v, sizeof(*v));
1593 return arch_atomic_long_fetch_and_relaxed(i, v);
1594}
1595
1596static __always_inline void
1597atomic_long_andnot(long i, atomic_long_t *v)
1598{
1599 instrument_atomic_read_write(v, sizeof(*v));
1600 arch_atomic_long_andnot(i, v);
1601}
1602
1603static __always_inline long
1604atomic_long_fetch_andnot(long i, atomic_long_t *v)
1605{
1606 kcsan_mb();
1607 instrument_atomic_read_write(v, sizeof(*v));
1608 return arch_atomic_long_fetch_andnot(i, v);
1609}
1610
1611static __always_inline long
1612atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1613{
1614 instrument_atomic_read_write(v, sizeof(*v));
1615 return arch_atomic_long_fetch_andnot_acquire(i, v);
1616}
1617
1618static __always_inline long
1619atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1620{
1621 kcsan_release();
1622 instrument_atomic_read_write(v, sizeof(*v));
1623 return arch_atomic_long_fetch_andnot_release(i, v);
1624}
1625
1626static __always_inline long
1627atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1628{
1629 instrument_atomic_read_write(v, sizeof(*v));
1630 return arch_atomic_long_fetch_andnot_relaxed(i, v);
1631}
1632
1633static __always_inline void
1634atomic_long_or(long i, atomic_long_t *v)
1635{
1636 instrument_atomic_read_write(v, sizeof(*v));
1637 arch_atomic_long_or(i, v);
1638}
1639
1640static __always_inline long
1641atomic_long_fetch_or(long i, atomic_long_t *v)
1642{
1643 kcsan_mb();
1644 instrument_atomic_read_write(v, sizeof(*v));
1645 return arch_atomic_long_fetch_or(i, v);
1646}
1647
1648static __always_inline long
1649atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1650{
1651 instrument_atomic_read_write(v, sizeof(*v));
1652 return arch_atomic_long_fetch_or_acquire(i, v);
1653}
1654
1655static __always_inline long
1656atomic_long_fetch_or_release(long i, atomic_long_t *v)
1657{
1658 kcsan_release();
1659 instrument_atomic_read_write(v, sizeof(*v));
1660 return arch_atomic_long_fetch_or_release(i, v);
1661}
1662
1663static __always_inline long
1664atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1665{
1666 instrument_atomic_read_write(v, sizeof(*v));
1667 return arch_atomic_long_fetch_or_relaxed(i, v);
1668}
1669
1670static __always_inline void
1671atomic_long_xor(long i, atomic_long_t *v)
1672{
1673 instrument_atomic_read_write(v, sizeof(*v));
1674 arch_atomic_long_xor(i, v);
1675}
1676
1677static __always_inline long
1678atomic_long_fetch_xor(long i, atomic_long_t *v)
1679{
1680 kcsan_mb();
1681 instrument_atomic_read_write(v, sizeof(*v));
1682 return arch_atomic_long_fetch_xor(i, v);
1683}
1684
1685static __always_inline long
1686atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1687{
1688 instrument_atomic_read_write(v, sizeof(*v));
1689 return arch_atomic_long_fetch_xor_acquire(i, v);
1690}
1691
1692static __always_inline long
1693atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1694{
1695 kcsan_release();
1696 instrument_atomic_read_write(v, sizeof(*v));
1697 return arch_atomic_long_fetch_xor_release(i, v);
1698}
1699
1700static __always_inline long
1701atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1702{
1703 instrument_atomic_read_write(v, sizeof(*v));
1704 return arch_atomic_long_fetch_xor_relaxed(i, v);
1705}
1706
1707static __always_inline long
1708atomic_long_xchg(atomic_long_t *v, long i)
1709{
1710 kcsan_mb();
1711 instrument_atomic_read_write(v, sizeof(*v));
1712 return arch_atomic_long_xchg(v, i);
1713}
1714
1715static __always_inline long
1716atomic_long_xchg_acquire(atomic_long_t *v, long i)
1717{
1718 instrument_atomic_read_write(v, sizeof(*v));
1719 return arch_atomic_long_xchg_acquire(v, i);
1720}
1721
1722static __always_inline long
1723atomic_long_xchg_release(atomic_long_t *v, long i)
1724{
1725 kcsan_release();
1726 instrument_atomic_read_write(v, sizeof(*v));
1727 return arch_atomic_long_xchg_release(v, i);
1728}
1729
1730static __always_inline long
1731atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1732{
1733 instrument_atomic_read_write(v, sizeof(*v));
1734 return arch_atomic_long_xchg_relaxed(v, i);
1735}
1736
1737static __always_inline long
1738atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1739{
1740 kcsan_mb();
1741 instrument_atomic_read_write(v, sizeof(*v));
1742 return arch_atomic_long_cmpxchg(v, old, new);
1743}
1744
1745static __always_inline long
1746atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1747{
1748 instrument_atomic_read_write(v, sizeof(*v));
1749 return arch_atomic_long_cmpxchg_acquire(v, old, new);
1750}
1751
1752static __always_inline long
1753atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1754{
1755 kcsan_release();
1756 instrument_atomic_read_write(v, sizeof(*v));
1757 return arch_atomic_long_cmpxchg_release(v, old, new);
1758}
1759
1760static __always_inline long
1761atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1762{
1763 instrument_atomic_read_write(v, sizeof(*v));
1764 return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1765}
1766
1767static __always_inline bool
1768atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1769{
1770 kcsan_mb();
1771 instrument_atomic_read_write(v, sizeof(*v));
1772 instrument_atomic_read_write(old, sizeof(*old));
1773 return arch_atomic_long_try_cmpxchg(v, old, new);
1774}
1775
1776static __always_inline bool
1777atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1778{
1779 instrument_atomic_read_write(v, sizeof(*v));
1780 instrument_atomic_read_write(old, sizeof(*old));
1781 return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1782}
1783
1784static __always_inline bool
1785atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1786{
1787 kcsan_release();
1788 instrument_atomic_read_write(v, sizeof(*v));
1789 instrument_atomic_read_write(old, sizeof(*old));
1790 return arch_atomic_long_try_cmpxchg_release(v, old, new);
1791}
1792
1793static __always_inline bool
1794atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1795{
1796 instrument_atomic_read_write(v, sizeof(*v));
1797 instrument_atomic_read_write(old, sizeof(*old));
1798 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1799}
1800
1801static __always_inline bool
1802atomic_long_sub_and_test(long i, atomic_long_t *v)
1803{
1804 kcsan_mb();
1805 instrument_atomic_read_write(v, sizeof(*v));
1806 return arch_atomic_long_sub_and_test(i, v);
1807}
1808
1809static __always_inline bool
1810atomic_long_dec_and_test(atomic_long_t *v)
1811{
1812 kcsan_mb();
1813 instrument_atomic_read_write(v, sizeof(*v));
1814 return arch_atomic_long_dec_and_test(v);
1815}
1816
1817static __always_inline bool
1818atomic_long_inc_and_test(atomic_long_t *v)
1819{
1820 kcsan_mb();
1821 instrument_atomic_read_write(v, sizeof(*v));
1822 return arch_atomic_long_inc_and_test(v);
1823}
1824
1825static __always_inline bool
1826atomic_long_add_negative(long i, atomic_long_t *v)
1827{
1828 kcsan_mb();
1829 instrument_atomic_read_write(v, sizeof(*v));
1830 return arch_atomic_long_add_negative(i, v);
1831}
1832
1833static __always_inline long
1834atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835{
1836 kcsan_mb();
1837 instrument_atomic_read_write(v, sizeof(*v));
1838 return arch_atomic_long_fetch_add_unless(v, a, u);
1839}
1840
1841static __always_inline bool
1842atomic_long_add_unless(atomic_long_t *v, long a, long u)
1843{
1844 kcsan_mb();
1845 instrument_atomic_read_write(v, sizeof(*v));
1846 return arch_atomic_long_add_unless(v, a, u);
1847}
1848
1849static __always_inline bool
1850atomic_long_inc_not_zero(atomic_long_t *v)
1851{
1852 kcsan_mb();
1853 instrument_atomic_read_write(v, sizeof(*v));
1854 return arch_atomic_long_inc_not_zero(v);
1855}
1856
1857static __always_inline bool
1858atomic_long_inc_unless_negative(atomic_long_t *v)
1859{
1860 kcsan_mb();
1861 instrument_atomic_read_write(v, sizeof(*v));
1862 return arch_atomic_long_inc_unless_negative(v);
1863}
1864
1865static __always_inline bool
1866atomic_long_dec_unless_positive(atomic_long_t *v)
1867{
1868 kcsan_mb();
1869 instrument_atomic_read_write(v, sizeof(*v));
1870 return arch_atomic_long_dec_unless_positive(v);
1871}
1872
1873static __always_inline long
1874atomic_long_dec_if_positive(atomic_long_t *v)
1875{
1876 kcsan_mb();
1877 instrument_atomic_read_write(v, sizeof(*v));
1878 return arch_atomic_long_dec_if_positive(v);
1879}
1880
1881#define xchg(ptr, ...) \
1882({ \
1883 typeof(ptr) __ai_ptr = (ptr); \
1884 kcsan_mb(); \
1885 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886 arch_xchg(__ai_ptr, __VA_ARGS__); \
1887})
1888
1889#define xchg_acquire(ptr, ...) \
1890({ \
1891 typeof(ptr) __ai_ptr = (ptr); \
1892 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1894})
1895
1896#define xchg_release(ptr, ...) \
1897({ \
1898 typeof(ptr) __ai_ptr = (ptr); \
1899 kcsan_release(); \
1900 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901 arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1902})
1903
1904#define xchg_relaxed(ptr, ...) \
1905({ \
1906 typeof(ptr) __ai_ptr = (ptr); \
1907 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1909})
1910
1911#define cmpxchg(ptr, ...) \
1912({ \
1913 typeof(ptr) __ai_ptr = (ptr); \
1914 kcsan_mb(); \
1915 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1917})
1918
1919#define cmpxchg_acquire(ptr, ...) \
1920({ \
1921 typeof(ptr) __ai_ptr = (ptr); \
1922 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1924})
1925
1926#define cmpxchg_release(ptr, ...) \
1927({ \
1928 typeof(ptr) __ai_ptr = (ptr); \
1929 kcsan_release(); \
1930 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1932})
1933
1934#define cmpxchg_relaxed(ptr, ...) \
1935({ \
1936 typeof(ptr) __ai_ptr = (ptr); \
1937 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1939})
1940
1941#define cmpxchg64(ptr, ...) \
1942({ \
1943 typeof(ptr) __ai_ptr = (ptr); \
1944 kcsan_mb(); \
1945 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1947})
1948
1949#define cmpxchg64_acquire(ptr, ...) \
1950({ \
1951 typeof(ptr) __ai_ptr = (ptr); \
1952 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1954})
1955
1956#define cmpxchg64_release(ptr, ...) \
1957({ \
1958 typeof(ptr) __ai_ptr = (ptr); \
1959 kcsan_release(); \
1960 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1962})
1963
1964#define cmpxchg64_relaxed(ptr, ...) \
1965({ \
1966 typeof(ptr) __ai_ptr = (ptr); \
1967 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1969})
1970
1971#define try_cmpxchg(ptr, oldp, ...) \
1972({ \
1973 typeof(ptr) __ai_ptr = (ptr); \
1974 typeof(oldp) __ai_oldp = (oldp); \
1975 kcsan_mb(); \
1976 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1979})
1980
1981#define try_cmpxchg_acquire(ptr, oldp, ...) \
1982({ \
1983 typeof(ptr) __ai_ptr = (ptr); \
1984 typeof(oldp) __ai_oldp = (oldp); \
1985 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1988})
1989
1990#define try_cmpxchg_release(ptr, oldp, ...) \
1991({ \
1992 typeof(ptr) __ai_ptr = (ptr); \
1993 typeof(oldp) __ai_oldp = (oldp); \
1994 kcsan_release(); \
1995 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1998})
1999
2000#define try_cmpxchg_relaxed(ptr, oldp, ...) \
2001({ \
2002 typeof(ptr) __ai_ptr = (ptr); \
2003 typeof(oldp) __ai_oldp = (oldp); \
2004 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2007})
2008
2009#define try_cmpxchg64(ptr, oldp, ...) \
2010({ \
2011 typeof(ptr) __ai_ptr = (ptr); \
2012 typeof(oldp) __ai_oldp = (oldp); \
2013 kcsan_mb(); \
2014 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016 arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017})
2018
2019#define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020({ \
2021 typeof(ptr) __ai_ptr = (ptr); \
2022 typeof(oldp) __ai_oldp = (oldp); \
2023 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025 arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026})
2027
2028#define try_cmpxchg64_release(ptr, oldp, ...) \
2029({ \
2030 typeof(ptr) __ai_ptr = (ptr); \
2031 typeof(oldp) __ai_oldp = (oldp); \
2032 kcsan_release(); \
2033 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035 arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036})
2037
2038#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039({ \
2040 typeof(ptr) __ai_ptr = (ptr); \
2041 typeof(oldp) __ai_oldp = (oldp); \
2042 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044 arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045})
2046
2047#define cmpxchg_local(ptr, ...) \
2048({ \
2049 typeof(ptr) __ai_ptr = (ptr); \
2050 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2051 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2052})
2053
2054#define cmpxchg64_local(ptr, ...) \
2055({ \
2056 typeof(ptr) __ai_ptr = (ptr); \
2057 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2058 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2059})
2060
2061#define sync_cmpxchg(ptr, ...) \
2062({ \
2063 typeof(ptr) __ai_ptr = (ptr); \
2064 kcsan_mb(); \
2065 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2066 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2067})
2068
2069#define cmpxchg_double(ptr, ...) \
2070({ \
2071 typeof(ptr) __ai_ptr = (ptr); \
2072 kcsan_mb(); \
2073 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2074 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2075})
2076
2077
2078#define cmpxchg_double_local(ptr, ...) \
2079({ \
2080 typeof(ptr) __ai_ptr = (ptr); \
2081 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2082 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2083})
2084
2085#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086// 764f741eb77a7ad565dc8d99ce2837d5542e8aee
2087

source code of linux/include/linux/atomic/atomic-instrumented.h